github.com/demonoid81/moby@v0.0.0-20200517203328-62dd8e17c460/pkg/tailfile/tailfile.go (about) 1 // Package tailfile provides helper functions to read the nth lines of any 2 // ReadSeeker. 3 package tailfile // import "github.com/demonoid81/moby/pkg/tailfile" 4 5 import ( 6 "bufio" 7 "bytes" 8 "context" 9 "errors" 10 "io" 11 "os" 12 ) 13 14 const blockSize = 1024 15 16 var eol = []byte("\n") 17 18 // ErrNonPositiveLinesNumber is an error returned if the lines number was negative. 19 var ErrNonPositiveLinesNumber = errors.New("The number of lines to extract from the file must be positive") 20 21 // TailFile returns last n lines of the passed in file. 22 func TailFile(f *os.File, n int) ([][]byte, error) { 23 size, err := f.Seek(0, io.SeekEnd) 24 if err != nil { 25 return nil, err 26 } 27 28 rAt := io.NewSectionReader(f, 0, size) 29 r, nLines, err := NewTailReader(context.Background(), rAt, n) 30 if err != nil { 31 return nil, err 32 } 33 34 buf := make([][]byte, 0, nLines) 35 scanner := bufio.NewScanner(r) 36 37 for scanner.Scan() { 38 buf = append(buf, scanner.Bytes()) 39 } 40 return buf, nil 41 } 42 43 // SizeReaderAt is an interface used to get a ReaderAt as well as the size of the underlying reader. 44 // Note that the size of the underlying reader should not change when using this interface. 45 type SizeReaderAt interface { 46 io.ReaderAt 47 Size() int64 48 } 49 50 // NewTailReader scopes the passed in reader to just the last N lines passed in 51 func NewTailReader(ctx context.Context, r SizeReaderAt, reqLines int) (io.Reader, int, error) { 52 return NewTailReaderWithDelimiter(ctx, r, reqLines, eol) 53 } 54 55 // NewTailReaderWithDelimiter scopes the passed in reader to just the last N lines passed in 56 // In this case a "line" is defined by the passed in delimiter. 57 // 58 // Delimiter lengths should be generally small, no more than 12 bytes 59 func NewTailReaderWithDelimiter(ctx context.Context, r SizeReaderAt, reqLines int, delimiter []byte) (io.Reader, int, error) { 60 if reqLines < 1 { 61 return nil, 0, ErrNonPositiveLinesNumber 62 } 63 if len(delimiter) == 0 { 64 return nil, 0, errors.New("must provide a delimiter") 65 } 66 var ( 67 size = r.Size() 68 tailStart int64 69 tailEnd = size 70 found int 71 ) 72 73 if int64(len(delimiter)) >= size { 74 return bytes.NewReader(nil), 0, nil 75 } 76 77 scanner := newScanner(r, delimiter) 78 for scanner.Scan(ctx) { 79 if err := scanner.Err(); err != nil { 80 return nil, 0, scanner.Err() 81 } 82 83 found++ 84 if found == 1 { 85 tailEnd = scanner.End() 86 } 87 if found == reqLines { 88 break 89 } 90 } 91 92 tailStart = scanner.Start(ctx) 93 94 if found == 0 { 95 return bytes.NewReader(nil), 0, nil 96 } 97 98 if found < reqLines && tailStart != 0 { 99 tailStart = 0 100 } 101 return io.NewSectionReader(r, tailStart, tailEnd-tailStart), found, nil 102 } 103 104 func newScanner(r SizeReaderAt, delim []byte) *scanner { 105 size := r.Size() 106 readSize := blockSize 107 if readSize > int(size) { 108 readSize = int(size) 109 } 110 // silly case... 111 if len(delim) >= readSize/2 { 112 readSize = len(delim)*2 + 2 113 } 114 115 return &scanner{ 116 r: r, 117 pos: size, 118 buf: make([]byte, readSize), 119 delim: delim, 120 } 121 } 122 123 type scanner struct { 124 r SizeReaderAt 125 pos int64 126 buf []byte 127 delim []byte 128 err error 129 idx int 130 } 131 132 func (s *scanner) Start(ctx context.Context) int64 { 133 if s.idx > 0 { 134 idx := bytes.LastIndex(s.buf[:s.idx], s.delim) 135 if idx >= 0 { 136 return s.pos + int64(idx) + int64(len(s.delim)) 137 } 138 } 139 140 // slow path 141 buf := make([]byte, len(s.buf)) 142 copy(buf, s.buf) 143 144 readAhead := &scanner{ 145 r: s.r, 146 pos: s.pos, 147 delim: s.delim, 148 idx: s.idx, 149 buf: buf, 150 } 151 152 if !readAhead.Scan(ctx) { 153 return 0 154 } 155 return readAhead.End() 156 } 157 158 func (s *scanner) End() int64 { 159 return s.pos + int64(s.idx) + int64(len(s.delim)) 160 } 161 162 func (s *scanner) Err() error { 163 return s.err 164 } 165 166 func (s *scanner) Scan(ctx context.Context) bool { 167 if s.err != nil { 168 return false 169 } 170 171 for { 172 select { 173 case <-ctx.Done(): 174 s.err = ctx.Err() 175 return false 176 default: 177 } 178 179 idx := s.idx - len(s.delim) 180 if idx < 0 { 181 readSize := int(s.pos) 182 if readSize > len(s.buf) { 183 readSize = len(s.buf) 184 } 185 186 if readSize < len(s.delim) { 187 return false 188 } 189 190 offset := s.pos - int64(readSize) 191 n, err := s.r.ReadAt(s.buf[:readSize], offset) 192 if err != nil && err != io.EOF { 193 s.err = err 194 return false 195 } 196 197 s.pos -= int64(n) 198 idx = n 199 } 200 201 s.idx = bytes.LastIndex(s.buf[:idx], s.delim) 202 if s.idx >= 0 { 203 return true 204 } 205 206 if len(s.delim) > 1 && s.pos > 0 { 207 // in this case, there may be a partial delimiter at the front of the buffer, so set the position forward 208 // up to the maximum size partial that could be there so it can be read again in the next iteration with any 209 // potential remainder. 210 // An example where delimiter is `####`: 211 // [##asdfqwerty] 212 // ^ 213 // This resets the position to where the arrow is pointing. 214 // It could actually check if a partial exists and at the front, but that is pretty similar to the indexing 215 // code above though a bit more complex since each byte has to be checked (`len(delimiter)-1`) factorial). 216 // It's much simpler and cleaner to just re-read `len(delimiter)-1` bytes again. 217 s.pos += int64(len(s.delim)) - 1 218 } 219 220 } 221 }