github.com/foolbread/tail@v1.0.1-0.20170719092341-aac4b99b272b/tail.go (about) 1 // Copyright (c) 2015 HPE Software Inc. All rights reserved. 2 // Copyright (c) 2013 ActiveState Software Inc. All rights reserved. 3 4 package tail 5 6 import ( 7 "bufio" 8 "errors" 9 "fmt" 10 "io" 11 "io/ioutil" 12 "log" 13 "os" 14 "strings" 15 "sync" 16 "time" 17 18 "github.com/foolbread/tail/ratelimiter" 19 "github.com/foolbread/tail/util" 20 "github.com/foolbread/tail/watch" 21 "gopkg.in/tomb.v1" 22 ) 23 24 var ( 25 ErrStop = errors.New("tail should now stop") 26 ) 27 28 const ( 29 unknowOffset = -1 30 ) 31 32 type Line struct { 33 Text string 34 Time time.Time 35 Err error // Error from tail 36 Offset int64 // line offset 37 } 38 39 // NewLine returns a Line with present time. 40 func NewLine(text string) *Line { 41 return &Line{text, time.Now(), nil,unknowOffset} 42 } 43 44 // SeekInfo represents arguments to `os.Seek` 45 type SeekInfo struct { 46 Offset int64 47 Whence int // os.SEEK_* 48 } 49 50 type logger interface { 51 Fatal(v ...interface{}) 52 Fatalf(format string, v ...interface{}) 53 Fatalln(v ...interface{}) 54 Panic(v ...interface{}) 55 Panicf(format string, v ...interface{}) 56 Panicln(v ...interface{}) 57 Print(v ...interface{}) 58 Printf(format string, v ...interface{}) 59 Println(v ...interface{}) 60 } 61 62 // Config is used to specify how a file must be tailed. 63 type Config struct { 64 // File-specifc 65 Location *SeekInfo // Seek to this location before tailing 66 ReOpen bool // Reopen recreated files (tail -F) 67 MustExist bool // Fail early if the file does not exist 68 Poll bool // Poll for file changes instead of using inotify 69 Pipe bool // Is a named pipe (mkfifo) 70 RateLimiter *ratelimiter.LeakyBucket 71 72 // Generic IO 73 Follow bool // Continue looking for new lines (tail -f) 74 MaxLineSize int // If non-zero, split longer lines into multiple lines 75 76 // Logger, when nil, is set to tail.DefaultLogger 77 // To disable logging: set field to tail.DiscardingLogger 78 Logger logger 79 } 80 81 type Tail struct { 82 Filename string 83 Lines chan *Line 84 Config 85 86 file *os.File 87 reader *bufio.Reader 88 89 watcher watch.FileWatcher 90 changes *watch.FileChanges 91 92 tomb.Tomb // provides: Done, Kill, Dying 93 94 lk sync.Mutex 95 96 offset int64 97 } 98 99 var ( 100 // DefaultLogger is used when Config.Logger == nil 101 DefaultLogger = log.New(os.Stderr, "", log.LstdFlags) 102 // DiscardingLogger can be used to disable logging output 103 DiscardingLogger = log.New(ioutil.Discard, "", 0) 104 ) 105 106 // TailFile begins tailing the file. Output stream is made available 107 // via the `Tail.Lines` channel. To handle errors during tailing, 108 // invoke the `Wait` or `Err` method after finishing reading from the 109 // `Lines` channel. 110 func TailFile(filename string, config Config) (*Tail, error) { 111 if config.ReOpen && !config.Follow { 112 util.Fatal("cannot set ReOpen without Follow.") 113 } 114 115 t := &Tail{ 116 Filename: filename, 117 Lines: make(chan *Line), 118 Config: config, 119 } 120 121 // when Logger was not specified in config, use default logger 122 if t.Logger == nil { 123 t.Logger = log.New(os.Stderr, "", log.LstdFlags) 124 } 125 126 if t.Poll { 127 t.watcher = watch.NewPollingFileWatcher(filename) 128 } else { 129 t.watcher = watch.NewInotifyFileWatcher(filename) 130 } 131 132 if t.MustExist { 133 var err error 134 t.file, err = OpenFile(t.Filename) 135 if err != nil { 136 return nil, err 137 } 138 } 139 140 go t.tailFileSync() 141 142 return t, nil 143 } 144 145 // Return the file's current position, like stdio's ftell(). 146 // But this value is not very accurate. 147 // it may readed one line in the chan(tail.Lines), 148 // so it may lost one line. 149 func (tail *Tail) Tell() (offset int64, err error) { 150 if tail.file == nil { 151 return 152 } 153 offset, err = tail.file.Seek(0, os.SEEK_CUR) 154 if err != nil { 155 return 156 } 157 158 tail.lk.Lock() 159 defer tail.lk.Unlock() 160 if tail.reader == nil { 161 return 162 } 163 164 offset -= int64(tail.reader.Buffered()) 165 return 166 } 167 168 // Stop stops the tailing activity. 169 func (tail *Tail) Stop() error { 170 tail.Kill(nil) 171 return tail.Wait() 172 } 173 174 // StopAtEOF stops tailing as soon as the end of the file is reached. 175 func (tail *Tail) StopAtEOF() error { 176 tail.Kill(errStopAtEOF) 177 return tail.Wait() 178 } 179 180 var errStopAtEOF = errors.New("tail: stop at eof") 181 182 func (tail *Tail) close() { 183 close(tail.Lines) 184 tail.closeFile() 185 } 186 187 func (tail *Tail) closeFile() { 188 if tail.file != nil { 189 tail.file.Close() 190 tail.file = nil 191 } 192 } 193 194 func (tail *Tail) reopen() error { 195 tail.closeFile() 196 for { 197 var err error 198 tail.file, err = OpenFile(tail.Filename) 199 if err != nil { 200 if os.IsNotExist(err) { 201 tail.Logger.Printf("Waiting for %s to appear...", tail.Filename) 202 if err := tail.watcher.BlockUntilExists(&tail.Tomb); err != nil { 203 if err == tomb.ErrDying { 204 return err 205 } 206 return fmt.Errorf("Failed to detect creation of %s: %s", tail.Filename, err) 207 } 208 continue 209 } 210 return fmt.Errorf("Unable to open file %s: %s", tail.Filename, err) 211 } 212 213 tail.offset,err = tail.file.Seek(0,os.SEEK_CUR) 214 if err != nil{ 215 return fmt.Errorf("get new offset %s: %s", tail.Filename, err) 216 } 217 break 218 } 219 return nil 220 } 221 222 func (tail *Tail) readLine() (string, error) { 223 tail.lk.Lock() 224 line, err := tail.reader.ReadString('\n') 225 tail.lk.Unlock() 226 if err != nil { 227 // Note ReadString "returns the data read before the error" in 228 // case of an error, including EOF, so we return it as is. The 229 // caller is expected to process it if err is EOF. 230 return line, err 231 } 232 233 line = strings.TrimRight(line, "\n") 234 235 return line, err 236 } 237 238 func (tail *Tail) tailFileSync() { 239 defer tail.Done() 240 defer tail.close() 241 242 if !tail.MustExist { 243 // deferred first open. 244 err := tail.reopen() 245 if err != nil { 246 if err != tomb.ErrDying { 247 tail.Kill(err) 248 } 249 return 250 } 251 } 252 253 // Seek to requested location on first open of the file. 254 if tail.Location != nil { 255 var err error 256 tail.offset, err = tail.file.Seek(tail.Location.Offset, tail.Location.Whence) 257 tail.Logger.Printf("Seeked %s - %+v\n", tail.Filename, tail.Location) 258 if err != nil { 259 tail.Killf("Seek error on %s: %s", tail.Filename, err) 260 return 261 } 262 } 263 264 tail.openReader() 265 266 var offset int64 267 var err error 268 269 // Read line by line. 270 for { 271 // do not seek in named pipes 272 if !tail.Pipe { 273 // grab the position in case we need to back up in the event of a half-line 274 offset, err = tail.Tell() 275 if err != nil { 276 tail.Kill(err) 277 return 278 } 279 } 280 281 line, err := tail.readLine() 282 283 // Process `line` even if err is EOF. 284 if err == nil { 285 cooloff := !tail.sendLine(line) 286 if cooloff { 287 // Wait a second before seeking till the end of 288 // file when rate limit is reached. 289 msg := ("Too much log activity; waiting a second " + 290 "before resuming tailing") 291 tail.Lines <- &Line{msg, time.Now(), errors.New(msg),unknowOffset} 292 select { 293 case <-time.After(time.Second): 294 case <-tail.Dying(): 295 return 296 } 297 if err := tail.seekEnd(); err != nil { 298 tail.Kill(err) 299 return 300 } 301 } 302 } else if err == io.EOF { 303 if !tail.Follow { 304 if line != "" { 305 tail.sendLine(line) 306 } 307 return 308 } 309 310 if tail.Follow && line != "" { 311 // this has the potential to never return the last line if 312 // it's not followed by a newline; seems a fair trade here 313 err := tail.seekTo(SeekInfo{Offset: offset, Whence: 0}) 314 if err != nil { 315 tail.Kill(err) 316 return 317 } 318 } 319 320 // When EOF is reached, wait for more data to become 321 // available. Wait strategy is based on the `tail.watcher` 322 // implementation (inotify or polling). 323 err := tail.waitForChanges() 324 if err != nil { 325 if err != ErrStop { 326 tail.Kill(err) 327 } 328 return 329 } 330 } else { 331 // non-EOF error 332 tail.Killf("Error reading %s: %s", tail.Filename, err) 333 return 334 } 335 336 select { 337 case <-tail.Dying(): 338 if tail.Err() == errStopAtEOF { 339 continue 340 } 341 return 342 default: 343 } 344 } 345 } 346 347 // waitForChanges waits until the file has been appended, deleted, 348 // moved or truncated. When moved or deleted - the file will be 349 // reopened if ReOpen is true. Truncated files are always reopened. 350 func (tail *Tail) waitForChanges() error { 351 if tail.changes == nil { 352 pos, err := tail.file.Seek(0, os.SEEK_CUR) 353 if err != nil { 354 return err 355 } 356 tail.changes, err = tail.watcher.ChangeEvents(&tail.Tomb, pos) 357 if err != nil { 358 return err 359 } 360 } 361 362 select { 363 case <-tail.changes.Modified: 364 return nil 365 case <-tail.changes.Deleted: 366 tail.changes = nil 367 if tail.ReOpen { 368 // XXX: we must not log from a library. 369 tail.Logger.Printf("Re-opening moved/deleted file %s ...", tail.Filename) 370 if err := tail.reopen(); err != nil { 371 return err 372 } 373 tail.Logger.Printf("Successfully reopened %s", tail.Filename) 374 tail.openReader() 375 return nil 376 } else { 377 tail.Logger.Printf("Stopping tail as file no longer exists: %s", tail.Filename) 378 return ErrStop 379 } 380 case <-tail.changes.Truncated: 381 // Always reopen truncated files (Follow is true) 382 tail.Logger.Printf("Re-opening truncated file %s ...", tail.Filename) 383 if err := tail.reopen(); err != nil { 384 return err 385 } 386 tail.Logger.Printf("Successfully reopened truncated %s", tail.Filename) 387 tail.openReader() 388 return nil 389 case <-tail.Dying(): 390 return ErrStop 391 } 392 panic("unreachable") 393 } 394 395 func (tail *Tail) openReader() { 396 if tail.MaxLineSize > 0 { 397 // add 2 to account for newline characters 398 tail.reader = bufio.NewReaderSize(tail.file, tail.MaxLineSize+2) 399 } else { 400 tail.reader = bufio.NewReader(tail.file) 401 } 402 } 403 404 func (tail *Tail) seekEnd() error { 405 return tail.seekTo(SeekInfo{Offset: 0, Whence: os.SEEK_END}) 406 } 407 408 func (tail *Tail) seekTo(pos SeekInfo) error { 409 var err error 410 tail.offset, err = tail.file.Seek(pos.Offset, pos.Whence) 411 if err != nil { 412 return fmt.Errorf("Seek error on %s: %s", tail.Filename, err) 413 } 414 // Reset the read buffer whenever the file is re-seek'ed 415 tail.reader.Reset(tail.file) 416 return nil 417 } 418 419 // sendLine sends the line(s) to Lines channel, splitting longer lines 420 // if necessary. Return false if rate limit is reached. 421 func (tail *Tail) sendLine(line string) bool { 422 now := time.Now() 423 lines := []string{line} 424 425 // Split longer lines 426 if tail.MaxLineSize > 0 && len(line) > tail.MaxLineSize { 427 lines = util.PartitionString(line, tail.MaxLineSize) 428 } 429 430 for _, line := range lines { 431 tail.Lines <- &Line{line, now, nil,tail.offset} 432 tail.offset += int64(len(line)) 433 } 434 //for /n 435 tail.offset++ 436 437 if tail.Config.RateLimiter != nil { 438 ok := tail.Config.RateLimiter.Pour(uint16(len(lines))) 439 if !ok { 440 tail.Logger.Printf("Leaky bucket full (%v); entering 1s cooloff period.\n", 441 tail.Filename) 442 return false 443 } 444 } 445 446 return true 447 } 448 449 // Cleanup removes inotify watches added by the tail package. This function is 450 // meant to be invoked from a process's exit handler. Linux kernel may not 451 // automatically remove inotify watches after the process exits. 452 func (tail *Tail) Cleanup() { 453 watch.Cleanup(tail.Filename) 454 }