github.com/TheSilentForest/tail@v1.0.1-0.20190327153203-b5d4c85e68b5/tail.go (about) 1 // Copyright (c) 2015 HPE Software Inc. All rights reserved. 2 // Copyright (c) 2013 ActiveState Software Inc. All rights reserved. 3 4 package tail 5 6 import ( 7 "bufio" 8 "errors" 9 "fmt" 10 "io" 11 "io/ioutil" 12 "log" 13 "os" 14 "strings" 15 "sync" 16 "time" 17 18 "github.com/TheSilentForest/tail/ratelimiter" 19 "github.com/TheSilentForest/tail/util" 20 "github.com/TheSilentForest/tail/watch" 21 "gopkg.in/tomb.v1" 22 ) 23 24 var ( 25 ErrStop = errors.New("tail should now stop") 26 ) 27 28 type Line struct { 29 Text string 30 Time time.Time 31 Err error // Error from tail 32 Offset int64 33 } 34 35 // NewLine returns a Line with present time. 36 func NewLine(text string) *Line { 37 return &Line{Text: text, Time: time.Now()} 38 } 39 40 // SeekInfo represents arguments to `os.Seek` 41 type SeekInfo struct { 42 Offset int64 43 Whence int // os.SEEK_* 44 } 45 46 type logger interface { 47 Fatal(v ...interface{}) 48 Fatalf(format string, v ...interface{}) 49 Fatalln(v ...interface{}) 50 Panic(v ...interface{}) 51 Panicf(format string, v ...interface{}) 52 Panicln(v ...interface{}) 53 Print(v ...interface{}) 54 Printf(format string, v ...interface{}) 55 Println(v ...interface{}) 56 } 57 58 type DateParseFunc func(line string) (time.Time, error) 59 60 func DefaultDateParseFunc(line string) (time.Time, error) { 61 return time.Now(), nil 62 } 63 64 // Config is used to specify how a file must be tailed. 65 type Config struct { 66 // File-specifc 67 Location *SeekInfo // Seek to this location before tailing 68 ReOpen bool // Reopen recreated files (tail -F) 69 MustExist bool // Fail early if the file does not exist 70 Poll bool // Poll for file changes instead of using inotify 71 Pipe bool // Is a named pipe (mkfifo) 72 RateLimiter *ratelimiter.LeakyBucket 73 74 // Generic IO 75 Follow bool // Continue looking for new lines (tail -f) 76 MaxLineSize int // If non-zero, split longer lines into multiple lines 77 78 // Logger, when nil, is set to tail.DefaultLogger 79 // To disable logging: set field to tail.DiscardingLogger 80 Logger logger 81 DateParse DateParseFunc 82 } 83 84 type Tail struct { 85 Filename string 86 Lines chan *Line 87 Config 88 89 file *os.File 90 reader *bufio.Reader 91 92 watcher watch.FileWatcher 93 changes *watch.FileChanges 94 95 tomb.Tomb // provides: Done, Kill, Dying 96 97 lk sync.Mutex 98 } 99 100 var ( 101 // DefaultLogger is used when Config.Logger == nil 102 DefaultLogger = log.New(os.Stderr, "", log.LstdFlags) 103 // DiscardingLogger can be used to disable logging output 104 DiscardingLogger = log.New(ioutil.Discard, "", 0) 105 ) 106 107 // TailFile begins tailing the file. Output stream is made available 108 // via the `Tail.Lines` channel. To handle errors during tailing, 109 // invoke the `Wait` or `Err` method after finishing reading from the 110 // `Lines` channel. 111 func TailFile(filename string, config Config) (*Tail, error) { 112 if config.DateParse == nil { 113 config.DateParse = DefaultDateParseFunc 114 } 115 116 if config.ReOpen && !config.Follow { 117 util.Fatal("cannot set ReOpen without Follow.") 118 } 119 120 t := &Tail{ 121 Filename: filename, 122 Lines: make(chan *Line), 123 Config: config, 124 } 125 126 // when Logger was not specified in config, use default logger 127 if t.Logger == nil { 128 t.Logger = log.New(os.Stderr, "", log.LstdFlags) 129 } 130 131 if t.Poll { 132 t.watcher = watch.NewPollingFileWatcher(filename) 133 } else { 134 t.watcher = watch.NewInotifyFileWatcher(filename) 135 } 136 137 if t.MustExist { 138 var err error 139 t.file, err = OpenFile(t.Filename) 140 if err != nil { 141 return nil, err 142 } 143 } 144 145 go t.tailFileSync() 146 147 return t, nil 148 } 149 150 // Return the file's current position, like stdio's ftell(). 151 // But this value is not very accurate. 152 // it may readed one line in the chan(tail.Lines), 153 // so it may lost one line. 154 func (tail *Tail) Tell() (offset int64, err error) { 155 if tail.file == nil { 156 return 157 } 158 offset, err = tail.file.Seek(0, os.SEEK_CUR) 159 if err != nil { 160 return 161 } 162 163 tail.lk.Lock() 164 defer tail.lk.Unlock() 165 if tail.reader == nil { 166 return 167 } 168 169 offset -= int64(tail.reader.Buffered()) 170 return 171 } 172 173 // Stop stops the tailing activity. 174 func (tail *Tail) Stop() error { 175 tail.Kill(nil) 176 return tail.Wait() 177 } 178 179 // StopAtEOF stops tailing as soon as the end of the file is reached. 180 func (tail *Tail) StopAtEOF() error { 181 tail.Kill(errStopAtEOF) 182 return tail.Wait() 183 } 184 185 var errStopAtEOF = errors.New("tail: stop at eof") 186 187 func (tail *Tail) close() { 188 close(tail.Lines) 189 tail.closeFile() 190 } 191 192 func (tail *Tail) closeFile() { 193 if tail.file != nil { 194 tail.file.Close() 195 tail.file = nil 196 } 197 } 198 199 func (tail *Tail) reopen() error { 200 tail.closeFile() 201 for { 202 var err error 203 tail.file, err = OpenFile(tail.Filename) 204 if err != nil { 205 if os.IsNotExist(err) { 206 tail.Logger.Printf("Waiting for %s to appear...", tail.Filename) 207 if err := tail.watcher.BlockUntilExists(&tail.Tomb); err != nil { 208 if err == tomb.ErrDying { 209 return err 210 } 211 return fmt.Errorf("Failed to detect creation of %s: %s", tail.Filename, err) 212 } 213 continue 214 } 215 return fmt.Errorf("Unable to open file %s: %s", tail.Filename, err) 216 } 217 break 218 } 219 return nil 220 } 221 222 func (tail *Tail) readLine() (string, error) { 223 tail.lk.Lock() 224 line, err := tail.reader.ReadString('\n') 225 tail.lk.Unlock() 226 if err != nil { 227 // Note ReadString "returns the data read before the error" in 228 // case of an error, including EOF, so we return it as is. The 229 // caller is expected to process it if err is EOF. 230 return line, err 231 } 232 233 line = strings.TrimRight(line, "\n") 234 235 return line, err 236 } 237 238 func (tail *Tail) tailFileSync() { 239 defer tail.Done() 240 defer tail.close() 241 242 if tail.DateParse == nil { 243 tail.DateParse = DefaultDateParseFunc 244 } 245 246 if !tail.MustExist { 247 // deferred first open. 248 err := tail.reopen() 249 if err != nil { 250 if err != tomb.ErrDying { 251 tail.Kill(err) 252 } 253 return 254 } 255 } 256 257 // Seek to requested location on first open of the file. 258 if tail.Location != nil { 259 _, err := tail.file.Seek(tail.Location.Offset, tail.Location.Whence) 260 tail.Logger.Printf("Seeked %s - %+v\n", tail.Filename, tail.Location) 261 if err != nil { 262 tail.Killf("Seek error on %s: %s", tail.Filename, err) 263 return 264 } 265 } 266 267 tail.openReader() 268 269 var offset int64 270 var err error 271 272 // Read line by line. 273 for { 274 // do not seek in named pipes 275 if !tail.Pipe { 276 // grab the position in case we need to back up in the event of a half-line 277 offset, err = tail.Tell() 278 if err != nil { 279 tail.Kill(err) 280 return 281 } 282 } 283 284 line, err := tail.readLine() 285 286 // Process `line` even if err is EOF. 287 if err == nil { 288 cooloff := !tail.sendLine(line, offset) 289 if cooloff { 290 // Wait a second before seeking till the end of 291 // file when rate limit is reached. 292 msg := ("Too much log activity; waiting a second " + 293 "before resuming tailing") 294 t, err := tail.DateParse(msg) 295 if err != nil { 296 tail.Logger.Printf("Error parsing date %s ...", err) 297 t = time.Now() 298 } 299 tail.Lines <- &Line{Text: msg, Time: t, Err: errors.New(msg), Offset: offset} 300 select { 301 case <-time.After(time.Second): 302 case <-tail.Dying(): 303 return 304 } 305 if err := tail.seekEnd(); err != nil { 306 tail.Kill(err) 307 return 308 } 309 } 310 } else if err == io.EOF { 311 if !tail.Follow { 312 if line != "" { 313 tail.sendLine(line, offset) 314 } 315 return 316 } 317 318 if tail.Follow && line != "" { 319 // this has the potential to never return the last line if 320 // it's not followed by a newline; seems a fair trade here 321 err := tail.seekTo(SeekInfo{Offset: offset, Whence: 0}) 322 if err != nil { 323 tail.Kill(err) 324 return 325 } 326 } 327 328 // When EOF is reached, wait for more data to become 329 // available. Wait strategy is based on the `tail.watcher` 330 // implementation (inotify or polling). 331 err := tail.waitForChanges() 332 if err != nil { 333 if err != ErrStop { 334 tail.Kill(err) 335 } 336 return 337 } 338 } else { 339 // non-EOF error 340 tail.Killf("Error reading %s: %s", tail.Filename, err) 341 return 342 } 343 344 select { 345 case <-tail.Dying(): 346 if tail.Err() == errStopAtEOF { 347 continue 348 } 349 return 350 default: 351 } 352 } 353 } 354 355 // waitForChanges waits until the file has been appended, deleted, 356 // moved or truncated. When moved or deleted - the file will be 357 // reopened if ReOpen is true. Truncated files are always reopened. 358 func (tail *Tail) waitForChanges() error { 359 if tail.changes == nil { 360 pos, err := tail.file.Seek(0, os.SEEK_CUR) 361 if err != nil { 362 return err 363 } 364 tail.changes, err = tail.watcher.ChangeEvents(&tail.Tomb, pos) 365 if err != nil { 366 return err 367 } 368 } 369 370 select { 371 case <-tail.changes.Modified: 372 return nil 373 case <-tail.changes.Deleted: 374 tail.changes = nil 375 if tail.ReOpen { 376 // XXX: we must not log from a library. 377 tail.Logger.Printf("Re-opening moved/deleted file %s ...", tail.Filename) 378 if err := tail.reopen(); err != nil { 379 return err 380 } 381 tail.Logger.Printf("Successfully reopened %s", tail.Filename) 382 tail.openReader() 383 return nil 384 } else { 385 tail.Logger.Printf("Stopping tail as file no longer exists: %s", tail.Filename) 386 return ErrStop 387 } 388 case <-tail.changes.Truncated: 389 // Always reopen truncated files (Follow is true) 390 tail.Logger.Printf("Re-opening truncated file %s ...", tail.Filename) 391 if err := tail.reopen(); err != nil { 392 return err 393 } 394 tail.Logger.Printf("Successfully reopened truncated %s", tail.Filename) 395 tail.openReader() 396 return nil 397 case <-tail.Dying(): 398 return ErrStop 399 } 400 panic("unreachable") 401 } 402 403 func (tail *Tail) openReader() { 404 if tail.MaxLineSize > 0 { 405 // add 2 to account for newline characters 406 tail.reader = bufio.NewReaderSize(tail.file, tail.MaxLineSize+2) 407 } else { 408 tail.reader = bufio.NewReader(tail.file) 409 } 410 } 411 412 func (tail *Tail) seekEnd() error { 413 return tail.seekTo(SeekInfo{Offset: 0, Whence: os.SEEK_END}) 414 } 415 416 func (tail *Tail) seekTo(pos SeekInfo) error { 417 _, err := tail.file.Seek(pos.Offset, pos.Whence) 418 if err != nil { 419 return fmt.Errorf("Seek error on %s: %s", tail.Filename, err) 420 } 421 // Reset the read buffer whenever the file is re-seek'ed 422 tail.reader.Reset(tail.file) 423 return nil 424 } 425 426 // sendLine sends the line(s) to Lines channel, splitting longer lines 427 // if necessary. Return false if rate limit is reached. 428 func (tail *Tail) sendLine(line string, offset int64) bool { 429 lines := []string{line} 430 431 // Split longer lines 432 if tail.MaxLineSize > 0 && len(line) > tail.MaxLineSize { 433 lines = util.PartitionString(line, tail.MaxLineSize) 434 } 435 436 for _, line := range lines { 437 t, err := tail.DateParse(line) 438 if err != nil { 439 tail.Logger.Printf("Error parsing date %s ...", err) 440 t = time.Now() 441 } 442 tail.Lines <- &Line{Text: line, Time: t, Offset: offset} 443 } 444 445 if tail.Config.RateLimiter != nil { 446 ok := tail.Config.RateLimiter.Pour(uint16(len(lines))) 447 if !ok { 448 tail.Logger.Printf("Leaky bucket full (%v); entering 1s cooloff period.\n", 449 tail.Filename) 450 return false 451 } 452 } 453 454 return true 455 } 456 457 // Cleanup removes inotify watches added by the tail package. This function is 458 // meant to be invoked from a process's exit handler. Linux kernel may not 459 // automatically remove inotify watches after the process exits. 460 func (tail *Tail) Cleanup() { 461 watch.Cleanup(tail.Filename) 462 }