github.com/kosmosec/tail@v1.1.0/tail.go (about) 1 // Copyright (c) 2015 HPE Software Inc. All rights reserved. 2 // Copyright (c) 2013 ActiveState Software Inc. All rights reserved. 3 4 package tail 5 6 import ( 7 "bufio" 8 "bytes" 9 "errors" 10 "fmt" 11 "io" 12 "io/ioutil" 13 "log" 14 "os" 15 "sync" 16 "time" 17 18 "github.com/hpcloud/tail/ratelimiter" 19 "github.com/hpcloud/tail/util" 20 "github.com/hpcloud/tail/watch" 21 "gopkg.in/tomb.v1" 22 ) 23 24 var ( 25 ErrStop = errors.New("tail should now stop") 26 ) 27 28 type Line struct { 29 Text []byte 30 Time time.Time 31 Err error // Error from tail 32 } 33 34 // NewLine returns a Line with present time. 35 func NewLine(text []byte) *Line { 36 return &Line{text, time.Now(), nil} 37 } 38 39 // SeekInfo represents arguments to `os.Seek` 40 type SeekInfo struct { 41 Offset int64 42 Whence int // os.SEEK_* 43 } 44 45 type logger interface { 46 Fatal(v ...interface{}) 47 Fatalf(format string, v ...interface{}) 48 Fatalln(v ...interface{}) 49 Panic(v ...interface{}) 50 Panicf(format string, v ...interface{}) 51 Panicln(v ...interface{}) 52 Print(v ...interface{}) 53 Printf(format string, v ...interface{}) 54 Println(v ...interface{}) 55 } 56 57 // Config is used to specify how a file must be tailed. 58 type Config struct { 59 // File-specifc 60 Location *SeekInfo // Seek to this location before tailing 61 ReOpen bool // Reopen recreated files (tail -F) 62 MustExist bool // Fail early if the file does not exist 63 Poll bool // Poll for file changes instead of using inotify 64 Pipe bool // Is a named pipe (mkfifo) 65 RateLimiter *ratelimiter.LeakyBucket 66 67 // Generic IO 68 Follow bool // Continue looking for new lines (tail -f) 69 MaxLineSize int // If non-zero, split longer lines into multiple lines 70 71 // Logger, when nil, is set to tail.DefaultLogger 72 // To disable logging: set field to tail.DiscardingLogger 73 Logger logger 74 } 75 76 type Tail struct { 77 Filename string 78 Lines chan *Line 79 Config 80 81 file *os.File 82 reader *bufio.Reader 83 84 watcher watch.FileWatcher 85 changes *watch.FileChanges 86 87 tomb.Tomb // provides: Done, Kill, Dying 88 89 lk sync.Mutex 90 } 91 92 var ( 93 // DefaultLogger is used when Config.Logger == nil 94 DefaultLogger = log.New(os.Stderr, "", log.LstdFlags) 95 // DiscardingLogger can be used to disable logging output 96 DiscardingLogger = log.New(ioutil.Discard, "", 0) 97 ) 98 99 // TailFile begins tailing the file. Output stream is made available 100 // via the `Tail.Lines` channel. To handle errors during tailing, 101 // invoke the `Wait` or `Err` method after finishing reading from the 102 // `Lines` channel. 103 func TailFile(filename string, config Config) (*Tail, error) { 104 if config.ReOpen && !config.Follow { 105 util.Fatal("cannot set ReOpen without Follow.") 106 } 107 108 t := &Tail{ 109 Filename: filename, 110 Lines: make(chan *Line), 111 Config: config, 112 } 113 114 // when Logger was not specified in config, use default logger 115 if t.Logger == nil { 116 t.Logger = log.New(os.Stderr, "", log.LstdFlags) 117 } 118 119 if t.Poll { 120 t.watcher = watch.NewPollingFileWatcher(filename) 121 } else { 122 t.watcher = watch.NewInotifyFileWatcher(filename) 123 } 124 125 if t.MustExist { 126 var err error 127 t.file, err = OpenFile(t.Filename) 128 if err != nil { 129 return nil, err 130 } 131 } 132 133 go t.tailFileSync() 134 135 return t, nil 136 } 137 138 // Return the file's current position, like stdio's ftell(). 139 // But this value is not very accurate. 140 // it may readed one line in the chan(tail.Lines), 141 // so it may lost one line. 142 func (tail *Tail) Tell() (offset int64, err error) { 143 if tail.file == nil { 144 return 145 } 146 offset, err = tail.file.Seek(0, os.SEEK_CUR) 147 if err != nil { 148 return 149 } 150 151 tail.lk.Lock() 152 defer tail.lk.Unlock() 153 if tail.reader == nil { 154 return 155 } 156 157 offset -= int64(tail.reader.Buffered()) 158 return 159 } 160 161 // Stop stops the tailing activity. 162 func (tail *Tail) Stop() error { 163 tail.Kill(nil) 164 return tail.Wait() 165 } 166 167 // StopAtEOF stops tailing as soon as the end of the file is reached. 168 func (tail *Tail) StopAtEOF() error { 169 tail.Kill(errStopAtEOF) 170 return tail.Wait() 171 } 172 173 var errStopAtEOF = errors.New("tail: stop at eof") 174 175 func (tail *Tail) close() { 176 close(tail.Lines) 177 tail.closeFile() 178 } 179 180 func (tail *Tail) closeFile() { 181 if tail.file != nil { 182 tail.file.Close() 183 tail.file = nil 184 } 185 } 186 187 func (tail *Tail) reopen() error { 188 tail.closeFile() 189 for { 190 var err error 191 tail.file, err = OpenFile(tail.Filename) 192 if err != nil { 193 if os.IsNotExist(err) { 194 tail.Logger.Printf("Waiting for %s to appear...", tail.Filename) 195 if err := tail.watcher.BlockUntilExists(&tail.Tomb); err != nil { 196 if err == tomb.ErrDying { 197 return err 198 } 199 return fmt.Errorf("Failed to detect creation of %s: %s", tail.Filename, err) 200 } 201 continue 202 } 203 return fmt.Errorf("Unable to open file %s: %s", tail.Filename, err) 204 } 205 break 206 } 207 return nil 208 } 209 210 func (tail *Tail) readLine() ([]byte, error) { 211 tail.lk.Lock() 212 buf := make([]byte, 256) 213 _, err := tail.reader.Read(buf) 214 215 //line, err := tail.reader.ReadBytes(0x0A) 216 tail.lk.Unlock() 217 if err != nil { 218 // Note ReadString "returns the data read before the error" in 219 // case of an error, including EOF, so we return it as is. The 220 // caller is expected to process it if err is EOF. 221 return buf, err 222 } 223 224 //line = strings.TrimRight(line, "\n") 225 //line = bytes.TrimRight(line, 0x0A) 226 buf = bytes.Trim(buf, "\x00") 227 228 return buf, err 229 } 230 231 func (tail *Tail) tailFileSync() { 232 defer tail.Done() 233 defer tail.close() 234 235 if !tail.MustExist { 236 // deferred first open. 237 err := tail.reopen() 238 if err != nil { 239 if err != tomb.ErrDying { 240 tail.Kill(err) 241 } 242 return 243 } 244 } 245 246 // Seek to requested location on first open of the file. 247 if tail.Location != nil { 248 _, err := tail.file.Seek(tail.Location.Offset, tail.Location.Whence) 249 tail.Logger.Printf("Seeked %s - %+v\n", tail.Filename, tail.Location) 250 if err != nil { 251 tail.Killf("Seek error on %s: %s", tail.Filename, err) 252 return 253 } 254 } 255 256 tail.openReader() 257 258 var offset int64 259 var err error 260 261 // Read line by line. 262 for { 263 // do not seek in named pipes 264 if !tail.Pipe { 265 // grab the position in case we need to back up in the event of a half-line 266 offset, err = tail.Tell() 267 if err != nil { 268 tail.Kill(err) 269 return 270 } 271 } 272 273 line, err := tail.readLine() 274 275 // Process `line` even if err is EOF. 276 if err == nil { 277 cooloff := !tail.sendLine(line) 278 if cooloff { 279 // Wait a second before seeking till the end of 280 // file when rate limit is reached. 281 msg := ("Too much log activity; waiting a second " + 282 "before resuming tailing") 283 tail.Lines <- &Line{[]byte(msg), time.Now(), errors.New(msg)} 284 select { 285 case <-time.After(time.Second): 286 case <-tail.Dying(): 287 return 288 } 289 if err := tail.seekEnd(); err != nil { 290 tail.Kill(err) 291 return 292 } 293 } 294 } else if err == io.EOF { 295 if !tail.Follow { 296 if len(line) != 0 { 297 tail.sendLine(line) 298 } 299 return 300 } 301 302 if tail.Follow && len(line) != 0 { 303 // this has the potential to never return the last line if 304 // it's not followed by a newline; seems a fair trade here 305 err := tail.seekTo(SeekInfo{Offset: offset, Whence: 0}) 306 if err != nil { 307 tail.Kill(err) 308 return 309 } 310 } 311 312 // When EOF is reached, wait for more data to become 313 // available. Wait strategy is based on the `tail.watcher` 314 // implementation (inotify or polling). 315 err := tail.waitForChanges() 316 if err != nil { 317 if err != ErrStop { 318 tail.Kill(err) 319 } 320 return 321 } 322 } else { 323 // non-EOF error 324 tail.Killf("Error reading %s: %s", tail.Filename, err) 325 return 326 } 327 328 select { 329 case <-tail.Dying(): 330 if tail.Err() == errStopAtEOF { 331 continue 332 } 333 return 334 default: 335 } 336 } 337 } 338 339 // waitForChanges waits until the file has been appended, deleted, 340 // moved or truncated. When moved or deleted - the file will be 341 // reopened if ReOpen is true. Truncated files are always reopened. 342 func (tail *Tail) waitForChanges() error { 343 if tail.changes == nil { 344 pos, err := tail.file.Seek(0, os.SEEK_CUR) 345 if err != nil { 346 return err 347 } 348 tail.changes, err = tail.watcher.ChangeEvents(&tail.Tomb, pos) 349 if err != nil { 350 return err 351 } 352 } 353 354 select { 355 case <-tail.changes.Modified: 356 return nil 357 case <-tail.changes.Deleted: 358 tail.changes = nil 359 if tail.ReOpen { 360 // XXX: we must not log from a library. 361 tail.Logger.Printf("Re-opening moved/deleted file %s ...", tail.Filename) 362 if err := tail.reopen(); err != nil { 363 return err 364 } 365 tail.Logger.Printf("Successfully reopened %s", tail.Filename) 366 tail.openReader() 367 return nil 368 } else { 369 tail.Logger.Printf("Stopping tail as file no longer exists: %s", tail.Filename) 370 return ErrStop 371 } 372 case <-tail.changes.Truncated: 373 // Always reopen truncated files (Follow is true) 374 tail.Logger.Printf("Re-opening truncated file %s ...", tail.Filename) 375 if err := tail.reopen(); err != nil { 376 return err 377 } 378 tail.Logger.Printf("Successfully reopened truncated %s", tail.Filename) 379 tail.openReader() 380 return nil 381 case <-tail.Dying(): 382 return ErrStop 383 } 384 panic("unreachable") 385 } 386 387 func (tail *Tail) openReader() { 388 if tail.MaxLineSize > 0 { 389 // add 2 to account for newline characters 390 tail.reader = bufio.NewReaderSize(tail.file, tail.MaxLineSize+2) 391 } else { 392 tail.reader = bufio.NewReader(tail.file) 393 } 394 } 395 396 func (tail *Tail) seekEnd() error { 397 return tail.seekTo(SeekInfo{Offset: 0, Whence: os.SEEK_END}) 398 } 399 400 func (tail *Tail) seekTo(pos SeekInfo) error { 401 _, err := tail.file.Seek(pos.Offset, pos.Whence) 402 if err != nil { 403 return fmt.Errorf("Seek error on %s: %s", tail.Filename, err) 404 } 405 // Reset the read buffer whenever the file is re-seek'ed 406 tail.reader.Reset(tail.file) 407 return nil 408 } 409 410 // sendLine sends the line(s) to Lines channel, splitting longer lines 411 // if necessary. Return false if rate limit is reached. 412 func (tail *Tail) sendLine(line []byte) bool { 413 now := time.Now() 414 lines := line 415 416 // Split longer lines 417 // if tail.MaxLineSize > 0 && len(line) > tail.MaxLineSize { 418 // lines = util.PartitionString(line, tail.MaxLineSize) 419 // } 420 421 //for _, line := range lines { 422 tail.Lines <- &Line{line, now, nil} 423 //} 424 425 if tail.Config.RateLimiter != nil { 426 ok := tail.Config.RateLimiter.Pour(uint16(len(lines))) 427 if !ok { 428 tail.Logger.Printf("Leaky bucket full (%v); entering 1s cooloff period.\n", 429 tail.Filename) 430 return false 431 } 432 } 433 434 return true 435 } 436 437 // Cleanup removes inotify watches added by the tail package. This function is 438 // meant to be invoked from a process's exit handler. Linux kernel may not 439 // automatically remove inotify watches after the process exits. 440 func (tail *Tail) Cleanup() { 441 watch.Cleanup(tail.Filename) 442 }