github.com/hooklift/nomad@v0.5.7-0.20170407200202-db11e7dd7b55/command/agent/fs_endpoint.go (about) 1 package agent 2 3 //go:generate codecgen -o fs_endpoint.generated.go fs_endpoint.go 4 5 import ( 6 "bytes" 7 "fmt" 8 "io" 9 "math" 10 "net/http" 11 "os" 12 "path/filepath" 13 "sort" 14 "strconv" 15 "strings" 16 "sync" 17 "syscall" 18 "time" 19 20 "gopkg.in/tomb.v1" 21 22 "github.com/docker/docker/pkg/ioutils" 23 "github.com/hashicorp/nomad/client/allocdir" 24 "github.com/hpcloud/tail/watch" 25 "github.com/ugorji/go/codec" 26 ) 27 28 var ( 29 allocIDNotPresentErr = fmt.Errorf("must provide a valid alloc id") 30 fileNameNotPresentErr = fmt.Errorf("must provide a file name") 31 taskNotPresentErr = fmt.Errorf("must provide task name") 32 logTypeNotPresentErr = fmt.Errorf("must provide log type (stdout/stderr)") 33 clientNotRunning = fmt.Errorf("node is not running a Nomad Client") 34 invalidOrigin = fmt.Errorf("origin must be start or end") 35 ) 36 37 const ( 38 // streamFrameSize is the maximum number of bytes to send in a single frame 39 streamFrameSize = 64 * 1024 40 41 // streamHeartbeatRate is the rate at which a heartbeat will occur to detect 42 // a closed connection without sending any additional data 43 streamHeartbeatRate = 1 * time.Second 44 45 // streamBatchWindow is the window in which file content is batched before 46 // being flushed if the frame size has not been hit. 47 streamBatchWindow = 200 * time.Millisecond 48 49 // nextLogCheckRate is the rate at which we check for a log entry greater 50 // than what we are watching for. This is to handle the case in which logs 51 // rotate faster than we can detect and we have to rely on a normal 52 // directory listing. 53 nextLogCheckRate = 100 * time.Millisecond 54 55 // deleteEvent and truncateEvent are the file events that can be sent in a 56 // StreamFrame 57 deleteEvent = "file deleted" 58 truncateEvent = "file truncated" 59 60 // OriginStart and OriginEnd are the available parameters for the origin 61 // argument when streaming a file. They respectively offset from the start 62 // and end of a file. 63 OriginStart = "start" 64 OriginEnd = "end" 65 ) 66 67 func (s *HTTPServer) FsRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) { 68 if s.agent.client == nil { 69 return nil, clientNotRunning 70 } 71 72 path := strings.TrimPrefix(req.URL.Path, "/v1/client/fs/") 73 switch { 74 case strings.HasPrefix(path, "ls/"): 75 return s.DirectoryListRequest(resp, req) 76 case strings.HasPrefix(path, "stat/"): 77 return s.FileStatRequest(resp, req) 78 case strings.HasPrefix(path, "readat/"): 79 return s.FileReadAtRequest(resp, req) 80 case strings.HasPrefix(path, "cat/"): 81 return s.FileCatRequest(resp, req) 82 case strings.HasPrefix(path, "stream/"): 83 return s.Stream(resp, req) 84 case strings.HasPrefix(path, "logs/"): 85 return s.Logs(resp, req) 86 default: 87 return nil, CodedError(404, ErrInvalidMethod) 88 } 89 } 90 91 func (s *HTTPServer) DirectoryListRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) { 92 var allocID, path string 93 94 if allocID = strings.TrimPrefix(req.URL.Path, "/v1/client/fs/ls/"); allocID == "" { 95 return nil, allocIDNotPresentErr 96 } 97 if path = req.URL.Query().Get("path"); path == "" { 98 path = "/" 99 } 100 fs, err := s.agent.client.GetAllocFS(allocID) 101 if err != nil { 102 return nil, err 103 } 104 return fs.List(path) 105 } 106 107 func (s *HTTPServer) FileStatRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) { 108 var allocID, path string 109 if allocID = strings.TrimPrefix(req.URL.Path, "/v1/client/fs/stat/"); allocID == "" { 110 return nil, allocIDNotPresentErr 111 } 112 if path = req.URL.Query().Get("path"); path == "" { 113 return nil, fileNameNotPresentErr 114 } 115 fs, err := s.agent.client.GetAllocFS(allocID) 116 if err != nil { 117 return nil, err 118 } 119 return fs.Stat(path) 120 } 121 122 func (s *HTTPServer) FileReadAtRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) { 123 var allocID, path string 124 var offset, limit int64 125 var err error 126 127 q := req.URL.Query() 128 129 if allocID = strings.TrimPrefix(req.URL.Path, "/v1/client/fs/readat/"); allocID == "" { 130 return nil, allocIDNotPresentErr 131 } 132 if path = q.Get("path"); path == "" { 133 return nil, fileNameNotPresentErr 134 } 135 136 if offset, err = strconv.ParseInt(q.Get("offset"), 10, 64); err != nil { 137 return nil, fmt.Errorf("error parsing offset: %v", err) 138 } 139 140 // Parse the limit 141 if limitStr := q.Get("limit"); limitStr != "" { 142 if limit, err = strconv.ParseInt(limitStr, 10, 64); err != nil { 143 return nil, fmt.Errorf("error parsing limit: %v", err) 144 } 145 } 146 147 fs, err := s.agent.client.GetAllocFS(allocID) 148 if err != nil { 149 return nil, err 150 } 151 152 rc, err := fs.ReadAt(path, offset) 153 if limit > 0 { 154 rc = &ReadCloserWrapper{ 155 Reader: io.LimitReader(rc, limit), 156 Closer: rc, 157 } 158 } 159 160 if err != nil { 161 return nil, err 162 } 163 164 io.Copy(resp, rc) 165 return nil, rc.Close() 166 } 167 168 // ReadCloserWrapper wraps a LimitReader so that a file is closed once it has been 169 // read 170 type ReadCloserWrapper struct { 171 io.Reader 172 io.Closer 173 } 174 175 func (s *HTTPServer) FileCatRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) { 176 var allocID, path string 177 var err error 178 179 q := req.URL.Query() 180 181 if allocID = strings.TrimPrefix(req.URL.Path, "/v1/client/fs/cat/"); allocID == "" { 182 return nil, allocIDNotPresentErr 183 } 184 if path = q.Get("path"); path == "" { 185 return nil, fileNameNotPresentErr 186 } 187 fs, err := s.agent.client.GetAllocFS(allocID) 188 if err != nil { 189 return nil, err 190 } 191 192 fileInfo, err := fs.Stat(path) 193 if err != nil { 194 return nil, err 195 } 196 if fileInfo.IsDir { 197 return nil, fmt.Errorf("file %q is a directory", path) 198 } 199 200 r, err := fs.ReadAt(path, int64(0)) 201 if err != nil { 202 return nil, err 203 } 204 io.Copy(resp, r) 205 return nil, r.Close() 206 } 207 208 var ( 209 // HeartbeatStreamFrame is the StreamFrame to send as a heartbeat, avoiding 210 // creating many instances of the empty StreamFrame 211 HeartbeatStreamFrame = &StreamFrame{} 212 ) 213 214 // StreamFrame is used to frame data of a file when streaming 215 type StreamFrame struct { 216 // Offset is the offset the data was read from 217 Offset int64 `json:",omitempty"` 218 219 // Data is the read data 220 Data []byte `json:",omitempty"` 221 222 // File is the file that the data was read from 223 File string `json:",omitempty"` 224 225 // FileEvent is the last file event that occurred that could cause the 226 // streams position to change or end 227 FileEvent string `json:",omitempty"` 228 } 229 230 // IsHeartbeat returns if the frame is a heartbeat frame 231 func (s *StreamFrame) IsHeartbeat() bool { 232 return s.Offset == 0 && len(s.Data) == 0 && s.File == "" && s.FileEvent == "" 233 } 234 235 func (s *StreamFrame) Clear() { 236 s.Offset = 0 237 s.Data = nil 238 s.File = "" 239 s.FileEvent = "" 240 } 241 242 func (s *StreamFrame) IsCleared() bool { 243 if s.Offset != 0 { 244 return false 245 } else if s.Data != nil { 246 return false 247 } else if s.File != "" { 248 return false 249 } else if s.FileEvent != "" { 250 return false 251 } else { 252 return true 253 } 254 } 255 256 // StreamFramer is used to buffer and send frames as well as heartbeat. 257 type StreamFramer struct { 258 // plainTxt determines whether we frame or just send plain text data. 259 plainTxt bool 260 261 out io.WriteCloser 262 enc *codec.Encoder 263 encLock sync.Mutex 264 265 frameSize int 266 267 heartbeat *time.Ticker 268 flusher *time.Ticker 269 270 shutdownCh chan struct{} 271 exitCh chan struct{} 272 273 // The mutex protects everything below 274 l sync.Mutex 275 276 // The current working frame 277 f StreamFrame 278 data *bytes.Buffer 279 280 // Captures whether the framer is running and any error that occurred to 281 // cause it to stop. 282 running bool 283 Err error 284 } 285 286 // NewStreamFramer creates a new stream framer that will output StreamFrames to 287 // the passed output. If plainTxt is set we do not frame and just batch plain 288 // text data. 289 func NewStreamFramer(out io.WriteCloser, plainTxt bool, 290 heartbeatRate, batchWindow time.Duration, frameSize int) *StreamFramer { 291 292 // Create a JSON encoder 293 enc := codec.NewEncoder(out, jsonHandle) 294 295 // Create the heartbeat and flush ticker 296 heartbeat := time.NewTicker(heartbeatRate) 297 flusher := time.NewTicker(batchWindow) 298 299 return &StreamFramer{ 300 plainTxt: plainTxt, 301 out: out, 302 enc: enc, 303 frameSize: frameSize, 304 heartbeat: heartbeat, 305 flusher: flusher, 306 data: bytes.NewBuffer(make([]byte, 0, 2*frameSize)), 307 shutdownCh: make(chan struct{}), 308 exitCh: make(chan struct{}), 309 } 310 } 311 312 // Destroy is used to cleanup the StreamFramer and flush any pending frames 313 func (s *StreamFramer) Destroy() { 314 s.l.Lock() 315 close(s.shutdownCh) 316 s.heartbeat.Stop() 317 s.flusher.Stop() 318 running := s.running 319 s.l.Unlock() 320 321 // Ensure things were flushed 322 if running { 323 <-s.exitCh 324 } 325 s.out.Close() 326 } 327 328 // Run starts a long lived goroutine that handles sending data as well as 329 // heartbeating 330 func (s *StreamFramer) Run() { 331 s.l.Lock() 332 defer s.l.Unlock() 333 if s.running { 334 return 335 } 336 337 s.running = true 338 go s.run() 339 } 340 341 // ExitCh returns a channel that will be closed when the run loop terminates. 342 func (s *StreamFramer) ExitCh() <-chan struct{} { 343 return s.exitCh 344 } 345 346 // run is the internal run method. It exits if Destroy is called or an error 347 // occurs, in which case the exit channel is closed. 348 func (s *StreamFramer) run() { 349 var err error 350 defer func() { 351 close(s.exitCh) 352 s.l.Lock() 353 s.running = false 354 s.Err = err 355 s.l.Unlock() 356 }() 357 358 OUTER: 359 for { 360 select { 361 case <-s.shutdownCh: 362 break OUTER 363 case <-s.flusher.C: 364 // Skip if there is nothing to flush 365 s.l.Lock() 366 if s.f.IsCleared() { 367 s.l.Unlock() 368 continue 369 } 370 371 // Read the data for the frame, and send it 372 s.f.Data = s.readData() 373 err = s.send(&s.f) 374 s.f.Clear() 375 s.l.Unlock() 376 if err != nil { 377 return 378 } 379 case <-s.heartbeat.C: 380 // Send a heartbeat frame 381 if err = s.send(HeartbeatStreamFrame); err != nil { 382 return 383 } 384 } 385 } 386 387 s.l.Lock() 388 if !s.f.IsCleared() { 389 s.f.Data = s.readData() 390 err = s.send(&s.f) 391 s.f.Clear() 392 } 393 s.l.Unlock() 394 } 395 396 // send takes a StreamFrame, encodes and sends it 397 func (s *StreamFramer) send(f *StreamFrame) error { 398 s.encLock.Lock() 399 defer s.encLock.Unlock() 400 if s.plainTxt { 401 _, err := io.Copy(s.out, bytes.NewReader(f.Data)) 402 return err 403 } 404 return s.enc.Encode(f) 405 } 406 407 // readData is a helper which reads the buffered data returning up to the frame 408 // size of data. Must be called with the lock held. The returned value is 409 // invalid on the next read or write into the StreamFramer buffer 410 func (s *StreamFramer) readData() []byte { 411 // Compute the amount to read from the buffer 412 size := s.data.Len() 413 if size > s.frameSize { 414 size = s.frameSize 415 } 416 if size == 0 { 417 return nil 418 } 419 d := s.data.Next(size) 420 return d 421 } 422 423 // Send creates and sends a StreamFrame based on the passed parameters. An error 424 // is returned if the run routine hasn't run or encountered an error. Send is 425 // asyncronous and does not block for the data to be transferred. 426 func (s *StreamFramer) Send(file, fileEvent string, data []byte, offset int64) error { 427 s.l.Lock() 428 defer s.l.Unlock() 429 430 // If we are not running, return the error that caused us to not run or 431 // indicated that it was never started. 432 if !s.running { 433 if s.Err != nil { 434 return s.Err 435 } 436 437 return fmt.Errorf("StreamFramer not running") 438 } 439 440 // Check if not mergeable 441 if !s.f.IsCleared() && (s.f.File != file || s.f.FileEvent != fileEvent) { 442 // Flush the old frame 443 s.f.Data = s.readData() 444 select { 445 case <-s.exitCh: 446 return nil 447 default: 448 } 449 err := s.send(&s.f) 450 s.f.Clear() 451 if err != nil { 452 return err 453 } 454 } 455 456 // Store the new data as the current frame. 457 if s.f.IsCleared() { 458 s.f.Offset = offset 459 s.f.File = file 460 s.f.FileEvent = fileEvent 461 } 462 463 // Write the data to the buffer 464 s.data.Write(data) 465 466 // Handle the delete case in which there is no data 467 force := false 468 if s.data.Len() == 0 && s.f.FileEvent != "" { 469 force = true 470 } 471 472 // Flush till we are under the max frame size 473 for s.data.Len() >= s.frameSize || force { 474 // Clear 475 if force { 476 force = false 477 } 478 479 // Create a new frame to send it 480 s.f.Data = s.readData() 481 select { 482 case <-s.exitCh: 483 return nil 484 default: 485 } 486 487 if err := s.send(&s.f); err != nil { 488 return err 489 } 490 491 // Update the offset 492 s.f.Offset += int64(len(s.f.Data)) 493 } 494 495 if s.data.Len() == 0 { 496 s.f.Clear() 497 } 498 499 return nil 500 } 501 502 // Stream streams the content of a file blocking on EOF. 503 // The parameters are: 504 // * path: path to file to stream. 505 // * offset: The offset to start streaming data at, defaults to zero. 506 // * origin: Either "start" or "end" and defines from where the offset is 507 // applied. Defaults to "start". 508 func (s *HTTPServer) Stream(resp http.ResponseWriter, req *http.Request) (interface{}, error) { 509 var allocID, path string 510 var err error 511 512 q := req.URL.Query() 513 514 if allocID = strings.TrimPrefix(req.URL.Path, "/v1/client/fs/stream/"); allocID == "" { 515 return nil, allocIDNotPresentErr 516 } 517 518 if path = q.Get("path"); path == "" { 519 return nil, fileNameNotPresentErr 520 } 521 522 var offset int64 523 offsetString := q.Get("offset") 524 if offsetString != "" { 525 var err error 526 if offset, err = strconv.ParseInt(offsetString, 10, 64); err != nil { 527 return nil, fmt.Errorf("error parsing offset: %v", err) 528 } 529 } 530 531 origin := q.Get("origin") 532 switch origin { 533 case "start", "end": 534 case "": 535 origin = "start" 536 default: 537 return nil, invalidOrigin 538 } 539 540 fs, err := s.agent.client.GetAllocFS(allocID) 541 if err != nil { 542 return nil, err 543 } 544 545 fileInfo, err := fs.Stat(path) 546 if err != nil { 547 return nil, err 548 } 549 if fileInfo.IsDir { 550 return nil, fmt.Errorf("file %q is a directory", path) 551 } 552 553 // If offsetting from the end subtract from the size 554 if origin == "end" { 555 offset = fileInfo.Size - offset 556 557 } 558 559 // Create an output that gets flushed on every write 560 output := ioutils.NewWriteFlusher(resp) 561 562 // Create the framer 563 framer := NewStreamFramer(output, false, streamHeartbeatRate, streamBatchWindow, streamFrameSize) 564 framer.Run() 565 defer framer.Destroy() 566 567 err = s.stream(offset, path, fs, framer, nil) 568 if err != nil && err != syscall.EPIPE { 569 return nil, err 570 } 571 572 return nil, nil 573 } 574 575 // stream is the internal method to stream the content of a file. eofCancelCh is 576 // used to cancel the stream if triggered while at EOF. If the connection is 577 // broken an EPIPE error is returned 578 func (s *HTTPServer) stream(offset int64, path string, 579 fs allocdir.AllocDirFS, framer *StreamFramer, 580 eofCancelCh chan error) error { 581 582 // Get the reader 583 f, err := fs.ReadAt(path, offset) 584 if err != nil { 585 return err 586 } 587 defer f.Close() 588 589 // Create a tomb to cancel watch events 590 t := tomb.Tomb{} 591 defer func() { 592 t.Kill(nil) 593 t.Done() 594 }() 595 596 // parseFramerErr takes an error and returns an error. The error will 597 // potentially change if it was caused by the connection being closed. 598 parseFramerErr := func(e error) error { 599 if e == nil { 600 return nil 601 } 602 603 if strings.Contains(e.Error(), io.ErrClosedPipe.Error()) { 604 // The pipe check is for tests 605 return syscall.EPIPE 606 } 607 608 // The connection was closed by our peer 609 if strings.Contains(e.Error(), syscall.EPIPE.Error()) || strings.Contains(e.Error(), syscall.ECONNRESET.Error()) { 610 return syscall.EPIPE 611 } 612 613 return err 614 } 615 616 // Create a variable to allow setting the last event 617 var lastEvent string 618 619 // Only create the file change watcher once. But we need to do it after we 620 // read and reach EOF. 621 var changes *watch.FileChanges 622 623 // Start streaming the data 624 data := make([]byte, streamFrameSize) 625 OUTER: 626 for { 627 // Read up to the max frame size 628 n, readErr := f.Read(data) 629 630 // Update the offset 631 offset += int64(n) 632 633 // Return non-EOF errors 634 if readErr != nil && readErr != io.EOF { 635 return readErr 636 } 637 638 // Send the frame 639 if n != 0 { 640 if err := framer.Send(path, lastEvent, data[:n], offset); err != nil { 641 return parseFramerErr(err) 642 } 643 } 644 645 // Clear the last event 646 if lastEvent != "" { 647 lastEvent = "" 648 } 649 650 // Just keep reading 651 if readErr == nil { 652 continue 653 } 654 655 // If EOF is hit, wait for a change to the file 656 if changes == nil { 657 changes, err = fs.ChangeEvents(path, offset, &t) 658 if err != nil { 659 return err 660 } 661 } 662 663 for { 664 select { 665 case <-changes.Modified: 666 continue OUTER 667 case <-changes.Deleted: 668 return parseFramerErr(framer.Send(path, deleteEvent, nil, offset)) 669 case <-changes.Truncated: 670 // Close the current reader 671 if err := f.Close(); err != nil { 672 return err 673 } 674 675 // Get a new reader at offset zero 676 offset = 0 677 var err error 678 f, err = fs.ReadAt(path, offset) 679 if err != nil { 680 return err 681 } 682 defer f.Close() 683 684 // Store the last event 685 lastEvent = truncateEvent 686 continue OUTER 687 case <-framer.ExitCh(): 688 return parseFramerErr(framer.Err) 689 case err, ok := <-eofCancelCh: 690 if !ok { 691 return nil 692 } 693 694 return err 695 } 696 } 697 } 698 } 699 700 // Logs streams the content of a log blocking on EOF. The parameters are: 701 // * task: task name to stream logs for. 702 // * type: stdout/stderr to stream. 703 // * follow: A boolean of whether to follow the logs. 704 // * offset: The offset to start streaming data at, defaults to zero. 705 // * origin: Either "start" or "end" and defines from where the offset is 706 // applied. Defaults to "start". 707 func (s *HTTPServer) Logs(resp http.ResponseWriter, req *http.Request) (interface{}, error) { 708 var allocID, task, logType string 709 var plain, follow bool 710 var err error 711 712 q := req.URL.Query() 713 714 if allocID = strings.TrimPrefix(req.URL.Path, "/v1/client/fs/logs/"); allocID == "" { 715 return nil, allocIDNotPresentErr 716 } 717 718 if task = q.Get("task"); task == "" { 719 return nil, taskNotPresentErr 720 } 721 722 if followStr := q.Get("follow"); followStr != "" { 723 if follow, err = strconv.ParseBool(followStr); err != nil { 724 return nil, fmt.Errorf("Failed to parse follow field to boolean: %v", err) 725 } 726 } 727 728 if plainStr := q.Get("plain"); plainStr != "" { 729 if plain, err = strconv.ParseBool(plainStr); err != nil { 730 return nil, fmt.Errorf("Failed to parse plain field to boolean: %v", err) 731 } 732 } 733 734 logType = q.Get("type") 735 switch logType { 736 case "stdout", "stderr": 737 default: 738 return nil, logTypeNotPresentErr 739 } 740 741 var offset int64 742 offsetString := q.Get("offset") 743 if offsetString != "" { 744 var err error 745 if offset, err = strconv.ParseInt(offsetString, 10, 64); err != nil { 746 return nil, fmt.Errorf("error parsing offset: %v", err) 747 } 748 } 749 750 origin := q.Get("origin") 751 switch origin { 752 case "start", "end": 753 case "": 754 origin = "start" 755 default: 756 return nil, invalidOrigin 757 } 758 759 fs, err := s.agent.client.GetAllocFS(allocID) 760 if err != nil { 761 return nil, err 762 } 763 764 alloc, err := s.agent.client.GetClientAlloc(allocID) 765 if err != nil { 766 return nil, err 767 } 768 769 // Check that the task is there 770 tg := alloc.Job.LookupTaskGroup(alloc.TaskGroup) 771 if tg == nil { 772 return nil, fmt.Errorf("Failed to lookup task group for allocation") 773 } else if taskStruct := tg.LookupTask(task); taskStruct == nil { 774 return nil, CodedError(404, fmt.Sprintf("task group %q does not have task with name %q", alloc.TaskGroup, task)) 775 } 776 777 state, ok := alloc.TaskStates[task] 778 if !ok || state.StartedAt.IsZero() { 779 return nil, CodedError(404, fmt.Sprintf("task %q not started yet. No logs available", task)) 780 } 781 782 // Create an output that gets flushed on every write 783 output := ioutils.NewWriteFlusher(resp) 784 785 return nil, s.logs(follow, plain, offset, origin, task, logType, fs, output) 786 } 787 788 func (s *HTTPServer) logs(follow, plain bool, offset int64, 789 origin, task, logType string, 790 fs allocdir.AllocDirFS, output io.WriteCloser) error { 791 792 // Create the framer 793 framer := NewStreamFramer(output, plain, streamHeartbeatRate, streamBatchWindow, streamFrameSize) 794 framer.Run() 795 defer framer.Destroy() 796 797 // Path to the logs 798 logPath := filepath.Join(allocdir.SharedAllocName, allocdir.LogDirName) 799 800 // nextIdx is the next index to read logs from 801 var nextIdx int64 802 switch origin { 803 case "start": 804 nextIdx = 0 805 case "end": 806 nextIdx = math.MaxInt64 807 offset *= -1 808 default: 809 return invalidOrigin 810 } 811 812 // Create a tomb to cancel watch events 813 t := tomb.Tomb{} 814 defer func() { 815 t.Kill(nil) 816 t.Done() 817 }() 818 819 for { 820 // Logic for picking next file is: 821 // 1) List log files 822 // 2) Pick log file closest to desired index 823 // 3) Open log file at correct offset 824 // 3a) No error, read contents 825 // 3b) If file doesn't exist, goto 1 as it may have been rotated out 826 entries, err := fs.List(logPath) 827 if err != nil { 828 return fmt.Errorf("failed to list entries: %v", err) 829 } 830 831 // If we are not following logs, determine the max index for the logs we are 832 // interested in so we can stop there. 833 maxIndex := int64(math.MaxInt64) 834 if !follow { 835 _, idx, _, err := findClosest(entries, maxIndex, 0, task, logType) 836 if err != nil { 837 return err 838 } 839 maxIndex = idx 840 } 841 842 logEntry, idx, openOffset, err := findClosest(entries, nextIdx, offset, task, logType) 843 if err != nil { 844 return err 845 } 846 847 var eofCancelCh chan error 848 exitAfter := false 849 if !follow && idx > maxIndex { 850 // Exceeded what was there initially so return 851 return nil 852 } else if !follow && idx == maxIndex { 853 // At the end 854 eofCancelCh = make(chan error) 855 close(eofCancelCh) 856 exitAfter = true 857 } else { 858 eofCancelCh = blockUntilNextLog(fs, &t, logPath, task, logType, idx+1) 859 } 860 861 p := filepath.Join(logPath, logEntry.Name) 862 err = s.stream(openOffset, p, fs, framer, eofCancelCh) 863 864 if err != nil { 865 // Check if there was an error where the file does not exist. That means 866 // it got rotated out from under us. 867 if os.IsNotExist(err) { 868 continue 869 } 870 871 // Check if the connection was closed 872 if err == syscall.EPIPE { 873 return nil 874 } 875 876 return fmt.Errorf("failed to stream %q: %v", p, err) 877 } 878 879 if exitAfter { 880 return nil 881 } 882 883 //Since we successfully streamed, update the overall offset/idx. 884 offset = int64(0) 885 nextIdx = idx + 1 886 } 887 } 888 889 // blockUntilNextLog returns a channel that will have data sent when the next 890 // log index or anything greater is created. 891 func blockUntilNextLog(fs allocdir.AllocDirFS, t *tomb.Tomb, logPath, task, logType string, nextIndex int64) chan error { 892 nextPath := filepath.Join(logPath, fmt.Sprintf("%s.%s.%d", task, logType, nextIndex)) 893 next := make(chan error, 1) 894 895 go func() { 896 eofCancelCh, err := fs.BlockUntilExists(nextPath, t) 897 if err != nil { 898 next <- err 899 close(next) 900 return 901 } 902 903 ticker := time.NewTicker(nextLogCheckRate) 904 defer ticker.Stop() 905 scanCh := ticker.C 906 for { 907 select { 908 case <-t.Dead(): 909 next <- fmt.Errorf("shutdown triggered") 910 close(next) 911 return 912 case err := <-eofCancelCh: 913 next <- err 914 close(next) 915 return 916 case <-scanCh: 917 entries, err := fs.List(logPath) 918 if err != nil { 919 next <- fmt.Errorf("failed to list entries: %v", err) 920 close(next) 921 return 922 } 923 924 indexes, err := logIndexes(entries, task, logType) 925 if err != nil { 926 next <- err 927 close(next) 928 return 929 } 930 931 // Scan and see if there are any entries larger than what we are 932 // waiting for. 933 for _, entry := range indexes { 934 if entry.idx >= nextIndex { 935 next <- nil 936 close(next) 937 return 938 } 939 } 940 } 941 } 942 }() 943 944 return next 945 } 946 947 // indexTuple and indexTupleArray are used to find the correct log entry to 948 // start streaming logs from 949 type indexTuple struct { 950 idx int64 951 entry *allocdir.AllocFileInfo 952 } 953 954 type indexTupleArray []indexTuple 955 956 func (a indexTupleArray) Len() int { return len(a) } 957 func (a indexTupleArray) Less(i, j int) bool { return a[i].idx < a[j].idx } 958 func (a indexTupleArray) Swap(i, j int) { a[i], a[j] = a[j], a[i] } 959 960 // logIndexes takes a set of entries and returns a indexTupleArray of 961 // the desired log file entries. If the indexes could not be determined, an 962 // error is returned. 963 func logIndexes(entries []*allocdir.AllocFileInfo, task, logType string) (indexTupleArray, error) { 964 var indexes []indexTuple 965 prefix := fmt.Sprintf("%s.%s.", task, logType) 966 for _, entry := range entries { 967 if entry.IsDir { 968 continue 969 } 970 971 // If nothing was trimmed, then it is not a match 972 idxStr := strings.TrimPrefix(entry.Name, prefix) 973 if idxStr == entry.Name { 974 continue 975 } 976 977 // Convert to an int 978 idx, err := strconv.Atoi(idxStr) 979 if err != nil { 980 return nil, fmt.Errorf("failed to convert %q to a log index: %v", idxStr, err) 981 } 982 983 indexes = append(indexes, indexTuple{idx: int64(idx), entry: entry}) 984 } 985 986 return indexTupleArray(indexes), nil 987 } 988 989 // findClosest takes a list of entries, the desired log index and desired log 990 // offset (which can be negative, treated as offset from end), task name and log 991 // type and returns the log entry, the log index, the offset to read from and a 992 // potential error. 993 func findClosest(entries []*allocdir.AllocFileInfo, desiredIdx, desiredOffset int64, 994 task, logType string) (*allocdir.AllocFileInfo, int64, int64, error) { 995 996 // Build the matching indexes 997 indexes, err := logIndexes(entries, task, logType) 998 if err != nil { 999 return nil, 0, 0, err 1000 } 1001 if len(indexes) == 0 { 1002 return nil, 0, 0, fmt.Errorf("log entry for task %q and log type %q not found", task, logType) 1003 } 1004 1005 // Binary search the indexes to get the desiredIdx 1006 sort.Sort(indexTupleArray(indexes)) 1007 i := sort.Search(len(indexes), func(i int) bool { return indexes[i].idx >= desiredIdx }) 1008 l := len(indexes) 1009 if i == l { 1010 // Use the last index if the number is bigger than all of them. 1011 i = l - 1 1012 } 1013 1014 // Get to the correct offset 1015 offset := desiredOffset 1016 idx := int64(i) 1017 for { 1018 s := indexes[idx].entry.Size 1019 1020 // Base case 1021 if offset == 0 { 1022 break 1023 } else if offset < 0 { 1024 // Going backwards 1025 if newOffset := s + offset; newOffset >= 0 { 1026 // Current file works 1027 offset = newOffset 1028 break 1029 } else if idx == 0 { 1030 // Already at the end 1031 offset = 0 1032 break 1033 } else { 1034 // Try the file before 1035 offset = newOffset 1036 idx -= 1 1037 continue 1038 } 1039 } else { 1040 // Going forward 1041 if offset <= s { 1042 // Current file works 1043 break 1044 } else if idx == int64(l-1) { 1045 // Already at the end 1046 offset = s 1047 break 1048 } else { 1049 // Try the next file 1050 offset = offset - s 1051 idx += 1 1052 continue 1053 } 1054 1055 } 1056 } 1057 1058 return indexes[idx].entry, indexes[idx].idx, offset, nil 1059 }