github.com/bshelton229/agent@v3.5.4+incompatible/agent/header_times_streamer.go (about) 1 package agent 2 3 import ( 4 "regexp" 5 "strconv" 6 "sync" 7 "time" 8 9 "github.com/buildkite/agent/logger" 10 ) 11 12 // If you change header parsing here make sure to change it in the 13 // buildkite.com frontend logic, too 14 15 var HeaderRegex = regexp.MustCompile("^(?:---|\\+\\+\\+|~~~)\\s(.+)?$") 16 var ANSIColorRegex = regexp.MustCompile(`\x1b\[([;\d]+)?[mK]`) 17 18 type HeaderTimesStreamer struct { 19 // The callback that will be called when a header time is ready for 20 // upload 21 UploadCallback func(int, int, map[string]string) 22 23 // The times that have found while scanning lines 24 times []string 25 timesMutex sync.Mutex 26 27 // Every time we get a new time, we increment the wait group, and 28 // decrement it after it has been uploaded. 29 uploadWaitGroup sync.WaitGroup 30 31 // Every time we go to scan a line, we increment the wait group, then 32 // decrement after it's finished scanning. That way when we stop the 33 // streamer, we can wait for all the lines to finish scanning first. 34 scanWaitGroup sync.WaitGroup 35 36 // A boolean to keep track if we're currently streaming header times 37 streaming bool 38 streamingMutex sync.Mutex 39 40 // We store the last index we uploaded to, so we don't have to keep 41 // uploading the same times 42 cursor int 43 } 44 45 func (h *HeaderTimesStreamer) Start() error { 46 h.streaming = true 47 48 go func() { 49 logger.Debug("[HeaderTimesStreamer] Streamer has started...") 50 51 for true { 52 // Break out of streaming if it's finished. We also 53 // need to aquire a read lock on the flag because it 54 // can be modified by other routines. 55 h.streamingMutex.Lock() 56 if !h.streaming { 57 break 58 } 59 h.streamingMutex.Unlock() 60 61 // Upload any pending header times 62 h.Upload() 63 64 // Sleep for a second and try upload some more later 65 time.Sleep(1 * time.Second) 66 } 67 68 logger.Debug("[HeaderTimesStreamer] Streamer has finished...") 69 }() 70 71 return nil 72 } 73 74 func (h *HeaderTimesStreamer) Scan(line string) { 75 // Keep track of how many line scans we need to do 76 h.scanWaitGroup.Add(1) 77 defer h.scanWaitGroup.Done() 78 79 if h.LineIsHeader(line) { 80 logger.Debug("[HeaderTimesStreamer] Found header %q", line) 81 82 // Aquire a lock on the times and then add the current time to 83 // our times slice. 84 h.timesMutex.Lock() 85 h.times = append(h.times, time.Now().UTC().Format(time.RFC3339Nano)) 86 h.timesMutex.Unlock() 87 88 // Add the time to the wait group 89 h.uploadWaitGroup.Add(1) 90 } 91 } 92 93 func (h *HeaderTimesStreamer) Upload() { 94 // Store the current cursor value 95 c := h.cursor 96 97 // Grab only the times that we haven't uploaded yet. We need to aquire 98 // a lock since other routines may be adding to it. 99 h.timesMutex.Lock() 100 length := len(h.times) 101 times := h.times[h.cursor:length] 102 h.timesMutex.Unlock() 103 104 // Construct the payload to send to the server 105 payload := map[string]string{} 106 for index, time := range times { 107 payload[strconv.Itoa(h.cursor+index)] = time 108 } 109 110 // Save the cursor we're up to 111 h.cursor = length 112 113 // How many times are we uploading this time 114 timesToUpload := len(times) 115 116 // Do we even have some times to upload 117 if timesToUpload > 0 { 118 // Call our callback with the times for upload 119 logger.Debug("[HeaderTimesStreamer] Uploading header times %d..%d", c, length-1) 120 h.UploadCallback(c, length, payload) 121 logger.Debug("[HeaderTimesStreamer] Finished uploading header times %d..%d", c, length-1) 122 123 // Decrement the wait group for every time we've uploaded. 124 h.uploadWaitGroup.Add(timesToUpload * -1) 125 } 126 } 127 128 func (h *HeaderTimesStreamer) Stop() { 129 logger.Debug("[HeaderTimesStreamer] Waiting for all the lines to be scanned") 130 h.scanWaitGroup.Wait() 131 132 logger.Debug("[HeaderTimesStreamer] Waiting for all the header times to be uploaded") 133 h.uploadWaitGroup.Wait() 134 135 // Since we're modifying the waitGroup and the streaming flag, we need 136 // to aquire a write lock. 137 h.streamingMutex.Lock() 138 h.streaming = false 139 h.streamingMutex.Unlock() 140 } 141 142 func (h *HeaderTimesStreamer) LinePreProcessor(line string) string { 143 // Make sure all ANSI colors are removed from the string before we 144 // check to see if it's a header (sometimes a color escape sequence may 145 // be the first thing on the line, which will cause the regex to ignore 146 // it) 147 return ANSIColorRegex.ReplaceAllString(line, "") 148 } 149 150 func (h *HeaderTimesStreamer) LineIsHeader(line string) bool { 151 // To avoid running the regex over every single line, we'll first do a 152 // length check. Hopefully there are no heeaders over 500 characters! 153 return len(line) < 500 && HeaderRegex.MatchString(line) 154 }