github.com/stevenmatthewt/agent@v3.5.4+incompatible/agent/log_streamer.go (about) 1 package agent 2 3 import ( 4 "errors" 5 "math" 6 "sync" 7 "sync/atomic" 8 9 "github.com/buildkite/agent/logger" 10 ) 11 12 type LogStreamer struct { 13 // How many log streamer workers are running at any one time 14 Concurrency int 15 16 // The maximum size of chunks 17 MaxChunkSizeBytes int 18 19 // A counter of how many chunks failed to upload 20 ChunksFailedCount int32 21 22 // The callback called when a chunk is ready for upload 23 Callback func(chunk *LogStreamerChunk) error 24 25 // The queue of chunks that are needing to be uploaded 26 queue chan *LogStreamerChunk 27 28 // Total size in bytes of the log 29 bytes int 30 31 // Each chunk is assigned an order 32 order int 33 34 // Every time we add a job to the queue, we increase the wait group 35 // queue so when the streamer shuts down, we can block until all work 36 // has been added. 37 chunkWaitGroup sync.WaitGroup 38 39 // Only allow processing one at a time 40 processMutex sync.Mutex 41 } 42 43 type LogStreamerChunk struct { 44 // The contents of the chunk 45 Data string 46 47 // The sequence number of this chunk 48 Order int 49 50 // The byte offset of this chunk 51 Offset int 52 53 // The byte size of this chunk 54 Size int 55 } 56 57 // Creates a new instance of the log streamer 58 func (ls LogStreamer) New() *LogStreamer { 59 ls.Concurrency = 3 60 ls.queue = make(chan *LogStreamerChunk, 1024) 61 62 return &ls 63 } 64 65 // Spins up x number of log streamer workers 66 func (ls *LogStreamer) Start() error { 67 if ls.MaxChunkSizeBytes == 0 { 68 return errors.New("Maximum chunk size must be more than 0. No logs will be sent.") 69 } 70 71 for i := 0; i < ls.Concurrency; i++ { 72 go Worker(i, ls) 73 } 74 75 return nil 76 } 77 78 // Takes the full process output, grabs the portion we don't have, and adds it 79 // to the stream queue 80 func (ls *LogStreamer) Process(output string) error { 81 bytes := len(output) 82 83 // Only allow one streamer process at a time 84 ls.processMutex.Lock() 85 86 if ls.bytes != bytes { 87 // Grab the part of the log that we haven't seen yet 88 blob := output[ls.bytes:bytes] 89 90 // How many chunks do we have that fit within the MaxChunkSizeBytes? 91 numberOfChunks := int(math.Ceil(float64(len(blob)) / float64(ls.MaxChunkSizeBytes))) 92 93 // Increase the wait group by the amount of chunks we're going 94 // to add 95 ls.chunkWaitGroup.Add(numberOfChunks) 96 97 for i := 0; i < numberOfChunks; i++ { 98 // Find the upper limit of the blob 99 upperLimit := (i + 1) * ls.MaxChunkSizeBytes 100 if upperLimit > len(blob) { 101 upperLimit = len(blob) 102 } 103 104 // Grab the 100kb section of the blob 105 partialChunk := blob[i*ls.MaxChunkSizeBytes : upperLimit] 106 107 // Increment the order 108 ls.order += 1 109 110 // Create the chunk and append it to our list 111 chunk := LogStreamerChunk{ 112 Data: partialChunk, 113 Order: ls.order, 114 Offset: ls.bytes, 115 Size: bytes - ls.bytes, 116 } 117 118 ls.queue <- &chunk 119 } 120 121 // Save the new amount of bytes 122 ls.bytes = bytes 123 } 124 125 ls.processMutex.Unlock() 126 127 return nil 128 } 129 130 // Waits for all the chunks to be uploaded, then shuts down all the workers 131 func (ls *LogStreamer) Stop() error { 132 logger.Debug("[LogStreamer] Waiting for all the chunks to be uploaded") 133 134 ls.chunkWaitGroup.Wait() 135 136 logger.Debug("[LogStreamer] Shutting down all workers") 137 138 for n := 0; n < ls.Concurrency; n++ { 139 ls.queue <- nil 140 } 141 142 return nil 143 } 144 145 // The actual log streamer worker 146 func Worker(id int, ls *LogStreamer) { 147 logger.Debug("[LogStreamer/Worker#%d] Worker is starting...", id) 148 149 var chunk *LogStreamerChunk 150 for { 151 // Get the next chunk (pointer) from the queue. This will block 152 // until something is returned. 153 chunk = <-ls.queue 154 155 // If the next chunk is nil, then there is no more work to do 156 if chunk == nil { 157 break 158 } 159 160 // Upload the chunk 161 err := ls.Callback(chunk) 162 if err != nil { 163 atomic.AddInt32(&ls.ChunksFailedCount, 1) 164 165 logger.Error("Giving up on uploading chunk %d, this will result in only a partial build log on Buildkite", chunk.Order) 166 } 167 168 // Signal to the chunkWaitGroup that this one is done 169 ls.chunkWaitGroup.Done() 170 } 171 172 logger.Debug("[LogStreamer/Worker#%d] Worker has shutdown", id) 173 }