github.com/demonoid81/moby@v0.0.0-20200517203328-62dd8e17c460/daemon/logger/copier.go (about) 1 package logger // import "github.com/demonoid81/moby/daemon/logger" 2 3 import ( 4 "bytes" 5 "io" 6 "sync" 7 "time" 8 9 types "github.com/demonoid81/moby/api/types/backend" 10 "github.com/demonoid81/moby/pkg/stringid" 11 "github.com/sirupsen/logrus" 12 ) 13 14 const ( 15 // readSize is the maximum bytes read during a single read 16 // operation. 17 readSize = 2 * 1024 18 19 // defaultBufSize provides a reasonable default for loggers that do 20 // not have an external limit to impose on log line size. 21 defaultBufSize = 16 * 1024 22 ) 23 24 // Copier can copy logs from specified sources to Logger and attach Timestamp. 25 // Writes are concurrent, so you need implement some sync in your logger. 26 type Copier struct { 27 // srcs is map of name -> reader pairs, for example "stdout", "stderr" 28 srcs map[string]io.Reader 29 dst Logger 30 copyJobs sync.WaitGroup 31 closeOnce sync.Once 32 closed chan struct{} 33 } 34 35 // NewCopier creates a new Copier 36 func NewCopier(srcs map[string]io.Reader, dst Logger) *Copier { 37 return &Copier{ 38 srcs: srcs, 39 dst: dst, 40 closed: make(chan struct{}), 41 } 42 } 43 44 // Run starts logs copying 45 func (c *Copier) Run() { 46 for src, w := range c.srcs { 47 c.copyJobs.Add(1) 48 go c.copySrc(src, w) 49 } 50 } 51 52 func (c *Copier) copySrc(name string, src io.Reader) { 53 defer c.copyJobs.Done() 54 55 bufSize := defaultBufSize 56 if sizedLogger, ok := c.dst.(SizedLogger); ok { 57 bufSize = sizedLogger.BufSize() 58 } 59 buf := make([]byte, bufSize) 60 61 n := 0 62 eof := false 63 var partialid string 64 var partialTS time.Time 65 var ordinal int 66 firstPartial := true 67 hasMorePartial := false 68 69 for { 70 select { 71 case <-c.closed: 72 return 73 default: 74 // Work out how much more data we are okay with reading this time. 75 upto := n + readSize 76 if upto > cap(buf) { 77 upto = cap(buf) 78 } 79 // Try to read that data. 80 if upto > n { 81 read, err := src.Read(buf[n:upto]) 82 if err != nil { 83 if err != io.EOF { 84 logReadsFailedCount.Inc(1) 85 logrus.Errorf("Error scanning log stream: %s", err) 86 return 87 } 88 eof = true 89 } 90 n += read 91 } 92 // If we have no data to log, and there's no more coming, we're done. 93 if n == 0 && eof { 94 return 95 } 96 // Break up the data that we've buffered up into lines, and log each in turn. 97 p := 0 98 99 for q := bytes.IndexByte(buf[p:n], '\n'); q >= 0; q = bytes.IndexByte(buf[p:n], '\n') { 100 select { 101 case <-c.closed: 102 return 103 default: 104 msg := NewMessage() 105 msg.Source = name 106 msg.Line = append(msg.Line, buf[p:p+q]...) 107 108 if hasMorePartial { 109 msg.PLogMetaData = &types.PartialLogMetaData{ID: partialid, Ordinal: ordinal, Last: true} 110 111 // reset 112 partialid = "" 113 ordinal = 0 114 firstPartial = true 115 hasMorePartial = false 116 } 117 if msg.PLogMetaData == nil { 118 msg.Timestamp = time.Now().UTC() 119 } else { 120 msg.Timestamp = partialTS 121 } 122 123 if logErr := c.dst.Log(msg); logErr != nil { 124 logWritesFailedCount.Inc(1) 125 logrus.Errorf("Failed to log msg %q for logger %s: %s", msg.Line, c.dst.Name(), logErr) 126 } 127 } 128 p += q + 1 129 } 130 // If there's no more coming, or the buffer is full but 131 // has no newlines, log whatever we haven't logged yet, 132 // noting that it's a partial log line. 133 if eof || (p == 0 && n == len(buf)) { 134 if p < n { 135 msg := NewMessage() 136 msg.Source = name 137 msg.Line = append(msg.Line, buf[p:n]...) 138 139 // Generate unique partialID for first partial. Use it across partials. 140 // Record timestamp for first partial. Use it across partials. 141 // Initialize Ordinal for first partial. Increment it across partials. 142 if firstPartial { 143 msg.Timestamp = time.Now().UTC() 144 partialTS = msg.Timestamp 145 partialid = stringid.GenerateRandomID() 146 ordinal = 1 147 firstPartial = false 148 totalPartialLogs.Inc(1) 149 } else { 150 msg.Timestamp = partialTS 151 } 152 msg.PLogMetaData = &types.PartialLogMetaData{ID: partialid, Ordinal: ordinal, Last: false} 153 ordinal++ 154 hasMorePartial = true 155 156 if logErr := c.dst.Log(msg); logErr != nil { 157 logWritesFailedCount.Inc(1) 158 logrus.Errorf("Failed to log msg %q for logger %s: %s", msg.Line, c.dst.Name(), logErr) 159 } 160 p = 0 161 n = 0 162 } 163 if eof { 164 return 165 } 166 } 167 // Move any unlogged data to the front of the buffer in preparation for another read. 168 if p > 0 { 169 copy(buf[0:], buf[p:n]) 170 n -= p 171 } 172 } 173 } 174 } 175 176 // Wait waits until all copying is done 177 func (c *Copier) Wait() { 178 c.copyJobs.Wait() 179 } 180 181 // Close closes the copier 182 func (c *Copier) Close() { 183 c.closeOnce.Do(func() { 184 close(c.closed) 185 }) 186 }