github.com/demonoid81/moby@v0.0.0-20200517203328-62dd8e17c460/daemon/logger/awslogs/cloudwatchlogs.go (about) 1 // Package awslogs provides the logdriver for forwarding container logs to Amazon CloudWatch Logs 2 package awslogs // import "github.com/demonoid81/moby/daemon/logger/awslogs" 3 4 import ( 5 "fmt" 6 "os" 7 "regexp" 8 "runtime" 9 "sort" 10 "strconv" 11 "strings" 12 "sync" 13 "time" 14 "unicode/utf8" 15 16 "github.com/aws/aws-sdk-go/aws" 17 "github.com/aws/aws-sdk-go/aws/awserr" 18 "github.com/aws/aws-sdk-go/aws/credentials/endpointcreds" 19 "github.com/aws/aws-sdk-go/aws/ec2metadata" 20 "github.com/aws/aws-sdk-go/aws/request" 21 "github.com/aws/aws-sdk-go/aws/session" 22 "github.com/aws/aws-sdk-go/service/cloudwatchlogs" 23 "github.com/demonoid81/moby/daemon/logger" 24 "github.com/demonoid81/moby/daemon/logger/loggerutils" 25 "github.com/demonoid81/moby/dockerversion" 26 "github.com/pkg/errors" 27 "github.com/sirupsen/logrus" 28 ) 29 30 const ( 31 name = "awslogs" 32 regionKey = "awslogs-region" 33 endpointKey = "awslogs-endpoint" 34 regionEnvKey = "AWS_REGION" 35 logGroupKey = "awslogs-group" 36 logStreamKey = "awslogs-stream" 37 logCreateGroupKey = "awslogs-create-group" 38 tagKey = "tag" 39 datetimeFormatKey = "awslogs-datetime-format" 40 multilinePatternKey = "awslogs-multiline-pattern" 41 credentialsEndpointKey = "awslogs-credentials-endpoint" 42 forceFlushIntervalKey = "awslogs-force-flush-interval-seconds" 43 maxBufferedEventsKey = "awslogs-max-buffered-events" 44 45 defaultForceFlushInterval = 5 * time.Second 46 defaultMaxBufferedEvents = 4096 47 48 // See: http://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_PutLogEvents.html 49 perEventBytes = 26 50 maximumBytesPerPut = 1048576 51 maximumLogEventsPerPut = 10000 52 53 // See: http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/cloudwatch_limits.html 54 // Because the events are interpreted as UTF-8 encoded Unicode, invalid UTF-8 byte sequences are replaced with the 55 // Unicode replacement character (U+FFFD), which is a 3-byte sequence in UTF-8. To compensate for that and to avoid 56 // splitting valid UTF-8 characters into invalid byte sequences, we calculate the length of each event assuming that 57 // this replacement happens. 58 maximumBytesPerEvent = 262144 - perEventBytes 59 60 resourceAlreadyExistsCode = "ResourceAlreadyExistsException" 61 dataAlreadyAcceptedCode = "DataAlreadyAcceptedException" 62 invalidSequenceTokenCode = "InvalidSequenceTokenException" 63 resourceNotFoundCode = "ResourceNotFoundException" 64 65 credentialsEndpoint = "http://169.254.170.2" 66 67 userAgentHeader = "User-Agent" 68 ) 69 70 type logStream struct { 71 logStreamName string 72 logGroupName string 73 logCreateGroup bool 74 logNonBlocking bool 75 forceFlushInterval time.Duration 76 multilinePattern *regexp.Regexp 77 client api 78 messages chan *logger.Message 79 lock sync.RWMutex 80 closed bool 81 sequenceToken *string 82 } 83 84 type logStreamConfig struct { 85 logStreamName string 86 logGroupName string 87 logCreateGroup bool 88 logNonBlocking bool 89 forceFlushInterval time.Duration 90 maxBufferedEvents int 91 multilinePattern *regexp.Regexp 92 } 93 94 var _ logger.SizedLogger = &logStream{} 95 96 type api interface { 97 CreateLogGroup(*cloudwatchlogs.CreateLogGroupInput) (*cloudwatchlogs.CreateLogGroupOutput, error) 98 CreateLogStream(*cloudwatchlogs.CreateLogStreamInput) (*cloudwatchlogs.CreateLogStreamOutput, error) 99 PutLogEvents(*cloudwatchlogs.PutLogEventsInput) (*cloudwatchlogs.PutLogEventsOutput, error) 100 } 101 102 type regionFinder interface { 103 Region() (string, error) 104 } 105 106 type wrappedEvent struct { 107 inputLogEvent *cloudwatchlogs.InputLogEvent 108 insertOrder int 109 } 110 type byTimestamp []wrappedEvent 111 112 // init registers the awslogs driver 113 func init() { 114 if err := logger.RegisterLogDriver(name, New); err != nil { 115 logrus.Fatal(err) 116 } 117 if err := logger.RegisterLogOptValidator(name, ValidateLogOpt); err != nil { 118 logrus.Fatal(err) 119 } 120 } 121 122 // eventBatch holds the events that are batched for submission and the 123 // associated data about it. 124 // 125 // Warning: this type is not threadsafe and must not be used 126 // concurrently. This type is expected to be consumed in a single go 127 // routine and never concurrently. 128 type eventBatch struct { 129 batch []wrappedEvent 130 bytes int 131 } 132 133 // New creates an awslogs logger using the configuration passed in on the 134 // context. Supported context configuration variables are awslogs-region, 135 // awslogs-endpoint, awslogs-group, awslogs-stream, awslogs-create-group, 136 // awslogs-multiline-pattern and awslogs-datetime-format. 137 // When available, configuration is also taken from environment variables 138 // AWS_REGION, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, the shared credentials 139 // file (~/.aws/credentials), and the EC2 Instance Metadata Service. 140 func New(info logger.Info) (logger.Logger, error) { 141 containerStreamConfig, err := newStreamConfig(info) 142 if err != nil { 143 return nil, err 144 } 145 client, err := newAWSLogsClient(info) 146 if err != nil { 147 return nil, err 148 } 149 150 containerStream := &logStream{ 151 logStreamName: containerStreamConfig.logStreamName, 152 logGroupName: containerStreamConfig.logGroupName, 153 logCreateGroup: containerStreamConfig.logCreateGroup, 154 logNonBlocking: containerStreamConfig.logNonBlocking, 155 forceFlushInterval: containerStreamConfig.forceFlushInterval, 156 multilinePattern: containerStreamConfig.multilinePattern, 157 client: client, 158 messages: make(chan *logger.Message, containerStreamConfig.maxBufferedEvents), 159 } 160 161 creationDone := make(chan bool) 162 if containerStream.logNonBlocking { 163 go func() { 164 backoff := 1 165 maxBackoff := 32 166 for { 167 // If logger is closed we are done 168 containerStream.lock.RLock() 169 if containerStream.closed { 170 containerStream.lock.RUnlock() 171 break 172 } 173 containerStream.lock.RUnlock() 174 err := containerStream.create() 175 if err == nil { 176 break 177 } 178 179 time.Sleep(time.Duration(backoff) * time.Second) 180 if backoff < maxBackoff { 181 backoff *= 2 182 } 183 logrus. 184 WithError(err). 185 WithField("container-id", info.ContainerID). 186 WithField("container-name", info.ContainerName). 187 Error("Error while trying to initialize awslogs. Retrying in: ", backoff, " seconds") 188 } 189 close(creationDone) 190 }() 191 } else { 192 if err = containerStream.create(); err != nil { 193 return nil, err 194 } 195 close(creationDone) 196 } 197 go containerStream.collectBatch(creationDone) 198 199 return containerStream, nil 200 } 201 202 // Parses most of the awslogs- options and prepares a config object to be used for newing the actual stream 203 // It has been formed out to ease Utest of the New above 204 func newStreamConfig(info logger.Info) (*logStreamConfig, error) { 205 logGroupName := info.Config[logGroupKey] 206 logStreamName, err := loggerutils.ParseLogTag(info, "{{.FullID}}") 207 if err != nil { 208 return nil, err 209 } 210 logCreateGroup := false 211 if info.Config[logCreateGroupKey] != "" { 212 logCreateGroup, err = strconv.ParseBool(info.Config[logCreateGroupKey]) 213 if err != nil { 214 return nil, err 215 } 216 } 217 218 logNonBlocking := info.Config["mode"] == "non-blocking" 219 220 forceFlushInterval := defaultForceFlushInterval 221 if info.Config[forceFlushIntervalKey] != "" { 222 forceFlushIntervalAsInt, err := strconv.Atoi(info.Config[forceFlushIntervalKey]) 223 if err != nil { 224 return nil, err 225 } 226 forceFlushInterval = time.Duration(forceFlushIntervalAsInt) * time.Second 227 } 228 229 maxBufferedEvents := int(defaultMaxBufferedEvents) 230 if info.Config[maxBufferedEventsKey] != "" { 231 maxBufferedEvents, err = strconv.Atoi(info.Config[maxBufferedEventsKey]) 232 if err != nil { 233 return nil, err 234 } 235 } 236 237 if info.Config[logStreamKey] != "" { 238 logStreamName = info.Config[logStreamKey] 239 } 240 241 multilinePattern, err := parseMultilineOptions(info) 242 if err != nil { 243 return nil, err 244 } 245 246 containerStreamConfig := &logStreamConfig{ 247 logStreamName: logStreamName, 248 logGroupName: logGroupName, 249 logCreateGroup: logCreateGroup, 250 logNonBlocking: logNonBlocking, 251 forceFlushInterval: forceFlushInterval, 252 maxBufferedEvents: maxBufferedEvents, 253 multilinePattern: multilinePattern, 254 } 255 256 return containerStreamConfig, nil 257 } 258 259 // Parses awslogs-multiline-pattern and awslogs-datetime-format options 260 // If awslogs-datetime-format is present, convert the format from strftime 261 // to regexp and return. 262 // If awslogs-multiline-pattern is present, compile regexp and return 263 func parseMultilineOptions(info logger.Info) (*regexp.Regexp, error) { 264 dateTimeFormat := info.Config[datetimeFormatKey] 265 multilinePatternKey := info.Config[multilinePatternKey] 266 // strftime input is parsed into a regular expression 267 if dateTimeFormat != "" { 268 // %. matches each strftime format sequence and ReplaceAllStringFunc 269 // looks up each format sequence in the conversion table strftimeToRegex 270 // to replace with a defined regular expression 271 r := regexp.MustCompile("%.") 272 multilinePatternKey = r.ReplaceAllStringFunc(dateTimeFormat, func(s string) string { 273 return strftimeToRegex[s] 274 }) 275 } 276 if multilinePatternKey != "" { 277 multilinePattern, err := regexp.Compile(multilinePatternKey) 278 if err != nil { 279 return nil, errors.Wrapf(err, "awslogs could not parse multiline pattern key %q", multilinePatternKey) 280 } 281 return multilinePattern, nil 282 } 283 return nil, nil 284 } 285 286 // Maps strftime format strings to regex 287 var strftimeToRegex = map[string]string{ 288 /*weekdayShort */ `%a`: `(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)`, 289 /*weekdayFull */ `%A`: `(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday)`, 290 /*weekdayZeroIndex */ `%w`: `[0-6]`, 291 /*dayZeroPadded */ `%d`: `(?:0[1-9]|[1,2][0-9]|3[0,1])`, 292 /*monthShort */ `%b`: `(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)`, 293 /*monthFull */ `%B`: `(?:January|February|March|April|May|June|July|August|September|October|November|December)`, 294 /*monthZeroPadded */ `%m`: `(?:0[1-9]|1[0-2])`, 295 /*yearCentury */ `%Y`: `\d{4}`, 296 /*yearZeroPadded */ `%y`: `\d{2}`, 297 /*hour24ZeroPadded */ `%H`: `(?:[0,1][0-9]|2[0-3])`, 298 /*hour12ZeroPadded */ `%I`: `(?:0[0-9]|1[0-2])`, 299 /*AM or PM */ `%p`: "[A,P]M", 300 /*minuteZeroPadded */ `%M`: `[0-5][0-9]`, 301 /*secondZeroPadded */ `%S`: `[0-5][0-9]`, 302 /*microsecondZeroPadded */ `%f`: `\d{6}`, 303 /*utcOffset */ `%z`: `[+-]\d{4}`, 304 /*tzName */ `%Z`: `[A-Z]{1,4}T`, 305 /*dayOfYearZeroPadded */ `%j`: `(?:0[0-9][1-9]|[1,2][0-9][0-9]|3[0-5][0-9]|36[0-6])`, 306 /*milliseconds */ `%L`: `\.\d{3}`, 307 } 308 309 // newRegionFinder is a variable such that the implementation 310 // can be swapped out for unit tests. 311 var newRegionFinder = func() (regionFinder, error) { 312 s, err := session.NewSession() 313 if err != nil { 314 return nil, err 315 } 316 return ec2metadata.New(s), nil 317 } 318 319 // newSDKEndpoint is a variable such that the implementation 320 // can be swapped out for unit tests. 321 var newSDKEndpoint = credentialsEndpoint 322 323 // newAWSLogsClient creates the service client for Amazon CloudWatch Logs. 324 // Customizations to the default client from the SDK include a Docker-specific 325 // User-Agent string and automatic region detection using the EC2 Instance 326 // Metadata Service when region is otherwise unspecified. 327 func newAWSLogsClient(info logger.Info) (api, error) { 328 var region, endpoint *string 329 if os.Getenv(regionEnvKey) != "" { 330 region = aws.String(os.Getenv(regionEnvKey)) 331 } 332 if info.Config[regionKey] != "" { 333 region = aws.String(info.Config[regionKey]) 334 } 335 if info.Config[endpointKey] != "" { 336 endpoint = aws.String(info.Config[endpointKey]) 337 } 338 if region == nil || *region == "" { 339 logrus.Info("Trying to get region from EC2 Metadata") 340 ec2MetadataClient, err := newRegionFinder() 341 if err != nil { 342 logrus.WithError(err).Error("could not create EC2 metadata client") 343 return nil, errors.Wrap(err, "could not create EC2 metadata client") 344 } 345 346 r, err := ec2MetadataClient.Region() 347 if err != nil { 348 logrus.WithError(err).Error("Could not get region from EC2 metadata, environment, or log option") 349 return nil, errors.New("Cannot determine region for awslogs driver") 350 } 351 region = &r 352 } 353 354 sess, err := session.NewSession() 355 if err != nil { 356 return nil, errors.New("Failed to create a service client session for awslogs driver") 357 } 358 359 // attach region to cloudwatchlogs config 360 sess.Config.Region = region 361 362 // attach endpoint to cloudwatchlogs config 363 if endpoint != nil { 364 sess.Config.Endpoint = endpoint 365 } 366 367 if uri, ok := info.Config[credentialsEndpointKey]; ok { 368 logrus.Debugf("Trying to get credentials from awslogs-credentials-endpoint") 369 370 endpoint := fmt.Sprintf("%s%s", newSDKEndpoint, uri) 371 creds := endpointcreds.NewCredentialsClient(*sess.Config, sess.Handlers, endpoint, 372 func(p *endpointcreds.Provider) { 373 p.ExpiryWindow = 5 * time.Minute 374 }) 375 376 // attach credentials to cloudwatchlogs config 377 sess.Config.Credentials = creds 378 } 379 380 logrus.WithFields(logrus.Fields{ 381 "region": *region, 382 }).Debug("Created awslogs client") 383 384 client := cloudwatchlogs.New(sess) 385 386 client.Handlers.Build.PushBackNamed(request.NamedHandler{ 387 Name: "DockerUserAgentHandler", 388 Fn: func(r *request.Request) { 389 currentAgent := r.HTTPRequest.Header.Get(userAgentHeader) 390 r.HTTPRequest.Header.Set(userAgentHeader, 391 fmt.Sprintf("Docker %s (%s) %s", 392 dockerversion.Version, runtime.GOOS, currentAgent)) 393 }, 394 }) 395 return client, nil 396 } 397 398 // Name returns the name of the awslogs logging driver 399 func (l *logStream) Name() string { 400 return name 401 } 402 403 func (l *logStream) BufSize() int { 404 return maximumBytesPerEvent 405 } 406 407 // Log submits messages for logging by an instance of the awslogs logging driver 408 func (l *logStream) Log(msg *logger.Message) error { 409 l.lock.RLock() 410 defer l.lock.RUnlock() 411 if l.closed { 412 return errors.New("awslogs is closed") 413 } 414 if l.logNonBlocking { 415 select { 416 case l.messages <- msg: 417 return nil 418 default: 419 return errors.New("awslogs buffer is full") 420 } 421 } 422 l.messages <- msg 423 return nil 424 } 425 426 // Close closes the instance of the awslogs logging driver 427 func (l *logStream) Close() error { 428 l.lock.Lock() 429 defer l.lock.Unlock() 430 if !l.closed { 431 close(l.messages) 432 } 433 l.closed = true 434 return nil 435 } 436 437 // create creates log group and log stream for the instance of the awslogs logging driver 438 func (l *logStream) create() error { 439 err := l.createLogStream() 440 if err == nil { 441 return nil 442 } 443 if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == resourceNotFoundCode && l.logCreateGroup { 444 if err := l.createLogGroup(); err != nil { 445 return errors.Wrap(err, "failed to create Cloudwatch log group") 446 } 447 err = l.createLogStream() 448 if err == nil { 449 return nil 450 } 451 } 452 return errors.Wrap(err, "failed to create Cloudwatch log stream") 453 } 454 455 // createLogGroup creates a log group for the instance of the awslogs logging driver 456 func (l *logStream) createLogGroup() error { 457 if _, err := l.client.CreateLogGroup(&cloudwatchlogs.CreateLogGroupInput{ 458 LogGroupName: aws.String(l.logGroupName), 459 }); err != nil { 460 if awsErr, ok := err.(awserr.Error); ok { 461 fields := logrus.Fields{ 462 "errorCode": awsErr.Code(), 463 "message": awsErr.Message(), 464 "origError": awsErr.OrigErr(), 465 "logGroupName": l.logGroupName, 466 "logCreateGroup": l.logCreateGroup, 467 } 468 if awsErr.Code() == resourceAlreadyExistsCode { 469 // Allow creation to succeed 470 logrus.WithFields(fields).Info("Log group already exists") 471 return nil 472 } 473 logrus.WithFields(fields).Error("Failed to create log group") 474 } 475 return err 476 } 477 return nil 478 } 479 480 // createLogStream creates a log stream for the instance of the awslogs logging driver 481 func (l *logStream) createLogStream() error { 482 input := &cloudwatchlogs.CreateLogStreamInput{ 483 LogGroupName: aws.String(l.logGroupName), 484 LogStreamName: aws.String(l.logStreamName), 485 } 486 487 _, err := l.client.CreateLogStream(input) 488 489 if err != nil { 490 if awsErr, ok := err.(awserr.Error); ok { 491 fields := logrus.Fields{ 492 "errorCode": awsErr.Code(), 493 "message": awsErr.Message(), 494 "origError": awsErr.OrigErr(), 495 "logGroupName": l.logGroupName, 496 "logStreamName": l.logStreamName, 497 } 498 if awsErr.Code() == resourceAlreadyExistsCode { 499 // Allow creation to succeed 500 logrus.WithFields(fields).Info("Log stream already exists") 501 return nil 502 } 503 logrus.WithFields(fields).Error("Failed to create log stream") 504 } 505 } 506 return err 507 } 508 509 // newTicker is used for time-based batching. newTicker is a variable such 510 // that the implementation can be swapped out for unit tests. 511 var newTicker = func(freq time.Duration) *time.Ticker { 512 return time.NewTicker(freq) 513 } 514 515 // collectBatch executes as a goroutine to perform batching of log events for 516 // submission to the log stream. If the awslogs-multiline-pattern or 517 // awslogs-datetime-format options have been configured, multiline processing 518 // is enabled, where log messages are stored in an event buffer until a multiline 519 // pattern match is found, at which point the messages in the event buffer are 520 // pushed to CloudWatch logs as a single log event. Multiline messages are processed 521 // according to the maximumBytesPerPut constraint, and the implementation only 522 // allows for messages to be buffered for a maximum of 2*batchPublishFrequency 523 // seconds. When events are ready to be processed for submission to CloudWatch 524 // Logs, the processEvents method is called. If a multiline pattern is not 525 // configured, log events are submitted to the processEvents method immediately. 526 func (l *logStream) collectBatch(created chan bool) { 527 // Wait for the logstream/group to be created 528 <-created 529 flushInterval := l.forceFlushInterval 530 if flushInterval <= 0 { 531 flushInterval = defaultForceFlushInterval 532 } 533 ticker := newTicker(flushInterval) 534 var eventBuffer []byte 535 var eventBufferTimestamp int64 536 var batch = newEventBatch() 537 for { 538 select { 539 case t := <-ticker.C: 540 // If event buffer is older than batch publish frequency flush the event buffer 541 if eventBufferTimestamp > 0 && len(eventBuffer) > 0 { 542 eventBufferAge := t.UnixNano()/int64(time.Millisecond) - eventBufferTimestamp 543 eventBufferExpired := eventBufferAge >= int64(flushInterval)/int64(time.Millisecond) 544 eventBufferNegative := eventBufferAge < 0 545 if eventBufferExpired || eventBufferNegative { 546 l.processEvent(batch, eventBuffer, eventBufferTimestamp) 547 eventBuffer = eventBuffer[:0] 548 } 549 } 550 l.publishBatch(batch) 551 batch.reset() 552 case msg, more := <-l.messages: 553 if !more { 554 // Flush event buffer and release resources 555 l.processEvent(batch, eventBuffer, eventBufferTimestamp) 556 l.publishBatch(batch) 557 batch.reset() 558 return 559 } 560 if eventBufferTimestamp == 0 { 561 eventBufferTimestamp = msg.Timestamp.UnixNano() / int64(time.Millisecond) 562 } 563 line := msg.Line 564 if l.multilinePattern != nil { 565 lineEffectiveLen := effectiveLen(string(line)) 566 if l.multilinePattern.Match(line) || effectiveLen(string(eventBuffer))+lineEffectiveLen > maximumBytesPerEvent { 567 // This is a new log event or we will exceed max bytes per event 568 // so flush the current eventBuffer to events and reset timestamp 569 l.processEvent(batch, eventBuffer, eventBufferTimestamp) 570 eventBufferTimestamp = msg.Timestamp.UnixNano() / int64(time.Millisecond) 571 eventBuffer = eventBuffer[:0] 572 } 573 // Append newline if event is less than max event size 574 if lineEffectiveLen < maximumBytesPerEvent { 575 line = append(line, "\n"...) 576 } 577 eventBuffer = append(eventBuffer, line...) 578 logger.PutMessage(msg) 579 } else { 580 l.processEvent(batch, line, msg.Timestamp.UnixNano()/int64(time.Millisecond)) 581 logger.PutMessage(msg) 582 } 583 } 584 } 585 } 586 587 // processEvent processes log events that are ready for submission to CloudWatch 588 // logs. Batching is performed on time- and size-bases. Time-based batching 589 // occurs at a 5 second interval (defined in the batchPublishFrequency const). 590 // Size-based batching is performed on the maximum number of events per batch 591 // (defined in maximumLogEventsPerPut) and the maximum number of total bytes in a 592 // batch (defined in maximumBytesPerPut). Log messages are split by the maximum 593 // bytes per event (defined in maximumBytesPerEvent). There is a fixed per-event 594 // byte overhead (defined in perEventBytes) which is accounted for in split- and 595 // batch-calculations. Because the events are interpreted as UTF-8 encoded 596 // Unicode, invalid UTF-8 byte sequences are replaced with the Unicode 597 // replacement character (U+FFFD), which is a 3-byte sequence in UTF-8. To 598 // compensate for that and to avoid splitting valid UTF-8 characters into 599 // invalid byte sequences, we calculate the length of each event assuming that 600 // this replacement happens. 601 func (l *logStream) processEvent(batch *eventBatch, bytes []byte, timestamp int64) { 602 for len(bytes) > 0 { 603 // Split line length so it does not exceed the maximum 604 splitOffset, lineBytes := findValidSplit(string(bytes), maximumBytesPerEvent) 605 line := bytes[:splitOffset] 606 event := wrappedEvent{ 607 inputLogEvent: &cloudwatchlogs.InputLogEvent{ 608 Message: aws.String(string(line)), 609 Timestamp: aws.Int64(timestamp), 610 }, 611 insertOrder: batch.count(), 612 } 613 614 added := batch.add(event, lineBytes) 615 if added { 616 bytes = bytes[splitOffset:] 617 } else { 618 l.publishBatch(batch) 619 batch.reset() 620 } 621 } 622 } 623 624 // effectiveLen counts the effective number of bytes in the string, after 625 // UTF-8 normalization. UTF-8 normalization includes replacing bytes that do 626 // not constitute valid UTF-8 encoded Unicode codepoints with the Unicode 627 // replacement codepoint U+FFFD (a 3-byte UTF-8 sequence, represented in Go as 628 // utf8.RuneError) 629 func effectiveLen(line string) int { 630 effectiveBytes := 0 631 for _, rune := range line { 632 effectiveBytes += utf8.RuneLen(rune) 633 } 634 return effectiveBytes 635 } 636 637 // findValidSplit finds the byte offset to split a string without breaking valid 638 // Unicode codepoints given a maximum number of total bytes. findValidSplit 639 // returns the byte offset for splitting a string or []byte, as well as the 640 // effective number of bytes if the string were normalized to replace invalid 641 // UTF-8 encoded bytes with the Unicode replacement character (a 3-byte UTF-8 642 // sequence, represented in Go as utf8.RuneError) 643 func findValidSplit(line string, maxBytes int) (splitOffset, effectiveBytes int) { 644 for offset, rune := range line { 645 splitOffset = offset 646 if effectiveBytes+utf8.RuneLen(rune) > maxBytes { 647 return splitOffset, effectiveBytes 648 } 649 effectiveBytes += utf8.RuneLen(rune) 650 } 651 splitOffset = len(line) 652 return 653 } 654 655 // publishBatch calls PutLogEvents for a given set of InputLogEvents, 656 // accounting for sequencing requirements (each request must reference the 657 // sequence token returned by the previous request). 658 func (l *logStream) publishBatch(batch *eventBatch) { 659 if batch.isEmpty() { 660 return 661 } 662 cwEvents := unwrapEvents(batch.events()) 663 664 nextSequenceToken, err := l.putLogEvents(cwEvents, l.sequenceToken) 665 666 if err != nil { 667 if awsErr, ok := err.(awserr.Error); ok { 668 if awsErr.Code() == dataAlreadyAcceptedCode { 669 // already submitted, just grab the correct sequence token 670 parts := strings.Split(awsErr.Message(), " ") 671 nextSequenceToken = &parts[len(parts)-1] 672 logrus.WithFields(logrus.Fields{ 673 "errorCode": awsErr.Code(), 674 "message": awsErr.Message(), 675 "logGroupName": l.logGroupName, 676 "logStreamName": l.logStreamName, 677 }).Info("Data already accepted, ignoring error") 678 err = nil 679 } else if awsErr.Code() == invalidSequenceTokenCode { 680 // sequence code is bad, grab the correct one and retry 681 parts := strings.Split(awsErr.Message(), " ") 682 token := parts[len(parts)-1] 683 nextSequenceToken, err = l.putLogEvents(cwEvents, &token) 684 } 685 } 686 } 687 if err != nil { 688 logrus.Error(err) 689 } else { 690 l.sequenceToken = nextSequenceToken 691 } 692 } 693 694 // putLogEvents wraps the PutLogEvents API 695 func (l *logStream) putLogEvents(events []*cloudwatchlogs.InputLogEvent, sequenceToken *string) (*string, error) { 696 input := &cloudwatchlogs.PutLogEventsInput{ 697 LogEvents: events, 698 SequenceToken: sequenceToken, 699 LogGroupName: aws.String(l.logGroupName), 700 LogStreamName: aws.String(l.logStreamName), 701 } 702 resp, err := l.client.PutLogEvents(input) 703 if err != nil { 704 if awsErr, ok := err.(awserr.Error); ok { 705 logrus.WithFields(logrus.Fields{ 706 "errorCode": awsErr.Code(), 707 "message": awsErr.Message(), 708 "origError": awsErr.OrigErr(), 709 "logGroupName": l.logGroupName, 710 "logStreamName": l.logStreamName, 711 }).Error("Failed to put log events") 712 } 713 return nil, err 714 } 715 return resp.NextSequenceToken, nil 716 } 717 718 // ValidateLogOpt looks for awslogs-specific log options awslogs-region, awslogs-endpoint 719 // awslogs-group, awslogs-stream, awslogs-create-group, awslogs-datetime-format, 720 // awslogs-multiline-pattern 721 func ValidateLogOpt(cfg map[string]string) error { 722 for key := range cfg { 723 switch key { 724 case logGroupKey: 725 case logStreamKey: 726 case logCreateGroupKey: 727 case regionKey: 728 case endpointKey: 729 case tagKey: 730 case datetimeFormatKey: 731 case multilinePatternKey: 732 case credentialsEndpointKey: 733 case forceFlushIntervalKey: 734 case maxBufferedEventsKey: 735 default: 736 return fmt.Errorf("unknown log opt '%s' for %s log driver", key, name) 737 } 738 } 739 if cfg[logGroupKey] == "" { 740 return fmt.Errorf("must specify a value for log opt '%s'", logGroupKey) 741 } 742 if cfg[logCreateGroupKey] != "" { 743 if _, err := strconv.ParseBool(cfg[logCreateGroupKey]); err != nil { 744 return fmt.Errorf("must specify valid value for log opt '%s': %v", logCreateGroupKey, err) 745 } 746 } 747 if cfg[forceFlushIntervalKey] != "" { 748 if value, err := strconv.Atoi(cfg[forceFlushIntervalKey]); err != nil || value <= 0 { 749 return fmt.Errorf("must specify a positive integer for log opt '%s': %v", forceFlushIntervalKey, cfg[forceFlushIntervalKey]) 750 } 751 } 752 if cfg[maxBufferedEventsKey] != "" { 753 if value, err := strconv.Atoi(cfg[maxBufferedEventsKey]); err != nil || value <= 0 { 754 return fmt.Errorf("must specify a positive integer for log opt '%s': %v", maxBufferedEventsKey, cfg[maxBufferedEventsKey]) 755 } 756 } 757 _, datetimeFormatKeyExists := cfg[datetimeFormatKey] 758 _, multilinePatternKeyExists := cfg[multilinePatternKey] 759 if datetimeFormatKeyExists && multilinePatternKeyExists { 760 return fmt.Errorf("you cannot configure log opt '%s' and '%s' at the same time", datetimeFormatKey, multilinePatternKey) 761 } 762 return nil 763 } 764 765 // Len returns the length of a byTimestamp slice. Len is required by the 766 // sort.Interface interface. 767 func (slice byTimestamp) Len() int { 768 return len(slice) 769 } 770 771 // Less compares two values in a byTimestamp slice by Timestamp. Less is 772 // required by the sort.Interface interface. 773 func (slice byTimestamp) Less(i, j int) bool { 774 iTimestamp, jTimestamp := int64(0), int64(0) 775 if slice != nil && slice[i].inputLogEvent.Timestamp != nil { 776 iTimestamp = *slice[i].inputLogEvent.Timestamp 777 } 778 if slice != nil && slice[j].inputLogEvent.Timestamp != nil { 779 jTimestamp = *slice[j].inputLogEvent.Timestamp 780 } 781 if iTimestamp == jTimestamp { 782 return slice[i].insertOrder < slice[j].insertOrder 783 } 784 return iTimestamp < jTimestamp 785 } 786 787 // Swap swaps two values in a byTimestamp slice with each other. Swap is 788 // required by the sort.Interface interface. 789 func (slice byTimestamp) Swap(i, j int) { 790 slice[i], slice[j] = slice[j], slice[i] 791 } 792 793 func unwrapEvents(events []wrappedEvent) []*cloudwatchlogs.InputLogEvent { 794 cwEvents := make([]*cloudwatchlogs.InputLogEvent, len(events)) 795 for i, input := range events { 796 cwEvents[i] = input.inputLogEvent 797 } 798 return cwEvents 799 } 800 801 func newEventBatch() *eventBatch { 802 return &eventBatch{ 803 batch: make([]wrappedEvent, 0), 804 bytes: 0, 805 } 806 } 807 808 // events returns a slice of wrappedEvents sorted in order of their 809 // timestamps and then by their insertion order (see `byTimestamp`). 810 // 811 // Warning: this method is not threadsafe and must not be used 812 // concurrently. 813 func (b *eventBatch) events() []wrappedEvent { 814 sort.Sort(byTimestamp(b.batch)) 815 return b.batch 816 } 817 818 // add adds an event to the batch of events accounting for the 819 // necessary overhead for an event to be logged. An error will be 820 // returned if the event cannot be added to the batch due to service 821 // limits. 822 // 823 // Warning: this method is not threadsafe and must not be used 824 // concurrently. 825 func (b *eventBatch) add(event wrappedEvent, size int) bool { 826 addBytes := size + perEventBytes 827 828 // verify we are still within service limits 829 switch { 830 case len(b.batch)+1 > maximumLogEventsPerPut: 831 return false 832 case b.bytes+addBytes > maximumBytesPerPut: 833 return false 834 } 835 836 b.bytes += addBytes 837 b.batch = append(b.batch, event) 838 839 return true 840 } 841 842 // count is the number of batched events. Warning: this method 843 // is not threadsafe and must not be used concurrently. 844 func (b *eventBatch) count() int { 845 return len(b.batch) 846 } 847 848 // size is the total number of bytes that the batch represents. 849 // 850 // Warning: this method is not threadsafe and must not be used 851 // concurrently. 852 func (b *eventBatch) size() int { 853 return b.bytes 854 } 855 856 func (b *eventBatch) isEmpty() bool { 857 zeroEvents := b.count() == 0 858 zeroSize := b.size() == 0 859 return zeroEvents && zeroSize 860 } 861 862 // reset prepares the batch for reuse. 863 func (b *eventBatch) reset() { 864 b.bytes = 0 865 b.batch = b.batch[:0] 866 }