go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/logdog/common/archive/entry_buffer.go (about)

     1  // Copyright 2021 The LUCI Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package archive
    16  
    17  import (
    18  	"fmt"
    19  	"strings"
    20  	"time"
    21  
    22  	cl "cloud.google.com/go/logging"
    23  	"google.golang.org/protobuf/types/known/durationpb"
    24  
    25  	"go.chromium.org/luci/logdog/api/logpb"
    26  )
    27  
    28  // entryBuffer buffers the lines from a series of LogEntry(s) and constructs logging.Entry(s)
    29  // with them.
    30  type entryBuffer struct {
    31  	buf        strings.Builder
    32  	seq        int
    33  	maxPayload int
    34  
    35  	streamStart     time.Time
    36  	streamID        string
    37  	lastTimeOffset  *durationpb.Duration // TimeOffset of the latest LogEntry passed to Append.
    38  	hasCompleteLine bool
    39  }
    40  
    41  // newEntryBuffer constructs and returns entryBuffer.
    42  func newEntryBuffer(maxPayload int, streamID string, desc *logpb.LogStreamDescriptor) *entryBuffer {
    43  	return &entryBuffer{
    44  		maxPayload:  maxPayload,
    45  		streamStart: desc.Timestamp.AsTime(),
    46  		streamID:    streamID,
    47  	}
    48  }
    49  
    50  // append appends a new LogEntry into the buffer and returns CloudLogging entries ready for sending.
    51  func (eb *entryBuffer) append(e *logpb.LogEntry) []*cl.Entry {
    52  	var ret []*cl.Entry
    53  	flushIf := func(cond bool) {
    54  		if !cond {
    55  			return
    56  		}
    57  		if e := eb.flush(); e != nil {
    58  			ret = append(ret, e)
    59  		}
    60  	}
    61  	eb.lastTimeOffset = e.TimeOffset
    62  	lines := e.GetText().GetLines()
    63  	for i, l := range lines {
    64  		lastLineAndIncomplete := i == len(lines)-1 && l.Delimiter == ""
    65  		flushIf(eb.hasCompleteLine && (eb.isExceeding(l) || lastLineAndIncomplete))
    66  		eb.safeAdd(l)
    67  	}
    68  	flushIf(eb.hasCompleteLine)
    69  	return ret
    70  }
    71  
    72  func (eb *entryBuffer) safeAdd(line *logpb.Text_Line) {
    73  	remaining := eb.maxPayload - eb.buf.Len()
    74  	n, _ := eb.buf.Write(line.Value[:min(remaining, len(line.Value))])
    75  	if d := line.Delimiter; d != "" {
    76  		remaining -= n
    77  		eb.buf.WriteString(d[:min(remaining, len(d))])
    78  		eb.hasCompleteLine = true
    79  	}
    80  }
    81  
    82  // isExceeding returns true if a given line with the buffered lines is going to exceed maxPayload.
    83  // False, otherwise.
    84  func (eb *entryBuffer) isExceeding(l *logpb.Text_Line) bool {
    85  	return l != nil && eb.buf.Len()+len(l.Value)+len(l.Delimiter) > eb.maxPayload
    86  }
    87  
    88  // flush flushes the buffer and return a logging.Entry with the content.
    89  // If the buffer contains no content, returns nil.
    90  func (eb *entryBuffer) flush() *cl.Entry {
    91  	if eb.buf.Len() == 0 {
    92  		return nil
    93  	}
    94  
    95  	ts := eb.streamStart
    96  	if eb.lastTimeOffset != nil {
    97  		ts = eb.streamStart.Add(eb.lastTimeOffset.AsDuration())
    98  	}
    99  
   100  	if eb.hasCompleteLine {
   101  		eb.hasCompleteLine = false
   102  	}
   103  	// Trim line end delimiters because it Cloud Logging UI renders it as an extra
   104  	// empty line at the end of the log message.
   105  	payload := strings.TrimRight(eb.buf.String(), "\n\r")
   106  	if payload == "" {
   107  		return nil
   108  	}
   109  
   110  	ret := &cl.Entry{
   111  		Payload:   payload,
   112  		Timestamp: ts,
   113  
   114  		// InsertID uniquely identifies each LogEntry to dedup entries in CloudLogging.
   115  		InsertID: fmt.Sprintf("%s/%d", eb.streamID, eb.seq),
   116  
   117  		// Set the Trace with the streamID so that Log entries from the same Log stream
   118  		// can be grouped together in CloudLogging UI.
   119  		Trace: eb.streamID,
   120  	}
   121  	eb.buf.Reset()
   122  	eb.seq += 1
   123  	return ret
   124  }
   125  
   126  func min(i, j int) int {
   127  	if i < j {
   128  		return i
   129  	}
   130  	return j
   131  }