go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/logdog/client/butler/bundler/parser.go (about)

     1  // Copyright 2015 The LUCI Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package bundler
    16  
    17  import (
    18  	"fmt"
    19  	"time"
    20  
    21  	"google.golang.org/protobuf/types/known/durationpb"
    22  
    23  	"go.chromium.org/luci/common/data/chunkstream"
    24  	"go.chromium.org/luci/logdog/api/logpb"
    25  	"go.chromium.org/luci/logdog/common/types"
    26  )
    27  
    28  // constraints is the set of Constraints to apply when generating a LogEntry.
    29  type constraints struct {
    30  	// limit is the maximum size, in bytes, of the serialized LogEntry protobuf
    31  	// that may be produced.
    32  	limit int
    33  
    34  	// allowSplit indicates that bundles should be generated to fill as much of
    35  	// the specified space as possible, splitting them across multiple bundles if
    36  	// necessary.
    37  	//
    38  	// The parser may choose to forego bundling if the result is very suboptimal,
    39  	// but is encouraged to fill the space if it's reasonable.
    40  	allowSplit bool
    41  
    42  	// closed means that bundles should be aggressively generated with the
    43  	// expectation that no further data will be buffered. It is only relevant
    44  	// if allowSplit is also true.
    45  	closed bool
    46  }
    47  
    48  // parser is a stateful presence bound to a single log stream. A parser yields
    49  // LogEntry messages one at a time and shapes them based on constraints.
    50  //
    51  // parser instances are owned by a single Stream and are not goroutine-safe.
    52  type parser interface {
    53  	// appendData adds a data chunk to this parser's chunk.Buffer, taking
    54  	// ownership of the Data.
    55  	appendData(Data)
    56  
    57  	// nextEntry returns the next LogEntry in the stream.
    58  	//
    59  	// This method may return nil if there is insuffuicient data to produce a
    60  	// LogEntry given the
    61  	nextEntry(*constraints) (*logpb.LogEntry, error)
    62  
    63  	bufferedBytes() int64
    64  
    65  	firstChunkTime() (time.Time, bool)
    66  }
    67  
    68  func newParser(d *logpb.LogStreamDescriptor, c *counter) (parser, error) {
    69  	base := baseParser{
    70  		counter:  c,
    71  		timeBase: d.Timestamp.AsTime(),
    72  	}
    73  
    74  	switch d.StreamType {
    75  	case logpb.StreamType_TEXT:
    76  		return &textParser{
    77  			baseParser: base,
    78  		}, nil
    79  
    80  	case logpb.StreamType_BINARY:
    81  		return &binaryParser{
    82  			baseParser: base,
    83  		}, nil
    84  
    85  	case logpb.StreamType_DATAGRAM:
    86  		return &datagramParser{
    87  			baseParser: base,
    88  			maxSize:    int64(types.MaxDatagramSize),
    89  		}, nil
    90  
    91  	default:
    92  		return nil, fmt.Errorf("unknown stream type: %v", d.StreamType)
    93  	}
    94  }
    95  
    96  // baseParser is a common set of parser capabilities.
    97  type baseParser struct {
    98  	chunkstream.Buffer
    99  
   100  	counter *counter
   101  
   102  	timeBase  time.Time
   103  	nextIndex uint64
   104  }
   105  
   106  func (p *baseParser) baseLogEntry(ts time.Time) *logpb.LogEntry {
   107  	e := logpb.LogEntry{
   108  		TimeOffset:  durationpb.New(ts.Sub(p.timeBase)),
   109  		PrefixIndex: uint64(p.counter.next()),
   110  		StreamIndex: p.nextIndex,
   111  	}
   112  	p.nextIndex++
   113  	return &e
   114  }
   115  
   116  func (p *baseParser) appendData(d Data) {
   117  	p.Append(d)
   118  }
   119  
   120  func (p *baseParser) bufferedBytes() int64 {
   121  	return p.Len()
   122  }
   123  
   124  func (p *baseParser) firstChunkTime() (time.Time, bool) {
   125  	// Get the first data chunk in our Buffer.
   126  	chunk := p.FirstChunk()
   127  	if chunk == nil {
   128  		return time.Time{}, false
   129  	}
   130  
   131  	return chunk.(Data).Timestamp(), true
   132  }
   133  
   134  func memoryCorruptionIf(cond bool, err error) {
   135  	if cond {
   136  		memoryCorruption(err)
   137  	}
   138  }
   139  
   140  func memoryCorruption(err error) {
   141  	if err != nil {
   142  		panic(fmt.Errorf("bundler: memory corruption: %s", err))
   143  	}
   144  }