go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/logdog/common/archive/index.go (about)

     1  // Copyright 2015 The LUCI Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package archive
    16  
    17  import (
    18  	"io"
    19  
    20  	"github.com/golang/protobuf/proto"
    21  	"go.chromium.org/luci/logdog/api/logpb"
    22  )
    23  
    24  // indexBuilder is a stateful engine that constructs an archival index.
    25  type indexBuilder struct {
    26  	*Manifest
    27  	index logpb.LogIndex
    28  
    29  	lastPrefixIndex uint64
    30  	lastStreamIndex uint64
    31  	lastBytes       uint64
    32  
    33  	latestBufferedEntry *logpb.LogIndex_Entry
    34  
    35  	sizeFunc func(proto.Message) int
    36  }
    37  
    38  func (i *indexBuilder) addLogEntry(le *logpb.LogEntry, offset int64) {
    39  	// Only calculate the size if we actually use it.
    40  	if i.ByteRange > 0 {
    41  		i.lastBytes += uint64(i.size(le))
    42  	}
    43  
    44  	// Update our stream properties.
    45  	i.index.LastPrefixIndex = le.PrefixIndex
    46  	i.index.LastStreamIndex = le.StreamIndex
    47  	i.index.LogEntryCount++
    48  
    49  	entry := logpb.LogIndex_Entry{
    50  		Sequence:    le.Sequence,
    51  		PrefixIndex: le.PrefixIndex,
    52  		StreamIndex: le.StreamIndex,
    53  		Offset:      uint64(offset),
    54  		TimeOffset:  le.TimeOffset,
    55  	}
    56  
    57  	// Do we index this LogEntry?
    58  	if len(i.index.Entries) > 0 {
    59  		if !((i.StreamIndexRange > 0 && (le.StreamIndex-i.lastStreamIndex) >= uint64(i.StreamIndexRange)) ||
    60  			(i.PrefixIndexRange > 0 && (le.PrefixIndex-i.lastPrefixIndex) >= uint64(i.PrefixIndexRange)) ||
    61  			(i.ByteRange > 0 && i.lastBytes >= uint64(i.ByteRange))) {
    62  			// Not going to index this entry. Buffer it as a terminator.
    63  			i.latestBufferedEntry = &entry
    64  			return
    65  		}
    66  
    67  		i.lastBytes = 0
    68  	}
    69  
    70  	i.index.Entries = append(i.index.Entries, &entry)
    71  	i.latestBufferedEntry = nil
    72  
    73  	// Update our counters.
    74  	i.lastStreamIndex = le.StreamIndex
    75  	i.lastPrefixIndex = le.PrefixIndex
    76  }
    77  
    78  func (i *indexBuilder) emit(w io.Writer) error {
    79  	// Always include the last stream entry in the index.
    80  	if i.latestBufferedEntry != nil {
    81  		i.index.Entries = append(i.index.Entries, i.latestBufferedEntry)
    82  	}
    83  
    84  	d, err := proto.Marshal(&i.index)
    85  	if err != nil {
    86  		return err
    87  	}
    88  
    89  	if _, err := w.Write(d); err != nil {
    90  		return err
    91  	}
    92  	return nil
    93  }
    94  
    95  func (i *indexBuilder) size(pb proto.Message) int {
    96  	if f := i.sizeFunc; f != nil {
    97  		return f(pb)
    98  	}
    99  	return proto.Size(pb)
   100  }