go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/logdog/server/archivist/storageSource.go (about)

     1  // Copyright 2016 The LUCI Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package archivist
    16  
    17  import (
    18  	"context"
    19  	"io"
    20  
    21  	log "go.chromium.org/luci/common/logging"
    22  	"go.chromium.org/luci/logdog/api/logpb"
    23  	"go.chromium.org/luci/logdog/common/storage"
    24  	"go.chromium.org/luci/logdog/common/types"
    25  )
    26  
    27  // storageSource is a renderer.Source that pulls log entries from intermediate
    28  // storage via its storage.Storage instance.
    29  type storageSource struct {
    30  	context.Context
    31  
    32  	st            storage.Storage    // the storage instance to read from
    33  	project       string             // the project of the log stream
    34  	path          types.StreamPath   // the path of the log stream
    35  	terminalIndex types.MessageIndex // if >= 0, discard logs beyond this
    36  
    37  	buf           []*logpb.LogEntry
    38  	lastIndex     types.MessageIndex
    39  	logEntryCount int64
    40  }
    41  
    42  func (s *storageSource) bufferEntries(start types.MessageIndex) error {
    43  	bytes := 0
    44  
    45  	req := storage.GetRequest{
    46  		Project: s.project,
    47  		Path:    s.path,
    48  		Index:   start,
    49  	}
    50  	return s.st.Get(s, req, func(e *storage.Entry) bool {
    51  		le, err := e.GetLogEntry()
    52  		if err != nil {
    53  			log.WithError(err).Errorf(s, "Failed to unmarshal LogEntry.")
    54  			return false
    55  		}
    56  		s.buf = append(s.buf, le)
    57  
    58  		// Stop loading if we've reached or exceeded our buffer size.
    59  		bytes += len(e.D)
    60  		return bytes < storageBufferSize
    61  	})
    62  }
    63  
    64  func (s *storageSource) NextLogEntry() (*logpb.LogEntry, error) {
    65  	if len(s.buf) == 0 {
    66  		s.buf = s.buf[:0]
    67  		if err := s.bufferEntries(s.lastIndex + 1); err != nil {
    68  			if err == storage.ErrDoesNotExist {
    69  				log.Warningf(s, "Archive target stream does not exist in intermediate storage.")
    70  				return nil, io.EOF
    71  			}
    72  
    73  			log.WithError(err).Errorf(s, "Failed to retrieve log stream from storage.")
    74  			return nil, err
    75  		}
    76  	}
    77  
    78  	// If we have no more buffered entries, we have exhausted our log stream.
    79  	if len(s.buf) == 0 {
    80  		// If we have a terminal index, but we didn't actually emit that index,
    81  		// mark that we have missing entries.
    82  		if s.terminalIndex >= 0 && s.lastIndex != s.terminalIndex {
    83  			log.Fields{
    84  				"terminalIndex": s.terminalIndex,
    85  				"lastIndex":     s.lastIndex,
    86  			}.Warningf(s, "Log stream stopped before terminal index.")
    87  		} else {
    88  			// Encountered end of stream.
    89  		}
    90  
    91  		return nil, io.EOF
    92  	}
    93  
    94  	// Pop the next log entry and advance the stream.
    95  	var le *logpb.LogEntry
    96  	le, s.buf = s.buf[0], s.buf[1:]
    97  
    98  	// If we're enforcing a maximum terminal index, return end of stream if this
    99  	// LogEntry exceeds that index.
   100  	sidx := types.MessageIndex(le.StreamIndex)
   101  	if s.terminalIndex >= 0 && sidx > s.terminalIndex {
   102  		log.Fields{
   103  			"index":         sidx,
   104  			"terminalIndex": s.terminalIndex,
   105  		}.Warningf(s, "Discarding log entries beyond expected terminal index.")
   106  		return nil, io.EOF
   107  	}
   108  
   109  	s.lastIndex = sidx
   110  	s.logEntryCount++
   111  	return le, nil
   112  }