github.com/TrueBlocks/trueblocks-core/src/apps/chifra@v0.0.0-20241022031540-b362680128f7/internal/scrape/save_timestamps.go (about)

     1  package scrapePkg
     2  
     3  import (
     4  	"context"
     5  	"encoding/binary"
     6  	"fmt"
     7  	"os"
     8  
     9  	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/base"
    10  	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/config"
    11  	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/logger"
    12  	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/tslib"
    13  )
    14  
    15  // TODO: Protect against overwriting files on disc
    16  
    17  func (bm *BlazeManager) WriteTimestamps(ctx context.Context, blocks []base.Blknum) error {
    18  	chain := bm.chain
    19  
    20  	// At all times, the timestamp file is complete (that is, there are no missing pieces
    21  	// and the last record is at block nTimestamps. We can append as we go (which is fast).
    22  	tsPath := config.PathToTimestamps(chain)
    23  	fp, err := os.OpenFile(tsPath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
    24  	if err != nil {
    25  		return err
    26  	}
    27  	defer func() {
    28  		tslib.ClearCache(chain)
    29  		fp.Close()
    30  	}()
    31  
    32  	nTimestamps, err := tslib.NTimestamps(chain)
    33  	if err != nil {
    34  		return err
    35  	}
    36  
    37  	if blocks[len(blocks)-1] < nTimestamps {
    38  		// We already have all of these timestamps, leave early
    39  		return nil
    40  
    41  	} else if blocks[0] > nTimestamps {
    42  		// we need to catch up (for example, the user truncated the timestamps file while debugging)
    43  		// don't get more than maxBlocks at a time
    44  		cnt := 0
    45  		maxBlocks := 1000
    46  		for block := nTimestamps; block < blocks[0] && cnt < maxBlocks; block++ {
    47  			if ctx.Err() != nil {
    48  				// This means the context got cancelled, i.e. we got a SIGINT.
    49  				return nil
    50  			}
    51  			ts := tslib.TimestampRecord{
    52  				Bn: uint32(block),
    53  				Ts: uint32(bm.opts.Conn.GetBlockTimestamp(block)),
    54  			}
    55  			msg := fmt.Sprintf("Backfilling timestamps (%d-%d) at ", cnt, maxBlocks)
    56  			logProgressTs(msg, block, blocks[len(blocks)-1])
    57  			if err := binary.Write(fp, binary.LittleEndian, &ts); err != nil {
    58  				return err
    59  			}
    60  			cnt++
    61  		}
    62  		// we must return early here, otherwise there will be skipped records
    63  		return nil
    64  	}
    65  
    66  	if ctx.Err() != nil {
    67  		// This means the context got cancelled, i.e. we got a SIGINT.
    68  		return nil
    69  	}
    70  
    71  	// Append to the timestamps file all the new timestamps but as we do that make sure we're
    72  	// not skipping anything at the front, in the middle, or at the end of the list
    73  	for _, block := range blocks {
    74  		if block < nTimestamps {
    75  			// We already have this timestampe, skip out
    76  			continue
    77  		}
    78  
    79  		if base.Blknum(bm.timestamps[block].Bn) != block {
    80  			return fmt.Errorf("timestamp missing at block %d", block)
    81  		}
    82  
    83  		ts := bm.timestamps[block]
    84  		if !bm.isHeadless {
    85  			logProgressTs("Updating timestamps ", block, blocks[len(blocks)-1])
    86  		}
    87  		if err := binary.Write(fp, binary.LittleEndian, &ts); err != nil {
    88  			return err
    89  		}
    90  
    91  	}
    92  
    93  	if !bm.isHeadless {
    94  		logger.Progress(true, fmt.Sprintf("Finished writing timestamps to block %-04d"+spaces,
    95  			blocks[len(blocks)-1],
    96  		))
    97  	}
    98  
    99  	return nil
   100  }
   101  
   102  func logProgressTs(msgIn string, cur, total base.Blknum) {
   103  	frequency := base.Blknum(13)
   104  	left := total - cur
   105  	msg := fmt.Sprintf("%s%-04d of %-04d (%-04d remaining)"+spaces,
   106  		msgIn,
   107  		cur,
   108  		total,
   109  		left,
   110  	)
   111  	logger.Progress((cur%frequency) == 0, msg)
   112  }