github.com/TrueBlocks/trueblocks-core/src/apps/chifra@v0.0.0-20241022031540-b362680128f7/internal/scrape/scrape_batch.go (about)

     1  package scrapePkg
     2  
     3  // Copyright 2021 The TrueBlocks Authors. All rights reserved.
     4  // Use of this source code is governed by a license that can
     5  // be found in the LICENSE file.
     6  
     7  import (
     8  	"context"
     9  	"errors"
    10  	"fmt"
    11  
    12  	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/base"
    13  	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/logger"
    14  )
    15  
    16  // ScrapeBatch is called each time around the forever loop. It calls into
    17  // HandleBlaze and writes the timestamps if there's no error.
    18  func (bm *BlazeManager) ScrapeBatch(ctx context.Context, blocks []base.Blknum) error {
    19  	chain := bm.chain
    20  
    21  	_ = bm.HandleBlaze(ctx, blocks)
    22  	if len(bm.errors) > 0 {
    23  		for _, err := range bm.errors {
    24  			logger.Error(fmt.Sprintf("error at block %d: %v", err.block, err.err))
    25  		}
    26  		_ = cleanEphemeralIndexFolders(chain)
    27  		return errors.New("encountered errors while scraping")
    28  	}
    29  	if ctx.Err() != nil {
    30  		return nil
    31  	}
    32  
    33  	// Check to see if we missed any blocks...
    34  	for _, block := range blocks {
    35  		if !bm.processedMap[block] {
    36  			for _, err := range bm.errors {
    37  				logger.Error(fmt.Sprintf("error at block %d: %v", err.block, err.err))
    38  			}
    39  			// We missed a block. We need to clean up and continue
    40  			// next time around the loop. This may happen if the
    41  			// node returns an error for example.
    42  			_ = cleanEphemeralIndexFolders(chain)
    43  			return fmt.Errorf("a block (%d) was not processed", block)
    44  		}
    45  	}
    46  
    47  	// defensive programming...
    48  	if len(blocks) != len(bm.processedMap) ||
    49  		len(blocks) != bm.nProcessed() ||
    50  		bm.nTimestamps != bm.nProcessed() {
    51  		for _, err := range bm.errors {
    52  			logger.Error(fmt.Sprintf("error at block %d: %v", err.block, err.err))
    53  		}
    54  		_ = cleanEphemeralIndexFolders(chain)
    55  		return fmt.Errorf(`check failed len(blocks): %d len(map): %d nRipe: %d nUnripe: %d nProcessed: %d nTs: %d`,
    56  			len(blocks),
    57  			len(bm.processedMap),
    58  			bm.nRipe,
    59  			bm.nUnripe,
    60  			bm.nProcessed(),
    61  			bm.nTimestamps,
    62  		)
    63  	}
    64  
    65  	if ctx.Err() != nil {
    66  		// This means the context got cancelled, i.e. we got a SIGINT.
    67  		return nil
    68  	}
    69  
    70  	return bm.WriteTimestamps(ctx, blocks)
    71  }