github.com/TrueBlocks/trueblocks-core/src/apps/chifra@v0.0.0-20241022031540-b362680128f7/internal/scrape/scrape_manager.go (about)

     1  package scrapePkg
     2  
     3  import (
     4  	"path/filepath"
     5  
     6  	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/base"
     7  	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/config"
     8  	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/tslib"
     9  	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/types"
    10  )
    11  
    12  // BlazeManager manages the scraper by keeping track of the progress of the scrape and
    13  // maintaining the timestamp array and processed map. The processed map helps us know
    14  // if every block was visited or not.
    15  type BlazeManager struct {
    16  	chain        string
    17  	timestamps   map[base.Blknum]tslib.TimestampRecord
    18  	processedMap map[base.Blknum]bool
    19  	opts         *ScrapeOptions
    20  	meta         *types.MetaData
    21  	startBlock   base.Blknum
    22  	blockCount   base.Blknum
    23  	ripeBlock    base.Blknum
    24  	nRipe        int
    25  	nUnripe      int
    26  	nTimestamps  int
    27  	nChannels    int
    28  	errors       []scrapeError
    29  	isHeadless   bool
    30  }
    31  
    32  type scrapeError struct {
    33  	block base.Blknum
    34  	err   error
    35  }
    36  
    37  // StartBlock returns the start block for the current pass of the scraper.
    38  func (bm *BlazeManager) StartBlock() base.Blknum {
    39  	return bm.startBlock
    40  }
    41  
    42  // BlockCount returns the number of blocks to process for this pass of the scraper.
    43  func (bm *BlazeManager) BlockCount() base.Blknum {
    44  	return bm.blockCount
    45  }
    46  
    47  // EndBlock returns the last block to process for this pass of the scraper.
    48  func (bm *BlazeManager) EndBlock() base.Blknum {
    49  	return bm.startBlock + bm.blockCount
    50  }
    51  
    52  // nProcessed returns the number of blocks processed so far (i.e., ripe + unripe).
    53  func (bm *BlazeManager) nProcessed() int {
    54  	return bm.nRipe + bm.nUnripe
    55  }
    56  
    57  // IsTestMode returns true if the scraper is running in test mode.
    58  func (bm *BlazeManager) IsTestMode() bool {
    59  	return bm.opts.Globals.TestMode
    60  }
    61  
    62  // AllowMissing returns true for all chains but mainnet and the value
    63  // of the config item on mainnet (false by default). The scraper will
    64  // halt if AllowMissing is false and a block with zero appearances is
    65  // encountered.
    66  func (bm *BlazeManager) AllowMissing() bool {
    67  	if bm.chain != "mainnet" {
    68  		return true
    69  	}
    70  	return config.GetScrape(bm.chain).AllowMissing
    71  }
    72  
    73  // PerChunk returns the number of blocks to process per chunk.
    74  func (bm *BlazeManager) PerChunk() uint64 {
    75  	return config.GetScrape(bm.chain).AppsPerChunk
    76  }
    77  
    78  // FirstSnap returns the first block to process.
    79  func (bm *BlazeManager) FirstSnap() base.Blknum {
    80  	return base.Blknum(config.GetScrape(bm.chain).FirstSnap)
    81  }
    82  
    83  // SnapTo returns the number of blocks to process per chunk.
    84  func (bm *BlazeManager) SnapTo() base.Blknum {
    85  	return base.Blknum(config.GetScrape(bm.chain).SnapToGrid)
    86  }
    87  
    88  // IsSnap returns true if the block is a snap point.
    89  func (bm *BlazeManager) IsSnap(block base.Blknum) bool {
    90  	return block >= bm.FirstSnap() && (block%bm.SnapTo()) == 0
    91  }
    92  
    93  // StageFolder returns the folder where the stage file is stored.
    94  func (bm *BlazeManager) StageFolder() string {
    95  	return filepath.Join(config.PathToIndex(bm.chain), "staging")
    96  }
    97  
    98  // RipeFolder returns the folder where the stage file is stored.
    99  func (bm *BlazeManager) RipeFolder() string {
   100  	return filepath.Join(config.PathToIndex(bm.chain), "ripe")
   101  }
   102  
   103  // UnripeFolder returns the folder where the stage file is stored.
   104  func (bm *BlazeManager) UnripeFolder() string {
   105  	return filepath.Join(config.PathToIndex(bm.chain), "unripe")
   106  }