github.com/TrueBlocks/trueblocks-core/src/apps/chifra@v0.0.0-20241022031540-b362680128f7/cmd/scrape.go (about)

     1  // Copyright 2016, 2024 The TrueBlocks Authors. All rights reserved.
     2  // Use of this source code is governed by a license that can
     3  // be found in the LICENSE file.
     4  /*
     5   * Parts of this file were auto generated. Edit only those parts of
     6   * the code inside of 'EXISTING_CODE' tags.
     7   */
     8  
     9  package cmd
    10  
    11  // EXISTING_CODE
    12  import (
    13  	"os"
    14  
    15  	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/internal/globals"
    16  	scrapePkg "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/internal/scrape"
    17  	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/caps"
    18  	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/file"
    19  	outputHelpers "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/output/helpers"
    20  	"github.com/spf13/cobra"
    21  )
    22  
    23  // EXISTING_CODE
    24  
    25  // scrapeCmd represents the scrape command
    26  var scrapeCmd = &cobra.Command{
    27  	Use:     usageScrape,
    28  	Long:    longScrape,
    29  	Version: versionText,
    30  	PreRun: outputHelpers.PreRunWithJsonWriter("scrape", func() *globals.GlobalOptions {
    31  		return &scrapePkg.GetOptions().Globals
    32  	}),
    33  	RunE: file.RunWithFileSupport("scrape", scrapePkg.RunScrape, scrapePkg.ResetOptions),
    34  	PostRun: outputHelpers.PostRunWithJsonWriter(func() *globals.GlobalOptions {
    35  		return &scrapePkg.GetOptions().Globals
    36  	}),
    37  }
    38  
    39  const usageScrape = `scrape [flags]`
    40  
    41  const longScrape = `Purpose:
    42    Scan the chain and update the TrueBlocks index of appearances.`
    43  
    44  const notesScrape = `
    45  Notes:
    46    - The --touch option may only be used for blocks after the latest scraped block (if any). It will be snapped back to the latest snap_to block.
    47    - This command requires your RPC to provide trace data. See the README for more information.
    48    - The --notify option requires proper configuration. Additionally, IPFS must be running locally. See the README.md file.`
    49  
    50  func init() {
    51  	var capabilities caps.Capability // capabilities for chifra scrape
    52  	capabilities = capabilities.Add(caps.Verbose)
    53  	capabilities = capabilities.Add(caps.Version)
    54  	capabilities = capabilities.Add(caps.Noop)
    55  	capabilities = capabilities.Add(caps.NoColor)
    56  	capabilities = capabilities.Add(caps.Chain)
    57  
    58  	scrapeCmd.Flags().SortFlags = false
    59  
    60  	scrapeCmd.Flags().Uint64VarP(&scrapePkg.GetOptions().BlockCnt, "block_cnt", "n", 2000, `maximum number of blocks to process per pass`)
    61  	scrapeCmd.Flags().Float64VarP(&scrapePkg.GetOptions().Sleep, "sleep", "s", 14, `seconds to sleep between scraper passes`)
    62  	scrapeCmd.Flags().StringVarP(&scrapePkg.GetOptions().Publisher, "publisher", "P", "", `for some query options, the publisher of the index (hidden)`)
    63  	scrapeCmd.Flags().Uint64VarP((*uint64)(&scrapePkg.GetOptions().Touch), "touch", "l", 0, `first block to visit when scraping (snapped back to most recent snap_to_grid mark)`)
    64  	scrapeCmd.Flags().Uint64VarP(&scrapePkg.GetOptions().RunCount, "run_count", "u", 0, `run the scraper this many times, then quit`)
    65  	scrapeCmd.Flags().BoolVarP(&scrapePkg.GetOptions().DryRun, "dry_run", "d", false, `show the configuration that would be applied if run,no changes are made`)
    66  	scrapeCmd.Flags().BoolVarP(&scrapePkg.GetOptions().Notify, "notify", "o", false, `enable the notify feature`)
    67  	scrapeCmd.Flags().Uint64VarP(&scrapePkg.GetOptions().Settings.AppsPerChunk, "apps_per_chunk", "", 2000000, `the number of appearances to build into a chunk before consolidating it (hidden)`)
    68  	scrapeCmd.Flags().Uint64VarP(&scrapePkg.GetOptions().Settings.SnapToGrid, "snap_to_grid", "", 250000, `an override to apps_per_chunk to snap-to-grid at every modulo of this value, this allows easier corrections to the index (hidden)`)
    69  	scrapeCmd.Flags().Uint64VarP(&scrapePkg.GetOptions().Settings.FirstSnap, "first_snap", "", 2000000, `the first block at which snap_to_grid is enabled (hidden)`)
    70  	scrapeCmd.Flags().Uint64VarP(&scrapePkg.GetOptions().Settings.UnripeDist, "unripe_dist", "", 28, `the distance (in blocks) from the front of the chain under which (inclusive) a block is considered unripe (hidden)`)
    71  	scrapeCmd.Flags().Uint64VarP(&scrapePkg.GetOptions().Settings.ChannelCount, "channel_count", "", 20, `number of concurrent processing channels (hidden)`)
    72  	scrapeCmd.Flags().BoolVarP(&scrapePkg.GetOptions().Settings.AllowMissing, "allow_missing", "", false, `do not report errors for blockchains that contain blocks with zero addresses (hidden)`)
    73  	if os.Getenv("TEST_MODE") != "true" {
    74  		_ = scrapeCmd.Flags().MarkHidden("publisher")
    75  		_ = scrapeCmd.Flags().MarkHidden("apps_per_chunk")
    76  		_ = scrapeCmd.Flags().MarkHidden("snap_to_grid")
    77  		_ = scrapeCmd.Flags().MarkHidden("first_snap")
    78  		_ = scrapeCmd.Flags().MarkHidden("unripe_dist")
    79  		_ = scrapeCmd.Flags().MarkHidden("channel_count")
    80  		_ = scrapeCmd.Flags().MarkHidden("allow_missing")
    81  	}
    82  	globals.InitGlobals("scrape", scrapeCmd, &scrapePkg.GetOptions().Globals, capabilities)
    83  
    84  	scrapeCmd.SetUsageTemplate(UsageWithNotes(notesScrape))
    85  	scrapeCmd.SetOut(os.Stderr)
    86  
    87  	// EXISTING_CODE
    88  	// EXISTING_CODE
    89  
    90  	chifraCmd.AddCommand(scrapeCmd)
    91  }