github.com/TrueBlocks/trueblocks-core/src/apps/chifra@v0.0.0-20241022031540-b362680128f7/internal/status/handle_modes.go (about)

     1  package statusPkg
     2  
     3  import (
     4  	"context"
     5  	"errors"
     6  	"fmt"
     7  	"strings"
     8  	"time"
     9  
    10  	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/base"
    11  	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/colors"
    12  	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/file"
    13  	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/index"
    14  	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/logger"
    15  	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/output"
    16  	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/tslib"
    17  	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/types"
    18  	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/utils"
    19  	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/walk"
    20  )
    21  
    22  func (opts *StatusOptions) HandleModes(rCtx *output.RenderCtx) error {
    23  	chain := opts.Globals.Chain
    24  	testMode := opts.Globals.TestMode
    25  
    26  	fetchData := func(modelChan chan types.Modeler, errorChan chan error) {
    27  		now := time.Now()
    28  
    29  		filenameChan := make(chan walk.CacheFileInfo)
    30  		var nRoutines int
    31  
    32  		counterMap := make(map[walk.CacheType]*types.CacheItem)
    33  		nRoutines = len(opts.ModeTypes)
    34  		for _, mT := range opts.ModeTypes {
    35  			counterMap[mT] = &types.CacheItem{
    36  				CacheItemType: walk.WalkCacheName(mT),
    37  				Items:         make([]any, 0),
    38  				LastCached:    now.Format("2006-01-02 15:04:05"),
    39  			}
    40  			var t CacheWalker
    41  			t.ctx, t.cancel = context.WithCancel(context.Background())
    42  			go walk.WalkCacheFolder(t.ctx, chain, mT, &t, filenameChan)
    43  		}
    44  
    45  		for result := range filenameChan {
    46  			cT := result.Type
    47  
    48  			switch cT {
    49  			case walk.Cache_NotACache:
    50  				nRoutines--
    51  				if nRoutines == 0 {
    52  					close(filenameChan)
    53  					logger.Progress(true, "                                           ")
    54  				}
    55  			default:
    56  				isIndex := func(cT walk.CacheType) bool {
    57  					m := map[walk.CacheType]bool{
    58  						walk.Index_Bloom:   true,
    59  						walk.Index_Final:   true,
    60  						walk.Index_Ripe:    true,
    61  						walk.Index_Staging: true,
    62  						walk.Index_Unripe:  true,
    63  						walk.Index_Maps:    true,
    64  					}
    65  					return m[cT]
    66  				}
    67  				if testMode && isIndex(cT) && (cT != walk.Index_Bloom && cT != walk.Index_Final) {
    68  					continue
    69  				} else if testMode && cT == walk.Cache_Results {
    70  					continue
    71  				}
    72  
    73  				if walk.IsCacheType(result.Path, cT, !result.IsDir /* checkExt */) {
    74  					if result.IsDir {
    75  						counterMap[cT].NFolders++
    76  						counterMap[cT].Path = walk.GetRootPathFromCacheType(chain, cT)
    77  					} else {
    78  						result.Data.(*CacheWalker).nSeen++
    79  						if result.Data.(*CacheWalker).nSeen >= opts.FirstRecord {
    80  							counterMap[cT].NFiles++
    81  							counterMap[cT].SizeInBytes += file.FileSize(result.Path)
    82  							if opts.Globals.Verbose && counterMap[cT].NFiles <= opts.MaxRecords {
    83  								result.FileRange = base.RangeFromFilename(result.Path)
    84  								result.TsRange.First, _ = tslib.FromBnToTs(chain, result.FileRange.First)
    85  								result.TsRange.Last, _ = tslib.FromBnToTs(chain, result.FileRange.Last)
    86  								cI, _ := walk.GetCacheItem(chain, testMode, cT, &result)
    87  								if isIndex(cT) {
    88  									bP := index.ToBloomPath(result.Path)
    89  									cI["bloomSizeBytes"] = file.FileSize(bP)
    90  									iP := index.ToIndexPath(result.Path)
    91  									cI["indexSizeBytes"] = file.FileSize(iP)
    92  								}
    93  								counterMap[cT].Items = append(counterMap[cT].Items, cI)
    94  							}
    95  						}
    96  					}
    97  
    98  					if counterMap[cT].NFiles >= opts.MaxRecords {
    99  						result.Data.(*CacheWalker).cancel()
   100  					}
   101  
   102  					smallMark := result.Data.(*CacheWalker).nSeen%100 == 0
   103  					logger.Progress(
   104  						smallMark && !utils.IsFuzzing(),
   105  						fmt.Sprintf("Found %d %s files", counterMap[cT].NFiles, cT))
   106  
   107  					if (result.Data.(*CacheWalker).nSeen+1)%100000 == 0 {
   108  						logger.Info(colors.Green, "Progress:", colors.Off, "Found", counterMap[cT].NFiles, "files and", counterMap[cT].NFolders, "folders after", result.Data.(*CacheWalker).nSeen+1, "files")
   109  					}
   110  
   111  				} else {
   112  					logger.Progress(!utils.IsFuzzing(), fmt.Sprintf("Skipped %s", result.Path))
   113  				}
   114  			}
   115  		}
   116  
   117  		status, err := opts.GetStatus(false)
   118  		if err != nil {
   119  			errorChan <- err
   120  			return
   121  		}
   122  
   123  		totalRecords := uint64(0)
   124  		for _, mT := range opts.ModeTypes {
   125  			if counterMap[mT] != nil {
   126  				status.Caches = append(status.Caches, *counterMap[mT])
   127  				totalRecords += counterMap[mT].NFiles
   128  			}
   129  		}
   130  
   131  		if totalRecords == 0 {
   132  			str := ""
   133  			for _, m := range opts.Modes {
   134  				str += m + " "
   135  			}
   136  			errorChan <- errors.New("no files were found in the [" + strings.Trim(str, " ") + "] caches")
   137  			return
   138  		}
   139  
   140  		modelChan <- status
   141  	}
   142  
   143  	extraOpts := map[string]any{
   144  		"chains": opts.Chains,
   145  	}
   146  
   147  	return output.StreamMany(rCtx, fetchData, opts.Globals.OutputOptsWithExtra(extraOpts))
   148  }
   149  
   150  type CacheWalker struct {
   151  	ctx    context.Context
   152  	cancel context.CancelFunc
   153  	nSeen  uint64
   154  }