github.com/fibonacci-chain/fbc@v0.0.0-20231124064014-c7636198c1e9/app/repair_state.go (about)

     1  package app
     2  
     3  import (
     4  	"fmt"
     5  	"io"
     6  	"log"
     7  	"path/filepath"
     8  	"strings"
     9  	"time"
    10  
    11  	"github.com/fibonacci-chain/fbc/app/config"
    12  	"github.com/fibonacci-chain/fbc/app/utils/appstatus"
    13  
    14  	"github.com/fibonacci-chain/fbc/libs/cosmos-sdk/server"
    15  	"github.com/fibonacci-chain/fbc/libs/cosmos-sdk/store/flatkv"
    16  	mpttypes "github.com/fibonacci-chain/fbc/libs/cosmos-sdk/store/mpt"
    17  	"github.com/fibonacci-chain/fbc/libs/cosmos-sdk/store/rootmulti"
    18  	sdk "github.com/fibonacci-chain/fbc/libs/cosmos-sdk/types"
    19  	"github.com/fibonacci-chain/fbc/libs/iavl"
    20  	cfg "github.com/fibonacci-chain/fbc/libs/tendermint/config"
    21  	"github.com/fibonacci-chain/fbc/libs/tendermint/global"
    22  	tmlog "github.com/fibonacci-chain/fbc/libs/tendermint/libs/log"
    23  	"github.com/fibonacci-chain/fbc/libs/tendermint/mock"
    24  	"github.com/fibonacci-chain/fbc/libs/tendermint/node"
    25  	"github.com/fibonacci-chain/fbc/libs/tendermint/proxy"
    26  	sm "github.com/fibonacci-chain/fbc/libs/tendermint/state"
    27  	blockindex "github.com/fibonacci-chain/fbc/libs/tendermint/state/indexer"
    28  	blockindexer "github.com/fibonacci-chain/fbc/libs/tendermint/state/indexer/block/kv"
    29  	bloxkindexnull "github.com/fibonacci-chain/fbc/libs/tendermint/state/indexer/block/null"
    30  	"github.com/fibonacci-chain/fbc/libs/tendermint/state/txindex"
    31  	"github.com/fibonacci-chain/fbc/libs/tendermint/state/txindex/kv"
    32  	"github.com/fibonacci-chain/fbc/libs/tendermint/state/txindex/null"
    33  	"github.com/fibonacci-chain/fbc/libs/tendermint/store"
    34  	"github.com/fibonacci-chain/fbc/libs/tendermint/types"
    35  	dbm "github.com/fibonacci-chain/fbc/libs/tm-db"
    36  	evmtypes "github.com/fibonacci-chain/fbc/x/evm/types"
    37  	"github.com/spf13/viper"
    38  )
    39  
    40  const (
    41  	applicationDB = "application"
    42  	blockStoreDB  = "blockstore"
    43  	stateDB       = "state"
    44  	txIndexDB     = "tx_index"
    45  	blockIndexDb  = "block_index"
    46  
    47  	FlagStartHeight       string = "start-height"
    48  	FlagEnableRepairState string = "enable-repair-state"
    49  )
    50  
    51  type repairApp struct {
    52  	db dbm.DB
    53  	*FBChainApp
    54  }
    55  
    56  func (app *repairApp) getLatestVersion() int64 {
    57  	rs := rootmulti.NewStore(app.db)
    58  	return rs.GetLatestVersion()
    59  }
    60  
    61  func repairStateOnStart(ctx *server.Context) {
    62  	// set flag
    63  	orgIgnoreSmbCheck := sm.IgnoreSmbCheck
    64  	orgIgnoreVersionCheck := iavl.GetIgnoreVersionCheck()
    65  	orgEnableFlatKV := viper.GetBool(flatkv.FlagEnable)
    66  	iavl.EnableAsyncCommit = false
    67  	viper.Set(flatkv.FlagEnable, false)
    68  	iavl.SetEnableFastStorage(appstatus.IsFastStorageStrategy())
    69  	iavl.SetForceReadIavl(true)
    70  
    71  	// repair state
    72  	RepairState(ctx, true)
    73  
    74  	//set original flag
    75  	iavl.SetForceReadIavl(false)
    76  	sm.SetIgnoreSmbCheck(orgIgnoreSmbCheck)
    77  	iavl.SetIgnoreVersionCheck(orgIgnoreVersionCheck)
    78  	iavl.EnableAsyncCommit = viper.GetBool(iavl.FlagIavlEnableAsyncCommit)
    79  	viper.Set(flatkv.FlagEnable, orgEnableFlatKV)
    80  	// load latest block height
    81  }
    82  
    83  func RepairState(ctx *server.Context, onStart bool) {
    84  	sm.SetIgnoreSmbCheck(true)
    85  	iavl.SetIgnoreVersionCheck(true)
    86  
    87  	// load latest block height
    88  	dataDir := filepath.Join(ctx.Config.RootDir, "data")
    89  	latestBlockHeight := latestBlockHeight(dataDir)
    90  	startBlockHeight := types.GetStartBlockHeight()
    91  	if latestBlockHeight <= startBlockHeight+2 {
    92  		log.Println(fmt.Sprintf("There is no need to repair data. The latest block height is %d, start block height is %d", latestBlockHeight, startBlockHeight))
    93  		return
    94  	}
    95  
    96  	config.RegisterDynamicConfig(ctx.Logger.With("module", "config"))
    97  
    98  	// create proxy app
    99  	proxyApp, repairApp, err := createRepairApp(ctx)
   100  	panicError(err)
   101  	defer repairApp.Close()
   102  
   103  	// get async commit version
   104  	commitVersion, err := repairApp.GetCommitVersion()
   105  	log.Println(fmt.Sprintf("repair state latestBlockHeight = %d \t commitVersion = %d", latestBlockHeight, commitVersion))
   106  	panicError(err)
   107  
   108  	if onStart && commitVersion == latestBlockHeight {
   109  		log.Println("no need to repair state on start")
   110  		return
   111  	}
   112  
   113  	// load state
   114  	stateStoreDB, err := sdk.NewDB(stateDB, dataDir)
   115  	panicError(err)
   116  	defer func() {
   117  		err := stateStoreDB.Close()
   118  		panicError(err)
   119  	}()
   120  	genesisDocProvider := node.DefaultGenesisDocProviderFunc(ctx.Config)
   121  	state, _, err := node.LoadStateFromDBOrGenesisDocProvider(stateStoreDB, genesisDocProvider)
   122  	panicError(err)
   123  
   124  	// load start version
   125  	startVersion := viper.GetInt64(FlagStartHeight)
   126  	if startVersion == 0 {
   127  		if onStart {
   128  			startVersion = commitVersion
   129  		} else {
   130  			if types.HigherThanMars(commitVersion) {
   131  				lastMptVersion := int64(repairApp.EvmKeeper.GetLatestStoredBlockHeight())
   132  				if lastMptVersion < commitVersion {
   133  					commitVersion = lastMptVersion
   134  				}
   135  			}
   136  			startVersion = commitVersion - 2 // case: state machine broken
   137  		}
   138  	}
   139  	if startVersion <= 0 {
   140  		panic("height too low, please restart from height 0 with genesis file")
   141  	}
   142  	log.Println(fmt.Sprintf("repair state at version = %d", startVersion))
   143  
   144  	err = repairApp.LoadStartVersion(startVersion)
   145  	panicError(err)
   146  
   147  	rawTrieDirtyDisabledFlag := viper.GetBool(mpttypes.FlagTrieDirtyDisabled)
   148  	mpttypes.TrieDirtyDisabled = true
   149  	repairApp.EvmKeeper.SetTargetMptVersion(startVersion)
   150  
   151  	// repair data by apply the latest two blocks
   152  	doRepair(ctx, state, stateStoreDB, proxyApp, startVersion, latestBlockHeight, dataDir)
   153  
   154  	mpttypes.TrieDirtyDisabled = rawTrieDirtyDisabledFlag
   155  }
   156  func createRepairApp(ctx *server.Context) (proxy.AppConns, *repairApp, error) {
   157  	rootDir := ctx.Config.RootDir
   158  	dataDir := filepath.Join(rootDir, "data")
   159  	db, err := sdk.NewDB(applicationDB, dataDir)
   160  	panicError(err)
   161  	repairApp := newRepairApp(ctx.Logger, db, nil)
   162  
   163  	clientCreator := proxy.NewLocalClientCreator(repairApp)
   164  	// Create the proxyApp and establish connections to the ABCI app (consensus, mempool, query).
   165  	proxyApp, err := createAndStartProxyAppConns(clientCreator)
   166  	return proxyApp, repairApp, err
   167  }
   168  
   169  func newRepairApp(logger tmlog.Logger, db dbm.DB, traceStore io.Writer) *repairApp {
   170  	return &repairApp{db, NewFBChainApp(
   171  		logger,
   172  		db,
   173  		traceStore,
   174  		false,
   175  		map[int64]bool{},
   176  		0,
   177  	)}
   178  }
   179  
   180  func doRepair(ctx *server.Context, state sm.State, stateStoreDB dbm.DB,
   181  	proxyApp proxy.AppConns, startHeight, latestHeight int64, dataDir string) {
   182  	stateCopy := state.Copy()
   183  	ctx.Logger.Debug("stateCopy", "state", fmt.Sprintf("%+v", stateCopy))
   184  	// construct state for repair
   185  	state = constructStartState(state, stateStoreDB, startHeight)
   186  	ctx.Logger.Debug("constructStartState", "state", fmt.Sprintf("%+v", state))
   187  	// repair state
   188  	eventBus := types.NewEventBus()
   189  	txStore, blockIndexStore, txindexServer, err := startEventBusAndIndexerService(ctx.Config, eventBus, ctx.Logger)
   190  	panicError(err)
   191  	blockExec := sm.NewBlockExecutor(stateStoreDB, ctx.Logger, proxyApp.Consensus(), mock.Mempool{}, sm.MockEvidencePool{})
   192  	blockExec.SetEventBus(eventBus)
   193  	// Save state synchronously during repair state
   194  	blockExec.SetIsAsyncSaveDB(false)
   195  	defer func() {
   196  		// stop sequence is important to avoid data missing: blockExecutor->eventBus->txIndexer
   197  		// keep the same sequence as node.go:OnStop
   198  		blockExec.Stop()
   199  
   200  		if eventBus != nil && eventBus.IsRunning() {
   201  			eventBus.Stop()
   202  			eventBus.Wait()
   203  		}
   204  
   205  		if txindexServer != nil && txindexServer.IsRunning() {
   206  			txindexServer.Stop()
   207  			txindexServer.Wait()
   208  		}
   209  
   210  		if txStore != nil {
   211  			err := txStore.Close()
   212  			panicError(err)
   213  		}
   214  		if blockIndexStore != nil {
   215  			err := blockIndexStore.Close()
   216  			panicError(err)
   217  		}
   218  	}()
   219  
   220  	global.SetGlobalHeight(startHeight + 1)
   221  	for height := startHeight + 1; height <= latestHeight; height++ {
   222  		repairBlock, repairBlockMeta := loadBlock(height, dataDir)
   223  		state, _, err = blockExec.ApplyBlockWithTrace(state, repairBlockMeta.BlockID, repairBlock)
   224  		panicError(err)
   225  		ctx.Logger.Debug("repairedState", "state", fmt.Sprintf("%+v", state))
   226  		res, err := proxyApp.Query().InfoSync(proxy.RequestInfo)
   227  		panicError(err)
   228  		repairedBlockHeight := res.LastBlockHeight
   229  		repairedAppHash := res.LastBlockAppHash
   230  		log.Println("Repaired block height", repairedBlockHeight)
   231  		log.Println("Repaired app hash", fmt.Sprintf("%X", repairedAppHash))
   232  	}
   233  }
   234  
   235  func startEventBusAndIndexerService(config *cfg.Config, eventBus *types.EventBus, logger tmlog.Logger) (txStore dbm.DB, blockIndexStore dbm.DB, indexerService *txindex.IndexerService, err error) {
   236  	eventBus.SetLogger(logger.With("module", "events"))
   237  	if err := eventBus.Start(); err != nil {
   238  		return nil, nil, nil, err
   239  	}
   240  	// Transaction indexing
   241  	var txIndexer txindex.TxIndexer
   242  	var blockIndexer blockindex.BlockIndexer
   243  	switch config.TxIndex.Indexer {
   244  	case "kv":
   245  		txStore, err = sdk.NewDB(txIndexDB, filepath.Join(config.RootDir, "data"))
   246  		if err != nil {
   247  			return nil, nil, nil, err
   248  		}
   249  		blockIndexStore, err := sdk.NewDB(blockIndexDb, filepath.Join(config.RootDir, "data"))
   250  		if err != nil {
   251  			return nil, nil, nil, err
   252  		}
   253  		switch {
   254  		case config.TxIndex.IndexKeys != "":
   255  			txIndexer = kv.NewTxIndex(txStore, kv.IndexEvents(splitAndTrimEmpty(config.TxIndex.IndexKeys, ",", " ")))
   256  		case config.TxIndex.IndexAllKeys:
   257  			txIndexer = kv.NewTxIndex(txStore, kv.IndexAllEvents())
   258  		default:
   259  			txIndexer = kv.NewTxIndex(txStore)
   260  		}
   261  		blockIndexer = blockindexer.New(dbm.NewPrefixDB(blockIndexStore, []byte("block_events")))
   262  	default:
   263  		txIndexer = &null.TxIndex{}
   264  		blockIndexer = &bloxkindexnull.BlockerIndexer{}
   265  	}
   266  
   267  	indexerService = txindex.NewIndexerService(txIndexer, blockIndexer, eventBus)
   268  	indexerService.SetLogger(logger.With("module", "txindex"))
   269  	if err := indexerService.Start(); err != nil {
   270  		if eventBus != nil {
   271  			eventBus.Stop()
   272  		}
   273  		if txStore != nil {
   274  			txStore.Close()
   275  		}
   276  
   277  		return nil, nil, nil, err
   278  	}
   279  	return txStore, blockIndexStore, indexerService, nil
   280  }
   281  
   282  // splitAndTrimEmpty slices s into all subslices separated by sep and returns a
   283  // slice of the string s with all leading and trailing Unicode code points
   284  // contained in cutset removed. If sep is empty, SplitAndTrim splits after each
   285  // UTF-8 sequence. First part is equivalent to strings.SplitN with a count of
   286  // -1.  also filter out empty strings, only return non-empty strings.
   287  func splitAndTrimEmpty(s, sep, cutset string) []string {
   288  	if s == "" {
   289  		return []string{}
   290  	}
   291  
   292  	spl := strings.Split(s, sep)
   293  	nonEmptyStrings := make([]string, 0, len(spl))
   294  	for i := 0; i < len(spl); i++ {
   295  		element := strings.Trim(spl[i], cutset)
   296  		if element != "" {
   297  			nonEmptyStrings = append(nonEmptyStrings, element)
   298  		}
   299  	}
   300  	return nonEmptyStrings
   301  }
   302  
   303  func constructStartState(state sm.State, stateStoreDB dbm.DB, startHeight int64) sm.State {
   304  	stateCopy := state.Copy()
   305  	validators, lastStoredHeight, err := sm.LoadValidatorsWithStoredHeight(stateStoreDB, startHeight+1)
   306  	lastValidators, err := sm.LoadValidators(stateStoreDB, startHeight)
   307  	if err != nil {
   308  		return stateCopy
   309  	}
   310  	nextValidators, err := sm.LoadValidators(stateStoreDB, startHeight+2)
   311  	if err != nil {
   312  		return stateCopy
   313  	}
   314  	consensusParams, err := sm.LoadConsensusParams(stateStoreDB, startHeight+1)
   315  	if err != nil {
   316  		return stateCopy
   317  	}
   318  	stateCopy.Validators = validators
   319  	stateCopy.LastValidators = lastValidators
   320  	stateCopy.NextValidators = nextValidators
   321  	stateCopy.ConsensusParams = consensusParams
   322  	stateCopy.LastBlockHeight = startHeight
   323  	stateCopy.LastHeightValidatorsChanged = lastStoredHeight
   324  	return stateCopy
   325  }
   326  
   327  func loadBlock(height int64, dataDir string) (*types.Block, *types.BlockMeta) {
   328  	storeDB, err := sdk.NewDB(blockStoreDB, dataDir)
   329  	defer storeDB.Close()
   330  	blockStore := store.NewBlockStore(storeDB)
   331  	panicError(err)
   332  	block := blockStore.LoadBlock(height)
   333  	meta := blockStore.LoadBlockMeta(height)
   334  	return block, meta
   335  }
   336  
   337  func latestBlockHeight(dataDir string) int64 {
   338  	storeDB, err := sdk.NewDB(blockStoreDB, dataDir)
   339  	panicError(err)
   340  	defer storeDB.Close()
   341  	blockStore := store.NewBlockStore(storeDB)
   342  	return blockStore.Height()
   343  }
   344  
   345  // panic if error is not nil
   346  func panicError(err error) {
   347  	if err != nil {
   348  		panic(err)
   349  	}
   350  }
   351  
   352  func createAndStartProxyAppConns(clientCreator proxy.ClientCreator) (proxy.AppConns, error) {
   353  	proxyApp := proxy.NewAppConns(clientCreator)
   354  	if err := proxyApp.Start(); err != nil {
   355  		return nil, fmt.Errorf("error starting proxy app connections: %v", err)
   356  	}
   357  	return proxyApp, nil
   358  }
   359  
   360  func (app *repairApp) Close() {
   361  	indexer := evmtypes.GetIndexer()
   362  	if indexer != nil {
   363  		for indexer.IsProcessing() {
   364  			time.Sleep(100 * time.Millisecond)
   365  		}
   366  	}
   367  	evmtypes.CloseIndexer()
   368  	err := app.db.Close()
   369  	panicError(err)
   370  }