github.com/fibonacci-chain/fbc@v0.0.0-20231124064014-c7636198c1e9/app/repair_state.go (about) 1 package app 2 3 import ( 4 "fmt" 5 "io" 6 "log" 7 "path/filepath" 8 "strings" 9 "time" 10 11 "github.com/fibonacci-chain/fbc/app/config" 12 "github.com/fibonacci-chain/fbc/app/utils/appstatus" 13 14 "github.com/fibonacci-chain/fbc/libs/cosmos-sdk/server" 15 "github.com/fibonacci-chain/fbc/libs/cosmos-sdk/store/flatkv" 16 mpttypes "github.com/fibonacci-chain/fbc/libs/cosmos-sdk/store/mpt" 17 "github.com/fibonacci-chain/fbc/libs/cosmos-sdk/store/rootmulti" 18 sdk "github.com/fibonacci-chain/fbc/libs/cosmos-sdk/types" 19 "github.com/fibonacci-chain/fbc/libs/iavl" 20 cfg "github.com/fibonacci-chain/fbc/libs/tendermint/config" 21 "github.com/fibonacci-chain/fbc/libs/tendermint/global" 22 tmlog "github.com/fibonacci-chain/fbc/libs/tendermint/libs/log" 23 "github.com/fibonacci-chain/fbc/libs/tendermint/mock" 24 "github.com/fibonacci-chain/fbc/libs/tendermint/node" 25 "github.com/fibonacci-chain/fbc/libs/tendermint/proxy" 26 sm "github.com/fibonacci-chain/fbc/libs/tendermint/state" 27 blockindex "github.com/fibonacci-chain/fbc/libs/tendermint/state/indexer" 28 blockindexer "github.com/fibonacci-chain/fbc/libs/tendermint/state/indexer/block/kv" 29 bloxkindexnull "github.com/fibonacci-chain/fbc/libs/tendermint/state/indexer/block/null" 30 "github.com/fibonacci-chain/fbc/libs/tendermint/state/txindex" 31 "github.com/fibonacci-chain/fbc/libs/tendermint/state/txindex/kv" 32 "github.com/fibonacci-chain/fbc/libs/tendermint/state/txindex/null" 33 "github.com/fibonacci-chain/fbc/libs/tendermint/store" 34 "github.com/fibonacci-chain/fbc/libs/tendermint/types" 35 dbm "github.com/fibonacci-chain/fbc/libs/tm-db" 36 evmtypes "github.com/fibonacci-chain/fbc/x/evm/types" 37 "github.com/spf13/viper" 38 ) 39 40 const ( 41 applicationDB = "application" 42 blockStoreDB = "blockstore" 43 stateDB = "state" 44 txIndexDB = "tx_index" 45 blockIndexDb = "block_index" 46 47 FlagStartHeight string = "start-height" 48 FlagEnableRepairState string = "enable-repair-state" 49 ) 50 51 type repairApp struct { 52 db dbm.DB 53 *FBChainApp 54 } 55 56 func (app *repairApp) getLatestVersion() int64 { 57 rs := rootmulti.NewStore(app.db) 58 return rs.GetLatestVersion() 59 } 60 61 func repairStateOnStart(ctx *server.Context) { 62 // set flag 63 orgIgnoreSmbCheck := sm.IgnoreSmbCheck 64 orgIgnoreVersionCheck := iavl.GetIgnoreVersionCheck() 65 orgEnableFlatKV := viper.GetBool(flatkv.FlagEnable) 66 iavl.EnableAsyncCommit = false 67 viper.Set(flatkv.FlagEnable, false) 68 iavl.SetEnableFastStorage(appstatus.IsFastStorageStrategy()) 69 iavl.SetForceReadIavl(true) 70 71 // repair state 72 RepairState(ctx, true) 73 74 //set original flag 75 iavl.SetForceReadIavl(false) 76 sm.SetIgnoreSmbCheck(orgIgnoreSmbCheck) 77 iavl.SetIgnoreVersionCheck(orgIgnoreVersionCheck) 78 iavl.EnableAsyncCommit = viper.GetBool(iavl.FlagIavlEnableAsyncCommit) 79 viper.Set(flatkv.FlagEnable, orgEnableFlatKV) 80 // load latest block height 81 } 82 83 func RepairState(ctx *server.Context, onStart bool) { 84 sm.SetIgnoreSmbCheck(true) 85 iavl.SetIgnoreVersionCheck(true) 86 87 // load latest block height 88 dataDir := filepath.Join(ctx.Config.RootDir, "data") 89 latestBlockHeight := latestBlockHeight(dataDir) 90 startBlockHeight := types.GetStartBlockHeight() 91 if latestBlockHeight <= startBlockHeight+2 { 92 log.Println(fmt.Sprintf("There is no need to repair data. The latest block height is %d, start block height is %d", latestBlockHeight, startBlockHeight)) 93 return 94 } 95 96 config.RegisterDynamicConfig(ctx.Logger.With("module", "config")) 97 98 // create proxy app 99 proxyApp, repairApp, err := createRepairApp(ctx) 100 panicError(err) 101 defer repairApp.Close() 102 103 // get async commit version 104 commitVersion, err := repairApp.GetCommitVersion() 105 log.Println(fmt.Sprintf("repair state latestBlockHeight = %d \t commitVersion = %d", latestBlockHeight, commitVersion)) 106 panicError(err) 107 108 if onStart && commitVersion == latestBlockHeight { 109 log.Println("no need to repair state on start") 110 return 111 } 112 113 // load state 114 stateStoreDB, err := sdk.NewDB(stateDB, dataDir) 115 panicError(err) 116 defer func() { 117 err := stateStoreDB.Close() 118 panicError(err) 119 }() 120 genesisDocProvider := node.DefaultGenesisDocProviderFunc(ctx.Config) 121 state, _, err := node.LoadStateFromDBOrGenesisDocProvider(stateStoreDB, genesisDocProvider) 122 panicError(err) 123 124 // load start version 125 startVersion := viper.GetInt64(FlagStartHeight) 126 if startVersion == 0 { 127 if onStart { 128 startVersion = commitVersion 129 } else { 130 if types.HigherThanMars(commitVersion) { 131 lastMptVersion := int64(repairApp.EvmKeeper.GetLatestStoredBlockHeight()) 132 if lastMptVersion < commitVersion { 133 commitVersion = lastMptVersion 134 } 135 } 136 startVersion = commitVersion - 2 // case: state machine broken 137 } 138 } 139 if startVersion <= 0 { 140 panic("height too low, please restart from height 0 with genesis file") 141 } 142 log.Println(fmt.Sprintf("repair state at version = %d", startVersion)) 143 144 err = repairApp.LoadStartVersion(startVersion) 145 panicError(err) 146 147 rawTrieDirtyDisabledFlag := viper.GetBool(mpttypes.FlagTrieDirtyDisabled) 148 mpttypes.TrieDirtyDisabled = true 149 repairApp.EvmKeeper.SetTargetMptVersion(startVersion) 150 151 // repair data by apply the latest two blocks 152 doRepair(ctx, state, stateStoreDB, proxyApp, startVersion, latestBlockHeight, dataDir) 153 154 mpttypes.TrieDirtyDisabled = rawTrieDirtyDisabledFlag 155 } 156 func createRepairApp(ctx *server.Context) (proxy.AppConns, *repairApp, error) { 157 rootDir := ctx.Config.RootDir 158 dataDir := filepath.Join(rootDir, "data") 159 db, err := sdk.NewDB(applicationDB, dataDir) 160 panicError(err) 161 repairApp := newRepairApp(ctx.Logger, db, nil) 162 163 clientCreator := proxy.NewLocalClientCreator(repairApp) 164 // Create the proxyApp and establish connections to the ABCI app (consensus, mempool, query). 165 proxyApp, err := createAndStartProxyAppConns(clientCreator) 166 return proxyApp, repairApp, err 167 } 168 169 func newRepairApp(logger tmlog.Logger, db dbm.DB, traceStore io.Writer) *repairApp { 170 return &repairApp{db, NewFBChainApp( 171 logger, 172 db, 173 traceStore, 174 false, 175 map[int64]bool{}, 176 0, 177 )} 178 } 179 180 func doRepair(ctx *server.Context, state sm.State, stateStoreDB dbm.DB, 181 proxyApp proxy.AppConns, startHeight, latestHeight int64, dataDir string) { 182 stateCopy := state.Copy() 183 ctx.Logger.Debug("stateCopy", "state", fmt.Sprintf("%+v", stateCopy)) 184 // construct state for repair 185 state = constructStartState(state, stateStoreDB, startHeight) 186 ctx.Logger.Debug("constructStartState", "state", fmt.Sprintf("%+v", state)) 187 // repair state 188 eventBus := types.NewEventBus() 189 txStore, blockIndexStore, txindexServer, err := startEventBusAndIndexerService(ctx.Config, eventBus, ctx.Logger) 190 panicError(err) 191 blockExec := sm.NewBlockExecutor(stateStoreDB, ctx.Logger, proxyApp.Consensus(), mock.Mempool{}, sm.MockEvidencePool{}) 192 blockExec.SetEventBus(eventBus) 193 // Save state synchronously during repair state 194 blockExec.SetIsAsyncSaveDB(false) 195 defer func() { 196 // stop sequence is important to avoid data missing: blockExecutor->eventBus->txIndexer 197 // keep the same sequence as node.go:OnStop 198 blockExec.Stop() 199 200 if eventBus != nil && eventBus.IsRunning() { 201 eventBus.Stop() 202 eventBus.Wait() 203 } 204 205 if txindexServer != nil && txindexServer.IsRunning() { 206 txindexServer.Stop() 207 txindexServer.Wait() 208 } 209 210 if txStore != nil { 211 err := txStore.Close() 212 panicError(err) 213 } 214 if blockIndexStore != nil { 215 err := blockIndexStore.Close() 216 panicError(err) 217 } 218 }() 219 220 global.SetGlobalHeight(startHeight + 1) 221 for height := startHeight + 1; height <= latestHeight; height++ { 222 repairBlock, repairBlockMeta := loadBlock(height, dataDir) 223 state, _, err = blockExec.ApplyBlockWithTrace(state, repairBlockMeta.BlockID, repairBlock) 224 panicError(err) 225 ctx.Logger.Debug("repairedState", "state", fmt.Sprintf("%+v", state)) 226 res, err := proxyApp.Query().InfoSync(proxy.RequestInfo) 227 panicError(err) 228 repairedBlockHeight := res.LastBlockHeight 229 repairedAppHash := res.LastBlockAppHash 230 log.Println("Repaired block height", repairedBlockHeight) 231 log.Println("Repaired app hash", fmt.Sprintf("%X", repairedAppHash)) 232 } 233 } 234 235 func startEventBusAndIndexerService(config *cfg.Config, eventBus *types.EventBus, logger tmlog.Logger) (txStore dbm.DB, blockIndexStore dbm.DB, indexerService *txindex.IndexerService, err error) { 236 eventBus.SetLogger(logger.With("module", "events")) 237 if err := eventBus.Start(); err != nil { 238 return nil, nil, nil, err 239 } 240 // Transaction indexing 241 var txIndexer txindex.TxIndexer 242 var blockIndexer blockindex.BlockIndexer 243 switch config.TxIndex.Indexer { 244 case "kv": 245 txStore, err = sdk.NewDB(txIndexDB, filepath.Join(config.RootDir, "data")) 246 if err != nil { 247 return nil, nil, nil, err 248 } 249 blockIndexStore, err := sdk.NewDB(blockIndexDb, filepath.Join(config.RootDir, "data")) 250 if err != nil { 251 return nil, nil, nil, err 252 } 253 switch { 254 case config.TxIndex.IndexKeys != "": 255 txIndexer = kv.NewTxIndex(txStore, kv.IndexEvents(splitAndTrimEmpty(config.TxIndex.IndexKeys, ",", " "))) 256 case config.TxIndex.IndexAllKeys: 257 txIndexer = kv.NewTxIndex(txStore, kv.IndexAllEvents()) 258 default: 259 txIndexer = kv.NewTxIndex(txStore) 260 } 261 blockIndexer = blockindexer.New(dbm.NewPrefixDB(blockIndexStore, []byte("block_events"))) 262 default: 263 txIndexer = &null.TxIndex{} 264 blockIndexer = &bloxkindexnull.BlockerIndexer{} 265 } 266 267 indexerService = txindex.NewIndexerService(txIndexer, blockIndexer, eventBus) 268 indexerService.SetLogger(logger.With("module", "txindex")) 269 if err := indexerService.Start(); err != nil { 270 if eventBus != nil { 271 eventBus.Stop() 272 } 273 if txStore != nil { 274 txStore.Close() 275 } 276 277 return nil, nil, nil, err 278 } 279 return txStore, blockIndexStore, indexerService, nil 280 } 281 282 // splitAndTrimEmpty slices s into all subslices separated by sep and returns a 283 // slice of the string s with all leading and trailing Unicode code points 284 // contained in cutset removed. If sep is empty, SplitAndTrim splits after each 285 // UTF-8 sequence. First part is equivalent to strings.SplitN with a count of 286 // -1. also filter out empty strings, only return non-empty strings. 287 func splitAndTrimEmpty(s, sep, cutset string) []string { 288 if s == "" { 289 return []string{} 290 } 291 292 spl := strings.Split(s, sep) 293 nonEmptyStrings := make([]string, 0, len(spl)) 294 for i := 0; i < len(spl); i++ { 295 element := strings.Trim(spl[i], cutset) 296 if element != "" { 297 nonEmptyStrings = append(nonEmptyStrings, element) 298 } 299 } 300 return nonEmptyStrings 301 } 302 303 func constructStartState(state sm.State, stateStoreDB dbm.DB, startHeight int64) sm.State { 304 stateCopy := state.Copy() 305 validators, lastStoredHeight, err := sm.LoadValidatorsWithStoredHeight(stateStoreDB, startHeight+1) 306 lastValidators, err := sm.LoadValidators(stateStoreDB, startHeight) 307 if err != nil { 308 return stateCopy 309 } 310 nextValidators, err := sm.LoadValidators(stateStoreDB, startHeight+2) 311 if err != nil { 312 return stateCopy 313 } 314 consensusParams, err := sm.LoadConsensusParams(stateStoreDB, startHeight+1) 315 if err != nil { 316 return stateCopy 317 } 318 stateCopy.Validators = validators 319 stateCopy.LastValidators = lastValidators 320 stateCopy.NextValidators = nextValidators 321 stateCopy.ConsensusParams = consensusParams 322 stateCopy.LastBlockHeight = startHeight 323 stateCopy.LastHeightValidatorsChanged = lastStoredHeight 324 return stateCopy 325 } 326 327 func loadBlock(height int64, dataDir string) (*types.Block, *types.BlockMeta) { 328 storeDB, err := sdk.NewDB(blockStoreDB, dataDir) 329 defer storeDB.Close() 330 blockStore := store.NewBlockStore(storeDB) 331 panicError(err) 332 block := blockStore.LoadBlock(height) 333 meta := blockStore.LoadBlockMeta(height) 334 return block, meta 335 } 336 337 func latestBlockHeight(dataDir string) int64 { 338 storeDB, err := sdk.NewDB(blockStoreDB, dataDir) 339 panicError(err) 340 defer storeDB.Close() 341 blockStore := store.NewBlockStore(storeDB) 342 return blockStore.Height() 343 } 344 345 // panic if error is not nil 346 func panicError(err error) { 347 if err != nil { 348 panic(err) 349 } 350 } 351 352 func createAndStartProxyAppConns(clientCreator proxy.ClientCreator) (proxy.AppConns, error) { 353 proxyApp := proxy.NewAppConns(clientCreator) 354 if err := proxyApp.Start(); err != nil { 355 return nil, fmt.Errorf("error starting proxy app connections: %v", err) 356 } 357 return proxyApp, nil 358 } 359 360 func (app *repairApp) Close() { 361 indexer := evmtypes.GetIndexer() 362 if indexer != nil { 363 for indexer.IsProcessing() { 364 time.Sleep(100 * time.Millisecond) 365 } 366 } 367 evmtypes.CloseIndexer() 368 err := app.db.Close() 369 panicError(err) 370 }