github.com/status-im/status-go@v1.1.0/services/wallet/transfer/commands_sequential.go (about) 1 package transfer 2 3 import ( 4 "context" 5 "math/big" 6 "sync/atomic" 7 "time" 8 9 "github.com/ethereum/go-ethereum/accounts/abi/bind" 10 "github.com/ethereum/go-ethereum/common" 11 "github.com/ethereum/go-ethereum/core/types" 12 "github.com/ethereum/go-ethereum/event" 13 "github.com/ethereum/go-ethereum/log" 14 "github.com/status-im/status-go/contracts" 15 nodetypes "github.com/status-im/status-go/eth-node/types" 16 "github.com/status-im/status-go/multiaccounts/accounts" 17 "github.com/status-im/status-go/rpc/chain" 18 "github.com/status-im/status-go/services/wallet/async" 19 "github.com/status-im/status-go/services/wallet/balance" 20 "github.com/status-im/status-go/services/wallet/blockchainstate" 21 "github.com/status-im/status-go/services/wallet/token" 22 "github.com/status-im/status-go/services/wallet/walletevent" 23 "github.com/status-im/status-go/transactions" 24 ) 25 26 var findBlocksRetryInterval = 5 * time.Second 27 28 const ( 29 transferHistoryTag = "transfer_history" 30 newTransferHistoryTag = "new_transfer_history" 31 32 transferHistoryLimit = 10000 33 transferHistoryLimitPerAccount = 5000 34 transferHistoryLimitPeriod = 24 * time.Hour 35 ) 36 37 type nonceInfo struct { 38 nonce *int64 39 blockNumber *big.Int 40 } 41 42 type findNewBlocksCommand struct { 43 *findBlocksCommand 44 contractMaker *contracts.ContractMaker 45 iteration int 46 blockChainState *blockchainstate.BlockChainState 47 lastNonces map[common.Address]nonceInfo 48 nonceCheckIntervalIterations int 49 logsCheckIntervalIterations int 50 } 51 52 func (c *findNewBlocksCommand) Command() async.Command { 53 return async.InfiniteCommand{ 54 Interval: 2 * time.Minute, 55 Runable: c.Run, 56 }.Run 57 } 58 59 var requestTimeout = 20 * time.Second 60 61 func (c *findNewBlocksCommand) detectTransfers(parent context.Context, accounts []common.Address) (*big.Int, []common.Address, error) { 62 bc, err := c.contractMaker.NewBalanceChecker(c.chainClient.NetworkID()) 63 if err != nil { 64 log.Error("findNewBlocksCommand error creating balance checker", "error", err, "chain", c.chainClient.NetworkID()) 65 return nil, nil, err 66 } 67 68 tokens, err := c.tokenManager.GetTokens(c.chainClient.NetworkID()) 69 if err != nil { 70 return nil, nil, err 71 } 72 tokenAddresses := []common.Address{} 73 nilAddress := common.Address{} 74 for _, token := range tokens { 75 if token.Address != nilAddress { 76 tokenAddresses = append(tokenAddresses, token.Address) 77 } 78 } 79 log.Debug("findNewBlocksCommand detectTransfers", "cnt", len(tokenAddresses)) 80 81 ctx, cancel := context.WithTimeout(parent, requestTimeout) 82 defer cancel() 83 blockNum, hashes, err := bc.BalancesHash(&bind.CallOpts{Context: ctx}, c.accounts, tokenAddresses) 84 if err != nil { 85 log.Error("findNewBlocksCommand can't get balances hashes", "error", err) 86 return nil, nil, err 87 } 88 89 addressesToCheck := []common.Address{} 90 for idx, account := range accounts { 91 blockRange, _, err := c.blockRangeDAO.getBlockRange(c.chainClient.NetworkID(), account) 92 if err != nil { 93 log.Error("findNewBlocksCommand can't get block range", "error", err, "account", account, "chain", c.chainClient.NetworkID()) 94 return nil, nil, err 95 } 96 97 checkHash := common.BytesToHash(hashes[idx][:]) 98 log.Debug("findNewBlocksCommand comparing hashes", "account", account, "network", c.chainClient.NetworkID(), "old hash", blockRange.balanceCheckHash, "new hash", checkHash.String()) 99 if checkHash.String() != blockRange.balanceCheckHash { 100 addressesToCheck = append(addressesToCheck, account) 101 } 102 103 blockRange.balanceCheckHash = checkHash.String() 104 105 err = c.blockRangeDAO.upsertRange(c.chainClient.NetworkID(), account, blockRange) 106 if err != nil { 107 log.Error("findNewBlocksCommand can't update balance check", "error", err, "account", account, "chain", c.chainClient.NetworkID()) 108 return nil, nil, err 109 } 110 } 111 112 return blockNum, addressesToCheck, nil 113 } 114 115 func (c *findNewBlocksCommand) detectNonceChange(parent context.Context, to *big.Int, accounts []common.Address) (map[common.Address]*big.Int, error) { 116 addressesWithChange := map[common.Address]*big.Int{} 117 for _, account := range accounts { 118 var oldNonce *int64 119 120 blockRange, _, err := c.blockRangeDAO.getBlockRange(c.chainClient.NetworkID(), account) 121 if err != nil { 122 log.Error("findNewBlocksCommand can't get block range", "error", err, "account", account, "chain", c.chainClient.NetworkID()) 123 return nil, err 124 } 125 126 lastNonceInfo, ok := c.lastNonces[account] 127 if !ok || lastNonceInfo.blockNumber.Cmp(blockRange.eth.LastKnown) != 0 { 128 log.Debug("Fetching old nonce", "at", blockRange.eth.LastKnown, "acc", account) 129 if blockRange.eth.LastKnown == nil { 130 blockRange.eth.LastKnown = big.NewInt(0) 131 oldNonce = new(int64) // At 0 block nonce is 0 132 } else { 133 oldNonce, err = c.balanceCacher.NonceAt(parent, c.chainClient, account, blockRange.eth.LastKnown) 134 if err != nil { 135 log.Error("findNewBlocksCommand can't get nonce", "error", err, "account", account, "chain", c.chainClient.NetworkID()) 136 return nil, err 137 } 138 } 139 } else { 140 oldNonce = lastNonceInfo.nonce 141 } 142 143 newNonce, err := c.balanceCacher.NonceAt(parent, c.chainClient, account, to) 144 if err != nil { 145 log.Error("findNewBlocksCommand can't get nonce", "error", err, "account", account, "chain", c.chainClient.NetworkID()) 146 return nil, err 147 } 148 149 log.Debug("Comparing nonces", "oldNonce", *oldNonce, "newNonce", *newNonce, "to", to, "acc", account) 150 151 if *newNonce != *oldNonce { 152 addressesWithChange[account] = blockRange.eth.LastKnown 153 } 154 155 if c.lastNonces == nil { 156 c.lastNonces = map[common.Address]nonceInfo{} 157 } 158 159 c.lastNonces[account] = nonceInfo{ 160 nonce: newNonce, 161 blockNumber: to, 162 } 163 } 164 165 return addressesWithChange, nil 166 } 167 168 var nonceCheckIntervalIterations = 30 169 var logsCheckIntervalIterations = 5 170 171 func (c *findNewBlocksCommand) Run(parent context.Context) error { 172 mnemonicWasNotShown, err := c.accountsDB.GetMnemonicWasNotShown() 173 if err != nil { 174 return err 175 } 176 177 accountsToCheck := []common.Address{} 178 // accounts which might have outgoing transfers initiated outside 179 // the application, e.g. watch only or restored from mnemonic phrase 180 accountsWithOutsideTransfers := []common.Address{} 181 182 for _, account := range c.accounts { 183 acc, err := c.accountsDB.GetAccountByAddress(nodetypes.Address(account)) 184 if err != nil { 185 return err 186 } 187 if mnemonicWasNotShown { 188 if acc.AddressWasNotShown { 189 log.Info("skip findNewBlocksCommand, mnemonic has not been shown and the address has not been shared yet", "address", account) 190 continue 191 } 192 } 193 if !mnemonicWasNotShown || acc.Type != accounts.AccountTypeGenerated { 194 accountsWithOutsideTransfers = append(accountsWithOutsideTransfers, account) 195 } 196 197 accountsToCheck = append(accountsToCheck, account) 198 } 199 200 if len(accountsToCheck) == 0 { 201 return nil 202 } 203 204 headNum, accountsWithDetectedChanges, err := c.detectTransfers(parent, accountsToCheck) 205 if err != nil { 206 log.Error("findNewBlocksCommand error on transfer detection", "error", err, "chain", c.chainClient.NetworkID()) 207 return err 208 } 209 210 c.blockChainState.SetLastBlockNumber(c.chainClient.NetworkID(), headNum.Uint64()) 211 212 if len(accountsWithDetectedChanges) != 0 { 213 log.Debug("findNewBlocksCommand detected accounts with changes, proceeding", "accounts", accountsWithDetectedChanges, "from", c.fromBlockNumber) 214 err = c.findAndSaveEthBlocks(parent, c.fromBlockNumber, headNum, accountsToCheck) 215 if err != nil { 216 return err 217 } 218 } else if c.iteration%c.nonceCheckIntervalIterations == 0 && len(accountsWithOutsideTransfers) > 0 { 219 log.Debug("findNewBlocksCommand nonce check", "accounts", accountsWithOutsideTransfers) 220 accountsWithNonceChanges, err := c.detectNonceChange(parent, headNum, accountsWithOutsideTransfers) 221 if err != nil { 222 return err 223 } 224 225 if len(accountsWithNonceChanges) > 0 { 226 log.Debug("findNewBlocksCommand detected nonce diff", "accounts", accountsWithNonceChanges) 227 for account, from := range accountsWithNonceChanges { 228 err = c.findAndSaveEthBlocks(parent, from, headNum, []common.Address{account}) 229 if err != nil { 230 return err 231 } 232 } 233 } 234 235 for _, account := range accountsToCheck { 236 if _, ok := accountsWithNonceChanges[account]; ok { 237 continue 238 } 239 err := c.markEthBlockRangeChecked(account, &BlockRange{nil, c.fromBlockNumber, headNum}) 240 if err != nil { 241 return err 242 } 243 } 244 } 245 246 if len(accountsWithDetectedChanges) != 0 || c.iteration%c.logsCheckIntervalIterations == 0 { 247 from := c.fromBlockNumber 248 if c.logsCheckLastKnownBlock != nil { 249 from = c.logsCheckLastKnownBlock 250 } 251 err = c.findAndSaveTokenBlocks(parent, from, headNum) 252 if err != nil { 253 return err 254 } 255 c.logsCheckLastKnownBlock = headNum 256 } 257 c.fromBlockNumber = headNum 258 c.iteration++ 259 260 return nil 261 } 262 263 func (c *findNewBlocksCommand) findAndSaveEthBlocks(parent context.Context, fromNum, headNum *big.Int, accounts []common.Address) error { 264 // Check ETH transfers for each account independently 265 mnemonicWasNotShown, err := c.accountsDB.GetMnemonicWasNotShown() 266 if err != nil { 267 return err 268 } 269 270 for _, account := range accounts { 271 if mnemonicWasNotShown { 272 acc, err := c.accountsDB.GetAccountByAddress(nodetypes.Address(account)) 273 if err != nil { 274 return err 275 } 276 if acc.AddressWasNotShown { 277 log.Info("skip findNewBlocksCommand, mnemonic has not been shown and the address has not been shared yet", "address", account) 278 continue 279 } 280 } 281 282 log.Debug("start findNewBlocksCommand", "account", account, "chain", c.chainClient.NetworkID(), "noLimit", c.noLimit, "from", fromNum, "to", headNum) 283 284 headers, startBlockNum, err := c.findBlocksWithEthTransfers(parent, account, fromNum, headNum) 285 if err != nil { 286 return err 287 } 288 289 if len(headers) > 0 { 290 log.Debug("findNewBlocksCommand saving headers", "len", len(headers), "lastBlockNumber", headNum, 291 "balance", c.balanceCacher.Cache().GetBalance(account, c.chainClient.NetworkID(), headNum), 292 "nonce", c.balanceCacher.Cache().GetNonce(account, c.chainClient.NetworkID(), headNum)) 293 294 err := c.db.SaveBlocks(c.chainClient.NetworkID(), headers) 295 if err != nil { 296 return err 297 } 298 299 c.blocksFound(headers) 300 } 301 302 err = c.markEthBlockRangeChecked(account, &BlockRange{startBlockNum, fromNum, headNum}) 303 if err != nil { 304 return err 305 } 306 307 log.Debug("end findNewBlocksCommand", "account", account, "chain", c.chainClient.NetworkID(), "noLimit", c.noLimit, "from", fromNum, "to", headNum) 308 } 309 310 return nil 311 } 312 313 func (c *findNewBlocksCommand) findAndSaveTokenBlocks(parent context.Context, fromNum, headNum *big.Int) error { 314 // Check token transfers for all accounts. 315 // Each account's last checked block can be different, so we can get duplicated headers, 316 // so we need to deduplicate them 317 const incomingOnly = false 318 erc20Headers, err := c.fastIndexErc20(parent, fromNum, headNum, incomingOnly) 319 if err != nil { 320 log.Error("findNewBlocksCommand fastIndexErc20", "err", err, "account", c.accounts, "chain", c.chainClient.NetworkID()) 321 return err 322 } 323 324 if len(erc20Headers) > 0 { 325 log.Debug("findNewBlocksCommand saving headers", "len", len(erc20Headers), "from", fromNum, "to", headNum) 326 327 // get not loaded headers from DB for all accs and blocks 328 preLoadedTransactions, err := c.db.GetTransactionsToLoad(c.chainClient.NetworkID(), common.Address{}, nil) 329 if err != nil { 330 return err 331 } 332 333 tokenBlocksFiltered := filterNewPreloadedTransactions(erc20Headers, preLoadedTransactions) 334 335 err = c.db.SaveBlocks(c.chainClient.NetworkID(), tokenBlocksFiltered) 336 if err != nil { 337 return err 338 } 339 340 c.blocksFound(tokenBlocksFiltered) 341 } 342 343 return c.markTokenBlockRangeChecked(c.accounts, fromNum, headNum) 344 } 345 346 func (c *findBlocksCommand) markTokenBlockRangeChecked(accounts []common.Address, from, to *big.Int) error { 347 log.Debug("markTokenBlockRangeChecked", "chain", c.chainClient.NetworkID(), "from", from.Uint64(), "to", to.Uint64()) 348 349 for _, account := range accounts { 350 err := c.blockRangeDAO.updateTokenRange(c.chainClient.NetworkID(), account, &BlockRange{FirstKnown: from, LastKnown: to}) 351 if err != nil { 352 log.Error("findNewBlocksCommand upsertTokenRange", "error", err) 353 return err 354 } 355 } 356 357 return nil 358 } 359 360 func filterNewPreloadedTransactions(erc20Headers []*DBHeader, preLoadedTransfers []*PreloadedTransaction) []*DBHeader { 361 var uniqueErc20Headers []*DBHeader 362 for _, header := range erc20Headers { 363 loaded := false 364 for _, transfer := range preLoadedTransfers { 365 if header.PreloadedTransactions[0].ID == transfer.ID { 366 loaded = true 367 break 368 } 369 } 370 371 if !loaded { 372 uniqueErc20Headers = append(uniqueErc20Headers, header) 373 } 374 } 375 376 return uniqueErc20Headers 377 } 378 379 func (c *findNewBlocksCommand) findBlocksWithEthTransfers(parent context.Context, account common.Address, fromOrig, toOrig *big.Int) (headers []*DBHeader, startBlockNum *big.Int, err error) { 380 log.Debug("start findNewBlocksCommand::findBlocksWithEthTransfers", "account", account, "chain", c.chainClient.NetworkID(), "noLimit", c.noLimit, "from", c.fromBlockNumber, "to", c.toBlockNumber) 381 382 rangeSize := big.NewInt(int64(c.defaultNodeBlockChunkSize)) 383 384 from, to := new(big.Int).Set(fromOrig), new(big.Int).Set(toOrig) 385 386 // Limit the range size to DefaultNodeBlockChunkSize 387 if new(big.Int).Sub(to, from).Cmp(rangeSize) > 0 { 388 from.Sub(to, rangeSize) 389 } 390 391 for { 392 if from.Cmp(to) == 0 { 393 log.Debug("findNewBlocksCommand empty range", "from", from, "to", to) 394 break 395 } 396 397 fromBlock := &Block{Number: from} 398 399 var newFromBlock *Block 400 var ethHeaders []*DBHeader 401 newFromBlock, ethHeaders, startBlockNum, err = c.fastIndex(parent, account, c.balanceCacher, fromBlock, to) 402 if err != nil { 403 log.Error("findNewBlocksCommand checkRange fastIndex", "err", err, "account", account, 404 "chain", c.chainClient.NetworkID()) 405 return nil, nil, err 406 } 407 log.Debug("findNewBlocksCommand checkRange", "chainID", c.chainClient.NetworkID(), "account", account, 408 "startBlock", startBlockNum, "newFromBlock", newFromBlock.Number, "toBlockNumber", to, "noLimit", c.noLimit) 409 410 headers = append(headers, ethHeaders...) 411 412 if startBlockNum != nil && startBlockNum.Cmp(from) >= 0 { 413 log.Debug("Checked all ranges, stop execution", "startBlock", startBlockNum, "from", from, "to", to) 414 break 415 } 416 417 nextFrom, nextTo := nextRange(c.defaultNodeBlockChunkSize, newFromBlock.Number, fromOrig) 418 419 if nextFrom.Cmp(from) == 0 && nextTo.Cmp(to) == 0 { 420 log.Debug("findNewBlocksCommand empty next range", "from", from, "to", to) 421 break 422 } 423 424 from = nextFrom 425 to = nextTo 426 } 427 428 log.Debug("end findNewBlocksCommand::findBlocksWithEthTransfers", "account", account, "chain", c.chainClient.NetworkID(), "noLimit", c.noLimit) 429 430 return headers, startBlockNum, nil 431 } 432 433 // TODO NewFindBlocksCommand 434 type findBlocksCommand struct { 435 accounts []common.Address 436 db *Database 437 accountsDB *accounts.Database 438 blockRangeDAO BlockRangeDAOer 439 chainClient chain.ClientInterface 440 balanceCacher balance.Cacher 441 feed *event.Feed 442 noLimit bool 443 tokenManager *token.Manager 444 fromBlockNumber *big.Int 445 logsCheckLastKnownBlock *big.Int 446 toBlockNumber *big.Int 447 blocksLoadedCh chan<- []*DBHeader 448 defaultNodeBlockChunkSize int 449 450 // Not to be set by the caller 451 resFromBlock *Block 452 startBlockNumber *big.Int 453 reachedETHHistoryStart bool 454 } 455 456 func (c *findBlocksCommand) Runner(interval ...time.Duration) async.Runner { 457 intvl := findBlocksRetryInterval 458 if len(interval) > 0 { 459 intvl = interval[0] 460 } 461 return async.FiniteCommandWithErrorCounter{ 462 FiniteCommand: async.FiniteCommand{ 463 Interval: intvl, 464 Runable: c.Run, 465 }, 466 ErrorCounter: async.NewErrorCounter(3, "findBlocksCommand"), 467 } 468 } 469 470 func (c *findBlocksCommand) Command(interval ...time.Duration) async.Command { 471 return c.Runner(interval...).Run 472 } 473 474 type ERC20BlockRange struct { 475 from *big.Int 476 to *big.Int 477 } 478 479 func (c *findBlocksCommand) ERC20ScanByBalance(parent context.Context, account common.Address, fromBlock, toBlock *big.Int, token common.Address) ([]ERC20BlockRange, error) { 480 var err error 481 batchSize := getErc20BatchSize(c.chainClient.NetworkID()) 482 ranges := [][]*big.Int{{fromBlock, toBlock}} 483 foundRanges := []ERC20BlockRange{} 484 cache := map[int64]*big.Int{} 485 for { 486 nextRanges := [][]*big.Int{} 487 for _, blockRange := range ranges { 488 from, to := blockRange[0], blockRange[1] 489 fromBalance, ok := cache[from.Int64()] 490 if !ok { 491 fromBalance, err = c.tokenManager.GetTokenBalanceAt(parent, c.chainClient, account, token, from) 492 if err != nil { 493 return nil, err 494 } 495 496 if fromBalance == nil { 497 fromBalance = big.NewInt(0) 498 } 499 cache[from.Int64()] = fromBalance 500 } 501 502 toBalance, ok := cache[to.Int64()] 503 if !ok { 504 toBalance, err = c.tokenManager.GetTokenBalanceAt(parent, c.chainClient, account, token, to) 505 if err != nil { 506 return nil, err 507 } 508 if toBalance == nil { 509 toBalance = big.NewInt(0) 510 } 511 cache[to.Int64()] = toBalance 512 } 513 514 if fromBalance.Cmp(toBalance) != 0 { 515 diff := new(big.Int).Sub(to, from) 516 if diff.Cmp(batchSize) <= 0 { 517 foundRanges = append(foundRanges, ERC20BlockRange{from, to}) 518 continue 519 } 520 521 halfOfDiff := new(big.Int).Div(diff, big.NewInt(2)) 522 mid := new(big.Int).Add(from, halfOfDiff) 523 524 nextRanges = append(nextRanges, []*big.Int{from, mid}) 525 nextRanges = append(nextRanges, []*big.Int{mid, to}) 526 } 527 } 528 529 if len(nextRanges) == 0 { 530 break 531 } 532 533 ranges = nextRanges 534 } 535 536 return foundRanges, nil 537 } 538 539 func (c *findBlocksCommand) checkERC20Tail(parent context.Context, account common.Address) ([]*DBHeader, error) { 540 log.Debug("checkERC20Tail", "account", account, "to block", c.startBlockNumber, "from", c.resFromBlock.Number) 541 tokens, err := c.tokenManager.GetTokens(c.chainClient.NetworkID()) 542 if err != nil { 543 return nil, err 544 } 545 addresses := make([]common.Address, len(tokens)) 546 for i, token := range tokens { 547 addresses[i] = token.Address 548 } 549 550 from := new(big.Int).Sub(c.resFromBlock.Number, big.NewInt(1)) 551 552 clients := make(map[uint64]chain.ClientInterface, 1) 553 clients[c.chainClient.NetworkID()] = c.chainClient 554 atBlocks := make(map[uint64]*big.Int, 1) 555 atBlocks[c.chainClient.NetworkID()] = from 556 balances, err := c.tokenManager.GetBalancesAtByChain(parent, clients, []common.Address{account}, addresses, atBlocks) 557 if err != nil { 558 return nil, err 559 } 560 561 foundRanges := []ERC20BlockRange{} 562 for token, balance := range balances[c.chainClient.NetworkID()][account] { 563 bigintBalance := big.NewInt(balance.ToInt().Int64()) 564 if bigintBalance.Cmp(big.NewInt(0)) <= 0 { 565 continue 566 } 567 result, err := c.ERC20ScanByBalance(parent, account, big.NewInt(0), from, token) 568 if err != nil { 569 return nil, err 570 } 571 572 foundRanges = append(foundRanges, result...) 573 } 574 575 uniqRanges := []ERC20BlockRange{} 576 rangesMap := map[string]bool{} 577 for _, rangeItem := range foundRanges { 578 key := rangeItem.from.String() + "-" + rangeItem.to.String() 579 if _, ok := rangesMap[key]; !ok { 580 rangesMap[key] = true 581 uniqRanges = append(uniqRanges, rangeItem) 582 } 583 } 584 585 foundHeaders := []*DBHeader{} 586 for _, rangeItem := range uniqRanges { 587 headers, err := c.fastIndexErc20(parent, rangeItem.from, rangeItem.to, true) 588 if err != nil { 589 return nil, err 590 } 591 foundHeaders = append(foundHeaders, headers...) 592 } 593 594 return foundHeaders, nil 595 } 596 597 func (c *findBlocksCommand) Run(parent context.Context) (err error) { 598 log.Debug("start findBlocksCommand", "accounts", c.accounts, "chain", c.chainClient.NetworkID(), "noLimit", c.noLimit, "from", c.fromBlockNumber, "to", c.toBlockNumber) 599 600 account := c.accounts[0] // For now this command supports only 1 account 601 mnemonicWasNotShown, err := c.accountsDB.GetMnemonicWasNotShown() 602 if err != nil { 603 return err 604 } 605 606 if mnemonicWasNotShown { 607 account, err := c.accountsDB.GetAccountByAddress(nodetypes.BytesToAddress(account.Bytes())) 608 if err != nil { 609 return err 610 } 611 if account.AddressWasNotShown { 612 log.Info("skip findBlocksCommand, mnemonic has not been shown and the address has not been shared yet", "address", account) 613 return nil 614 } 615 } 616 617 rangeSize := big.NewInt(int64(c.defaultNodeBlockChunkSize)) 618 from, to := new(big.Int).Set(c.fromBlockNumber), new(big.Int).Set(c.toBlockNumber) 619 620 // Limit the range size to DefaultNodeBlockChunkSize 621 if new(big.Int).Sub(to, from).Cmp(rangeSize) > 0 { 622 from.Sub(to, rangeSize) 623 } 624 625 for { 626 if from.Cmp(to) == 0 { 627 log.Debug("findBlocksCommand empty range", "from", from, "to", to) 628 break 629 } 630 631 var headers []*DBHeader 632 if c.reachedETHHistoryStart { 633 if c.fromBlockNumber.Cmp(zero) == 0 && c.startBlockNumber != nil && c.startBlockNumber.Cmp(zero) == 1 { 634 headers, err = c.checkERC20Tail(parent, account) 635 if err != nil { 636 log.Error("findBlocksCommand checkERC20Tail", "err", err, "account", account, "chain", c.chainClient.NetworkID()) 637 break 638 } 639 } 640 } else { 641 headers, err = c.checkRange(parent, from, to) 642 if err != nil { 643 break 644 } 645 } 646 647 if len(headers) > 0 { 648 log.Debug("findBlocksCommand saving headers", "len", len(headers), "lastBlockNumber", to, 649 "balance", c.balanceCacher.Cache().GetBalance(account, c.chainClient.NetworkID(), to), 650 "nonce", c.balanceCacher.Cache().GetNonce(account, c.chainClient.NetworkID(), to)) 651 652 err = c.db.SaveBlocks(c.chainClient.NetworkID(), headers) 653 if err != nil { 654 break 655 } 656 657 c.blocksFound(headers) 658 } 659 660 if c.reachedETHHistoryStart { 661 err = c.markTokenBlockRangeChecked([]common.Address{account}, big.NewInt(0), to) 662 if err != nil { 663 break 664 } 665 log.Debug("findBlocksCommand reached first ETH transfer and checked erc20 tail", "chain", c.chainClient.NetworkID(), "account", account) 666 break 667 } 668 669 err = c.markEthBlockRangeChecked(account, &BlockRange{c.startBlockNumber, c.resFromBlock.Number, to}) 670 if err != nil { 671 break 672 } 673 674 err = c.markTokenBlockRangeChecked([]common.Address{account}, c.resFromBlock.Number, to) 675 if err != nil { 676 break 677 } 678 679 // if we have found first ETH block and we have not reached the start of ETH history yet 680 if c.startBlockNumber != nil && c.fromBlockNumber.Cmp(from) == -1 { 681 log.Debug("ERC20 tail should be checked", "initial from", c.fromBlockNumber, "actual from", from, "first ETH block", c.startBlockNumber) 682 c.reachedETHHistoryStart = true 683 continue 684 } 685 686 if c.startBlockNumber != nil && c.startBlockNumber.Cmp(from) >= 0 { 687 log.Debug("Checked all ranges, stop execution", "startBlock", c.startBlockNumber, "from", from, "to", to) 688 break 689 } 690 691 nextFrom, nextTo := nextRange(c.defaultNodeBlockChunkSize, c.resFromBlock.Number, c.fromBlockNumber) 692 693 if nextFrom.Cmp(from) == 0 && nextTo.Cmp(to) == 0 { 694 log.Debug("findBlocksCommand empty next range", "from", from, "to", to) 695 break 696 } 697 698 from = nextFrom 699 to = nextTo 700 } 701 702 log.Debug("end findBlocksCommand", "account", account, "chain", c.chainClient.NetworkID(), "noLimit", c.noLimit, "err", err) 703 704 return err 705 } 706 707 func (c *findBlocksCommand) blocksFound(headers []*DBHeader) { 708 c.blocksLoadedCh <- headers 709 } 710 711 func (c *findBlocksCommand) markEthBlockRangeChecked(account common.Address, blockRange *BlockRange) error { 712 log.Debug("upsert block range", "Start", blockRange.Start, "FirstKnown", blockRange.FirstKnown, "LastKnown", blockRange.LastKnown, 713 "chain", c.chainClient.NetworkID(), "account", account) 714 715 err := c.blockRangeDAO.upsertEthRange(c.chainClient.NetworkID(), account, blockRange) 716 if err != nil { 717 log.Error("findBlocksCommand upsertRange", "error", err) 718 return err 719 } 720 721 return nil 722 } 723 724 func (c *findBlocksCommand) checkRange(parent context.Context, from *big.Int, to *big.Int) ( 725 foundHeaders []*DBHeader, err error) { 726 727 account := c.accounts[0] 728 fromBlock := &Block{Number: from} 729 730 newFromBlock, ethHeaders, startBlock, err := c.fastIndex(parent, account, c.balanceCacher, fromBlock, to) 731 if err != nil { 732 log.Error("findBlocksCommand checkRange fastIndex", "err", err, "account", account, 733 "chain", c.chainClient.NetworkID()) 734 return nil, err 735 } 736 log.Debug("findBlocksCommand checkRange", "chainID", c.chainClient.NetworkID(), "account", account, 737 "startBlock", startBlock, "newFromBlock", newFromBlock.Number, "toBlockNumber", to, "noLimit", c.noLimit) 738 739 // There could be incoming ERC20 transfers which don't change the balance 740 // and nonce of ETH account, so we keep looking for them 741 erc20Headers, err := c.fastIndexErc20(parent, newFromBlock.Number, to, false) 742 if err != nil { 743 log.Error("findBlocksCommand checkRange fastIndexErc20", "err", err, "account", account, "chain", c.chainClient.NetworkID()) 744 return nil, err 745 } 746 747 allHeaders := append(ethHeaders, erc20Headers...) 748 749 if len(allHeaders) > 0 { 750 foundHeaders = uniqueHeaderPerBlockHash(allHeaders) 751 } 752 753 c.resFromBlock = newFromBlock 754 c.startBlockNumber = startBlock 755 756 log.Debug("end findBlocksCommand checkRange", "chainID", c.chainClient.NetworkID(), "account", account, 757 "c.startBlock", c.startBlockNumber, "newFromBlock", newFromBlock.Number, 758 "toBlockNumber", to, "c.resFromBlock", c.resFromBlock.Number) 759 760 return 761 } 762 763 func loadBlockRangeInfo(chainID uint64, account common.Address, blockDAO BlockRangeDAOer) ( 764 *ethTokensBlockRanges, error) { 765 766 blockRange, _, err := blockDAO.getBlockRange(chainID, account) 767 if err != nil { 768 log.Error("failed to load block ranges from database", "chain", chainID, "account", account, 769 "error", err) 770 return nil, err 771 } 772 773 return blockRange, nil 774 } 775 776 // Returns if all blocks are loaded, which means that start block (beginning of account history) 777 // has been found and all block headers saved to the DB 778 func areAllHistoryBlocksLoaded(blockInfo *BlockRange) bool { 779 if blockInfo != nil && blockInfo.FirstKnown != nil && 780 ((blockInfo.Start != nil && blockInfo.Start.Cmp(blockInfo.FirstKnown) >= 0) || 781 blockInfo.FirstKnown.Cmp(zero) == 0) { 782 return true 783 } 784 785 return false 786 } 787 788 func areAllHistoryBlocksLoadedForAddress(blockRangeDAO BlockRangeDAOer, chainID uint64, 789 address common.Address) (bool, error) { 790 791 blockRange, _, err := blockRangeDAO.getBlockRange(chainID, address) 792 if err != nil { 793 log.Error("findBlocksCommand getBlockRange", "error", err) 794 return false, err 795 } 796 797 return areAllHistoryBlocksLoaded(blockRange.eth) && areAllHistoryBlocksLoaded(blockRange.tokens), nil 798 } 799 800 // run fast indexing for every accont up to canonical chain head minus safety depth. 801 // every account will run it from last synced header. 802 func (c *findBlocksCommand) fastIndex(ctx context.Context, account common.Address, bCacher balance.Cacher, 803 fromBlock *Block, toBlockNumber *big.Int) (resultingFrom *Block, headers []*DBHeader, 804 startBlock *big.Int, err error) { 805 806 log.Debug("fast index started", "chainID", c.chainClient.NetworkID(), "account", account, 807 "from", fromBlock.Number, "to", toBlockNumber) 808 809 start := time.Now() 810 group := async.NewGroup(ctx) 811 812 command := ðHistoricalCommand{ 813 chainClient: c.chainClient, 814 balanceCacher: bCacher, 815 address: account, 816 feed: c.feed, 817 from: fromBlock, 818 to: toBlockNumber, 819 noLimit: c.noLimit, 820 threadLimit: SequentialThreadLimit, 821 } 822 group.Add(command.Command()) 823 824 select { 825 case <-ctx.Done(): 826 err = ctx.Err() 827 log.Debug("fast indexer ctx Done", "error", err) 828 return 829 case <-group.WaitAsync(): 830 if command.error != nil { 831 err = command.error 832 return 833 } 834 resultingFrom = &Block{Number: command.resultingFrom} 835 headers = command.foundHeaders 836 startBlock = command.startBlock 837 log.Debug("fast indexer finished", "chainID", c.chainClient.NetworkID(), "account", account, "in", time.Since(start), 838 "startBlock", command.startBlock, "resultingFrom", resultingFrom.Number, "headers", len(headers)) 839 return 840 } 841 } 842 843 // run fast indexing for every accont up to canonical chain head minus safety depth. 844 // every account will run it from last synced header. 845 func (c *findBlocksCommand) fastIndexErc20(ctx context.Context, fromBlockNumber *big.Int, 846 toBlockNumber *big.Int, incomingOnly bool) ([]*DBHeader, error) { 847 848 start := time.Now() 849 group := async.NewGroup(ctx) 850 851 erc20 := &erc20HistoricalCommand{ 852 erc20: NewERC20TransfersDownloader(c.chainClient, c.accounts, types.LatestSignerForChainID(c.chainClient.ToBigInt()), incomingOnly), 853 chainClient: c.chainClient, 854 feed: c.feed, 855 from: fromBlockNumber, 856 to: toBlockNumber, 857 foundHeaders: []*DBHeader{}, 858 } 859 group.Add(erc20.Command()) 860 861 select { 862 case <-ctx.Done(): 863 return nil, ctx.Err() 864 case <-group.WaitAsync(): 865 headers := erc20.foundHeaders 866 log.Debug("fast indexer Erc20 finished", "chainID", c.chainClient.NetworkID(), 867 "in", time.Since(start), "headers", len(headers)) 868 return headers, nil 869 } 870 } 871 872 // Start transfers loop to load transfers for new blocks 873 func (c *loadBlocksAndTransfersCommand) startTransfersLoop(ctx context.Context) { 874 c.incLoops() 875 go func() { 876 defer func() { 877 c.decLoops() 878 }() 879 880 log.Debug("loadTransfersLoop start", "chain", c.chainClient.NetworkID()) 881 882 for { 883 select { 884 case <-ctx.Done(): 885 log.Debug("startTransfersLoop done", "chain", c.chainClient.NetworkID(), "error", ctx.Err()) 886 return 887 case dbHeaders := <-c.blocksLoadedCh: 888 log.Debug("loadTransfersOnDemand transfers received", "chain", c.chainClient.NetworkID(), "headers", len(dbHeaders)) 889 890 blocksByAddress := map[common.Address][]*big.Int{} 891 // iterate over headers and group them by address 892 for _, dbHeader := range dbHeaders { 893 blocksByAddress[dbHeader.Address] = append(blocksByAddress[dbHeader.Address], dbHeader.Number) 894 } 895 896 go func() { 897 _ = loadTransfers(ctx, c.blockDAO, c.db, c.chainClient, noBlockLimit, 898 blocksByAddress, c.transactionManager, c.pendingTxManager, c.tokenManager, c.feed) 899 }() 900 } 901 } 902 }() 903 } 904 905 func newLoadBlocksAndTransfersCommand(accounts []common.Address, db *Database, accountsDB *accounts.Database, 906 blockDAO *BlockDAO, blockRangesSeqDAO BlockRangeDAOer, chainClient chain.ClientInterface, feed *event.Feed, 907 transactionManager *TransactionManager, pendingTxManager *transactions.PendingTxTracker, 908 tokenManager *token.Manager, balanceCacher balance.Cacher, omitHistory bool, 909 blockChainState *blockchainstate.BlockChainState) *loadBlocksAndTransfersCommand { 910 911 return &loadBlocksAndTransfersCommand{ 912 accounts: accounts, 913 db: db, 914 blockRangeDAO: blockRangesSeqDAO, 915 accountsDB: accountsDB, 916 blockDAO: blockDAO, 917 chainClient: chainClient, 918 feed: feed, 919 balanceCacher: balanceCacher, 920 transactionManager: transactionManager, 921 pendingTxManager: pendingTxManager, 922 tokenManager: tokenManager, 923 blocksLoadedCh: make(chan []*DBHeader, 100), 924 omitHistory: omitHistory, 925 contractMaker: tokenManager.ContractMaker, 926 blockChainState: blockChainState, 927 } 928 } 929 930 type loadBlocksAndTransfersCommand struct { 931 accounts []common.Address 932 db *Database 933 accountsDB *accounts.Database 934 blockRangeDAO BlockRangeDAOer 935 blockDAO *BlockDAO 936 chainClient chain.ClientInterface 937 feed *event.Feed 938 balanceCacher balance.Cacher 939 // nonArchivalRPCNode bool // TODO Make use of it 940 transactionManager *TransactionManager 941 pendingTxManager *transactions.PendingTxTracker 942 tokenManager *token.Manager 943 blocksLoadedCh chan []*DBHeader 944 omitHistory bool 945 contractMaker *contracts.ContractMaker 946 blockChainState *blockchainstate.BlockChainState 947 948 // Not to be set by the caller 949 transfersLoaded map[common.Address]bool // For event RecentHistoryReady to be sent only once per account during app lifetime 950 loops atomic.Int32 951 } 952 953 func (c *loadBlocksAndTransfersCommand) incLoops() { 954 c.loops.Add(1) 955 } 956 957 func (c *loadBlocksAndTransfersCommand) decLoops() { 958 c.loops.Add(-1) 959 } 960 961 func (c *loadBlocksAndTransfersCommand) isStarted() bool { 962 return c.loops.Load() > 0 963 } 964 965 func (c *loadBlocksAndTransfersCommand) Run(parent context.Context) (err error) { 966 log.Debug("start load all transfers command", "chain", c.chainClient.NetworkID(), "accounts", c.accounts) 967 968 // Finite processes (to be restarted on error, but stopped on success or context cancel): 969 // fetching transfers for loaded blocks 970 // fetching history blocks 971 972 // Infinite processes (to be restarted on error), but stopped on context cancel: 973 // fetching new blocks 974 // fetching transfers for new blocks 975 976 ctx := parent 977 finiteGroup := async.NewAtomicGroup(ctx) 978 finiteGroup.SetName("finiteGroup") 979 defer func() { 980 finiteGroup.Stop() 981 finiteGroup.Wait() 982 }() 983 984 blockRanges, err := c.blockRangeDAO.getBlockRanges(c.chainClient.NetworkID(), c.accounts) 985 if err != nil { 986 return err 987 } 988 989 firstScan := false 990 var headNum *big.Int 991 for _, address := range c.accounts { 992 blockRange, ok := blockRanges[address] 993 if !ok || blockRange.tokens.LastKnown == nil { 994 firstScan = true 995 break 996 } 997 998 if headNum == nil || blockRange.tokens.LastKnown.Cmp(headNum) < 0 { 999 headNum = blockRange.tokens.LastKnown 1000 } 1001 } 1002 1003 fromNum := big.NewInt(0) 1004 if firstScan { 1005 headNum, err = getHeadBlockNumber(ctx, c.chainClient) 1006 if err != nil { 1007 return err 1008 } 1009 } 1010 1011 // It will start loadTransfersCommand which will run until all transfers from DB are loaded or any one failed to load 1012 err = c.startFetchingTransfersForLoadedBlocks(finiteGroup) 1013 if err != nil { 1014 log.Error("loadBlocksAndTransfersCommand fetchTransfersForLoadedBlocks", "error", err) 1015 return err 1016 } 1017 1018 if !c.isStarted() { 1019 c.startTransfersLoop(ctx) 1020 c.startFetchingNewBlocks(ctx, c.accounts, headNum, c.blocksLoadedCh) 1021 } 1022 1023 // It will start findBlocksCommands which will run until success when all blocks are loaded 1024 err = c.fetchHistoryBlocks(finiteGroup, c.accounts, fromNum, headNum, c.blocksLoadedCh) 1025 if err != nil { 1026 log.Error("loadBlocksAndTransfersCommand fetchHistoryBlocks", "error", err) 1027 return err 1028 } 1029 1030 select { 1031 case <-ctx.Done(): 1032 log.Debug("loadBlocksAndTransfers command cancelled", "chain", c.chainClient.NetworkID(), "accounts", c.accounts, "error", ctx.Err()) 1033 case <-finiteGroup.WaitAsync(): 1034 err = finiteGroup.Error() // if there was an error, rerun the command 1035 log.Debug("end loadBlocksAndTransfers command", "chain", c.chainClient.NetworkID(), "accounts", c.accounts, "error", err, "group", finiteGroup.Name()) 1036 } 1037 1038 return err 1039 } 1040 1041 func (c *loadBlocksAndTransfersCommand) Runner(interval ...time.Duration) async.Runner { 1042 // 30s - default interval for Infura's delay returned in error. That should increase chances 1043 // for request to succeed with the next attempt for now until we have a proper retry mechanism 1044 intvl := 30 * time.Second 1045 if len(interval) > 0 { 1046 intvl = interval[0] 1047 } 1048 1049 return async.FiniteCommand{ 1050 Interval: intvl, 1051 Runable: c.Run, 1052 } 1053 } 1054 1055 func (c *loadBlocksAndTransfersCommand) Command(interval ...time.Duration) async.Command { 1056 return c.Runner(interval...).Run 1057 } 1058 1059 func (c *loadBlocksAndTransfersCommand) fetchHistoryBlocks(group *async.AtomicGroup, accounts []common.Address, fromNum, toNum *big.Int, blocksLoadedCh chan []*DBHeader) (err error) { 1060 for _, account := range accounts { 1061 err = c.fetchHistoryBlocksForAccount(group, account, fromNum, toNum, c.blocksLoadedCh) 1062 if err != nil { 1063 return err 1064 } 1065 } 1066 return nil 1067 } 1068 1069 func (c *loadBlocksAndTransfersCommand) fetchHistoryBlocksForAccount(group *async.AtomicGroup, account common.Address, fromNum, toNum *big.Int, blocksLoadedCh chan []*DBHeader) error { 1070 1071 log.Debug("fetchHistoryBlocks start", "chainID", c.chainClient.NetworkID(), "account", account, "omit", c.omitHistory) 1072 1073 if c.omitHistory { 1074 blockRange := ðTokensBlockRanges{eth: &BlockRange{nil, big.NewInt(0), toNum}, tokens: &BlockRange{nil, big.NewInt(0), toNum}} 1075 err := c.blockRangeDAO.upsertRange(c.chainClient.NetworkID(), account, blockRange) 1076 log.Error("fetchHistoryBlocks upsertRange", "error", err) 1077 return err 1078 } 1079 1080 blockRange, err := loadBlockRangeInfo(c.chainClient.NetworkID(), account, c.blockRangeDAO) 1081 if err != nil { 1082 log.Error("fetchHistoryBlocks loadBlockRangeInfo", "error", err) 1083 return err 1084 } 1085 1086 ranges := [][]*big.Int{} 1087 // There are 2 history intervals: 1088 // 1) from 0 to FirstKnown 1089 // 2) from LastKnown to `toNum`` (head) 1090 // If we blockRange is nil, we need to load all blocks from `fromNum` to `toNum` 1091 // As current implementation checks ETH first then tokens, tokens ranges maybe behind ETH ranges in 1092 // cases when block searching was interrupted, so we use tokens ranges 1093 if blockRange.tokens.LastKnown != nil || blockRange.tokens.FirstKnown != nil { 1094 if blockRange.tokens.LastKnown != nil && toNum.Cmp(blockRange.tokens.LastKnown) > 0 { 1095 ranges = append(ranges, []*big.Int{blockRange.tokens.LastKnown, toNum}) 1096 } 1097 1098 if blockRange.tokens.FirstKnown != nil { 1099 if fromNum.Cmp(blockRange.tokens.FirstKnown) < 0 { 1100 ranges = append(ranges, []*big.Int{fromNum, blockRange.tokens.FirstKnown}) 1101 } else { 1102 if !c.transfersLoaded[account] { 1103 transfersLoaded, err := c.areAllTransfersLoaded(account) 1104 if err != nil { 1105 return err 1106 } 1107 1108 if transfersLoaded { 1109 if c.transfersLoaded == nil { 1110 c.transfersLoaded = make(map[common.Address]bool) 1111 } 1112 c.transfersLoaded[account] = true 1113 c.notifyHistoryReady(account) 1114 } 1115 } 1116 } 1117 } 1118 } else { 1119 ranges = append(ranges, []*big.Int{fromNum, toNum}) 1120 } 1121 1122 if len(ranges) > 0 { 1123 storage := chain.NewLimitsDBStorage(c.db.client) 1124 limiter := chain.NewRequestLimiter(storage) 1125 chainClient, _ := createChainClientWithLimiter(c.chainClient, account, limiter) 1126 if chainClient == nil { 1127 chainClient = c.chainClient 1128 } 1129 1130 for _, rangeItem := range ranges { 1131 log.Debug("range item", "r", rangeItem, "n", c.chainClient.NetworkID(), "a", account) 1132 1133 fbc := &findBlocksCommand{ 1134 accounts: []common.Address{account}, 1135 db: c.db, 1136 accountsDB: c.accountsDB, 1137 blockRangeDAO: c.blockRangeDAO, 1138 chainClient: chainClient, 1139 balanceCacher: c.balanceCacher, 1140 feed: c.feed, 1141 noLimit: false, 1142 fromBlockNumber: rangeItem[0], 1143 toBlockNumber: rangeItem[1], 1144 tokenManager: c.tokenManager, 1145 blocksLoadedCh: blocksLoadedCh, 1146 defaultNodeBlockChunkSize: DefaultNodeBlockChunkSize, 1147 } 1148 group.Add(fbc.Command()) 1149 } 1150 } 1151 1152 return nil 1153 } 1154 1155 func (c *loadBlocksAndTransfersCommand) startFetchingNewBlocks(ctx context.Context, addresses []common.Address, fromNum *big.Int, blocksLoadedCh chan<- []*DBHeader) { 1156 log.Debug("startFetchingNewBlocks start", "chainID", c.chainClient.NetworkID(), "accounts", addresses) 1157 1158 c.incLoops() 1159 go func() { 1160 defer func() { 1161 c.decLoops() 1162 }() 1163 1164 newBlocksCmd := &findNewBlocksCommand{ 1165 findBlocksCommand: &findBlocksCommand{ 1166 accounts: addresses, 1167 db: c.db, 1168 accountsDB: c.accountsDB, 1169 blockRangeDAO: c.blockRangeDAO, 1170 chainClient: c.chainClient, 1171 balanceCacher: c.balanceCacher, 1172 feed: c.feed, 1173 noLimit: false, 1174 fromBlockNumber: fromNum, 1175 tokenManager: c.tokenManager, 1176 blocksLoadedCh: blocksLoadedCh, 1177 defaultNodeBlockChunkSize: DefaultNodeBlockChunkSize, 1178 }, 1179 contractMaker: c.contractMaker, 1180 blockChainState: c.blockChainState, 1181 nonceCheckIntervalIterations: nonceCheckIntervalIterations, 1182 logsCheckIntervalIterations: logsCheckIntervalIterations, 1183 } 1184 group := async.NewGroup(ctx) 1185 group.Add(newBlocksCmd.Command()) 1186 1187 // No need to wait for the group since it is infinite 1188 <-ctx.Done() 1189 1190 log.Debug("startFetchingNewBlocks end", "chainID", c.chainClient.NetworkID(), "accounts", addresses, "error", ctx.Err()) 1191 }() 1192 } 1193 1194 func (c *loadBlocksAndTransfersCommand) getBlocksToLoad() (map[common.Address][]*big.Int, error) { 1195 blocksMap := make(map[common.Address][]*big.Int) 1196 for _, account := range c.accounts { 1197 blocks, err := c.blockDAO.GetBlocksToLoadByAddress(c.chainClient.NetworkID(), account, numberOfBlocksCheckedPerIteration) 1198 if err != nil { 1199 log.Error("loadBlocksAndTransfersCommand GetBlocksToLoadByAddress", "error", err) 1200 return nil, err 1201 } 1202 1203 if len(blocks) == 0 { 1204 log.Debug("fetchTransfers no blocks to load", "chainID", c.chainClient.NetworkID(), "account", account) 1205 continue 1206 } 1207 1208 blocksMap[account] = blocks 1209 } 1210 1211 if len(blocksMap) == 0 { 1212 log.Debug("fetchTransfers no blocks to load", "chainID", c.chainClient.NetworkID()) 1213 } 1214 1215 return blocksMap, nil 1216 } 1217 1218 func (c *loadBlocksAndTransfersCommand) startFetchingTransfersForLoadedBlocks(group *async.AtomicGroup) error { 1219 1220 log.Debug("fetchTransfers start", "chainID", c.chainClient.NetworkID(), "accounts", c.accounts) 1221 1222 blocksMap, err := c.getBlocksToLoad() 1223 if err != nil { 1224 return err 1225 } 1226 1227 go func() { 1228 txCommand := &loadTransfersCommand{ 1229 accounts: c.accounts, 1230 db: c.db, 1231 blockDAO: c.blockDAO, 1232 chainClient: c.chainClient, 1233 transactionManager: c.transactionManager, 1234 pendingTxManager: c.pendingTxManager, 1235 tokenManager: c.tokenManager, 1236 blocksByAddress: blocksMap, 1237 feed: c.feed, 1238 } 1239 1240 group.Add(txCommand.Command()) 1241 log.Debug("fetchTransfers end", "chainID", c.chainClient.NetworkID(), "accounts", c.accounts) 1242 }() 1243 1244 return nil 1245 } 1246 1247 func (c *loadBlocksAndTransfersCommand) notifyHistoryReady(account common.Address) { 1248 if c.feed != nil { 1249 c.feed.Send(walletevent.Event{ 1250 Type: EventRecentHistoryReady, 1251 Accounts: []common.Address{account}, 1252 ChainID: c.chainClient.NetworkID(), 1253 }) 1254 } 1255 } 1256 1257 func (c *loadBlocksAndTransfersCommand) areAllTransfersLoaded(account common.Address) (bool, error) { 1258 allBlocksLoaded, err := areAllHistoryBlocksLoadedForAddress(c.blockRangeDAO, c.chainClient.NetworkID(), account) 1259 if err != nil { 1260 log.Error("loadBlockAndTransfersCommand allHistoryBlocksLoaded", "error", err) 1261 return false, err 1262 } 1263 1264 if allBlocksLoaded { 1265 headers, err := c.blockDAO.GetBlocksToLoadByAddress(c.chainClient.NetworkID(), account, 1) 1266 if err != nil { 1267 log.Error("loadBlocksAndTransfersCommand GetFirstSavedBlock", "error", err) 1268 return false, err 1269 } 1270 1271 if len(headers) == 0 { 1272 return true, nil 1273 } 1274 } 1275 1276 return false, nil 1277 } 1278 1279 // TODO - make it a common method for every service that wants head block number, that will cache the latest block 1280 // and updates it on timeout 1281 func getHeadBlockNumber(parent context.Context, chainClient chain.ClientInterface) (*big.Int, error) { 1282 ctx, cancel := context.WithTimeout(parent, 3*time.Second) 1283 head, err := chainClient.HeaderByNumber(ctx, nil) 1284 cancel() 1285 if err != nil { 1286 log.Error("getHeadBlockNumber", "error", err) 1287 return nil, err 1288 } 1289 1290 return head.Number, err 1291 } 1292 1293 func nextRange(maxRangeSize int, prevFrom, zeroBlockNumber *big.Int) (*big.Int, *big.Int) { 1294 log.Debug("next range start", "from", prevFrom, "zeroBlockNumber", zeroBlockNumber) 1295 1296 rangeSize := big.NewInt(int64(maxRangeSize)) 1297 1298 to := big.NewInt(0).Set(prevFrom) 1299 from := big.NewInt(0).Sub(to, rangeSize) 1300 if from.Cmp(zeroBlockNumber) < 0 { 1301 from = new(big.Int).Set(zeroBlockNumber) 1302 } 1303 1304 log.Debug("next range end", "from", from, "to", to, "zeroBlockNumber", zeroBlockNumber) 1305 1306 return from, to 1307 } 1308 1309 func accountLimiterTag(account common.Address) string { 1310 return transferHistoryTag + "_" + account.String() 1311 } 1312 1313 func createChainClientWithLimiter(client chain.ClientInterface, account common.Address, limiter chain.RequestLimiter) (chain.ClientInterface, error) { 1314 // Each account has its own limit and a global limit for all accounts 1315 accountTag := accountLimiterTag(account) 1316 chainClient := chain.ClientWithTag(client, accountTag, transferHistoryTag) 1317 1318 // Check if limit is already reached, then skip the comamnd 1319 if allow, err := limiter.Allow(accountTag); !allow { 1320 log.Info("fetchHistoryBlocksForAccount limit reached", "account", account, "chain", chainClient.NetworkID(), "error", err) 1321 return nil, err 1322 } 1323 1324 if allow, err := limiter.Allow(transferHistoryTag); !allow { 1325 log.Info("fetchHistoryBlocksForAccount common limit reached", "chain", chainClient.NetworkID(), "error", err) 1326 return nil, err 1327 } 1328 1329 limit, _ := limiter.GetLimit(accountTag) 1330 if limit == nil { 1331 err := limiter.SetLimit(accountTag, transferHistoryLimitPerAccount, chain.LimitInfinitely) 1332 if err != nil { 1333 log.Error("fetchHistoryBlocksForAccount SetLimit", "error", err, "accountTag", accountTag) 1334 } 1335 } 1336 1337 // Here total limit per day is overwriten on each app start, that still saves us RPC calls, but allows to proceed 1338 // after app restart if the limit was reached. Currently there is no way to reset the limit from UI 1339 err := limiter.SetLimit(transferHistoryTag, transferHistoryLimit, transferHistoryLimitPeriod) 1340 if err != nil { 1341 log.Error("fetchHistoryBlocksForAccount SetLimit", "error", err, "groupTag", transferHistoryTag) 1342 } 1343 chainClient.SetLimiter(limiter) 1344 1345 return chainClient, nil 1346 }