github.com/status-im/status-go@v1.1.0/services/wallet/transfer/commands_sequential.go (about)

     1  package transfer
     2  
     3  import (
     4  	"context"
     5  	"math/big"
     6  	"sync/atomic"
     7  	"time"
     8  
     9  	"github.com/ethereum/go-ethereum/accounts/abi/bind"
    10  	"github.com/ethereum/go-ethereum/common"
    11  	"github.com/ethereum/go-ethereum/core/types"
    12  	"github.com/ethereum/go-ethereum/event"
    13  	"github.com/ethereum/go-ethereum/log"
    14  	"github.com/status-im/status-go/contracts"
    15  	nodetypes "github.com/status-im/status-go/eth-node/types"
    16  	"github.com/status-im/status-go/multiaccounts/accounts"
    17  	"github.com/status-im/status-go/rpc/chain"
    18  	"github.com/status-im/status-go/services/wallet/async"
    19  	"github.com/status-im/status-go/services/wallet/balance"
    20  	"github.com/status-im/status-go/services/wallet/blockchainstate"
    21  	"github.com/status-im/status-go/services/wallet/token"
    22  	"github.com/status-im/status-go/services/wallet/walletevent"
    23  	"github.com/status-im/status-go/transactions"
    24  )
    25  
    26  var findBlocksRetryInterval = 5 * time.Second
    27  
    28  const (
    29  	transferHistoryTag    = "transfer_history"
    30  	newTransferHistoryTag = "new_transfer_history"
    31  
    32  	transferHistoryLimit           = 10000
    33  	transferHistoryLimitPerAccount = 5000
    34  	transferHistoryLimitPeriod     = 24 * time.Hour
    35  )
    36  
    37  type nonceInfo struct {
    38  	nonce       *int64
    39  	blockNumber *big.Int
    40  }
    41  
    42  type findNewBlocksCommand struct {
    43  	*findBlocksCommand
    44  	contractMaker                *contracts.ContractMaker
    45  	iteration                    int
    46  	blockChainState              *blockchainstate.BlockChainState
    47  	lastNonces                   map[common.Address]nonceInfo
    48  	nonceCheckIntervalIterations int
    49  	logsCheckIntervalIterations  int
    50  }
    51  
    52  func (c *findNewBlocksCommand) Command() async.Command {
    53  	return async.InfiniteCommand{
    54  		Interval: 2 * time.Minute,
    55  		Runable:  c.Run,
    56  	}.Run
    57  }
    58  
    59  var requestTimeout = 20 * time.Second
    60  
    61  func (c *findNewBlocksCommand) detectTransfers(parent context.Context, accounts []common.Address) (*big.Int, []common.Address, error) {
    62  	bc, err := c.contractMaker.NewBalanceChecker(c.chainClient.NetworkID())
    63  	if err != nil {
    64  		log.Error("findNewBlocksCommand error creating balance checker", "error", err, "chain", c.chainClient.NetworkID())
    65  		return nil, nil, err
    66  	}
    67  
    68  	tokens, err := c.tokenManager.GetTokens(c.chainClient.NetworkID())
    69  	if err != nil {
    70  		return nil, nil, err
    71  	}
    72  	tokenAddresses := []common.Address{}
    73  	nilAddress := common.Address{}
    74  	for _, token := range tokens {
    75  		if token.Address != nilAddress {
    76  			tokenAddresses = append(tokenAddresses, token.Address)
    77  		}
    78  	}
    79  	log.Debug("findNewBlocksCommand detectTransfers", "cnt", len(tokenAddresses))
    80  
    81  	ctx, cancel := context.WithTimeout(parent, requestTimeout)
    82  	defer cancel()
    83  	blockNum, hashes, err := bc.BalancesHash(&bind.CallOpts{Context: ctx}, c.accounts, tokenAddresses)
    84  	if err != nil {
    85  		log.Error("findNewBlocksCommand can't get balances hashes", "error", err)
    86  		return nil, nil, err
    87  	}
    88  
    89  	addressesToCheck := []common.Address{}
    90  	for idx, account := range accounts {
    91  		blockRange, _, err := c.blockRangeDAO.getBlockRange(c.chainClient.NetworkID(), account)
    92  		if err != nil {
    93  			log.Error("findNewBlocksCommand can't get block range", "error", err, "account", account, "chain", c.chainClient.NetworkID())
    94  			return nil, nil, err
    95  		}
    96  
    97  		checkHash := common.BytesToHash(hashes[idx][:])
    98  		log.Debug("findNewBlocksCommand comparing hashes", "account", account, "network", c.chainClient.NetworkID(), "old hash", blockRange.balanceCheckHash, "new hash", checkHash.String())
    99  		if checkHash.String() != blockRange.balanceCheckHash {
   100  			addressesToCheck = append(addressesToCheck, account)
   101  		}
   102  
   103  		blockRange.balanceCheckHash = checkHash.String()
   104  
   105  		err = c.blockRangeDAO.upsertRange(c.chainClient.NetworkID(), account, blockRange)
   106  		if err != nil {
   107  			log.Error("findNewBlocksCommand can't update balance check", "error", err, "account", account, "chain", c.chainClient.NetworkID())
   108  			return nil, nil, err
   109  		}
   110  	}
   111  
   112  	return blockNum, addressesToCheck, nil
   113  }
   114  
   115  func (c *findNewBlocksCommand) detectNonceChange(parent context.Context, to *big.Int, accounts []common.Address) (map[common.Address]*big.Int, error) {
   116  	addressesWithChange := map[common.Address]*big.Int{}
   117  	for _, account := range accounts {
   118  		var oldNonce *int64
   119  
   120  		blockRange, _, err := c.blockRangeDAO.getBlockRange(c.chainClient.NetworkID(), account)
   121  		if err != nil {
   122  			log.Error("findNewBlocksCommand can't get block range", "error", err, "account", account, "chain", c.chainClient.NetworkID())
   123  			return nil, err
   124  		}
   125  
   126  		lastNonceInfo, ok := c.lastNonces[account]
   127  		if !ok || lastNonceInfo.blockNumber.Cmp(blockRange.eth.LastKnown) != 0 {
   128  			log.Debug("Fetching old nonce", "at", blockRange.eth.LastKnown, "acc", account)
   129  			if blockRange.eth.LastKnown == nil {
   130  				blockRange.eth.LastKnown = big.NewInt(0)
   131  				oldNonce = new(int64) // At 0 block nonce is 0
   132  			} else {
   133  				oldNonce, err = c.balanceCacher.NonceAt(parent, c.chainClient, account, blockRange.eth.LastKnown)
   134  				if err != nil {
   135  					log.Error("findNewBlocksCommand can't get nonce", "error", err, "account", account, "chain", c.chainClient.NetworkID())
   136  					return nil, err
   137  				}
   138  			}
   139  		} else {
   140  			oldNonce = lastNonceInfo.nonce
   141  		}
   142  
   143  		newNonce, err := c.balanceCacher.NonceAt(parent, c.chainClient, account, to)
   144  		if err != nil {
   145  			log.Error("findNewBlocksCommand can't get nonce", "error", err, "account", account, "chain", c.chainClient.NetworkID())
   146  			return nil, err
   147  		}
   148  
   149  		log.Debug("Comparing nonces", "oldNonce", *oldNonce, "newNonce", *newNonce, "to", to, "acc", account)
   150  
   151  		if *newNonce != *oldNonce {
   152  			addressesWithChange[account] = blockRange.eth.LastKnown
   153  		}
   154  
   155  		if c.lastNonces == nil {
   156  			c.lastNonces = map[common.Address]nonceInfo{}
   157  		}
   158  
   159  		c.lastNonces[account] = nonceInfo{
   160  			nonce:       newNonce,
   161  			blockNumber: to,
   162  		}
   163  	}
   164  
   165  	return addressesWithChange, nil
   166  }
   167  
   168  var nonceCheckIntervalIterations = 30
   169  var logsCheckIntervalIterations = 5
   170  
   171  func (c *findNewBlocksCommand) Run(parent context.Context) error {
   172  	mnemonicWasNotShown, err := c.accountsDB.GetMnemonicWasNotShown()
   173  	if err != nil {
   174  		return err
   175  	}
   176  
   177  	accountsToCheck := []common.Address{}
   178  	// accounts which might have outgoing transfers initiated outside
   179  	// the application, e.g. watch only or restored from mnemonic phrase
   180  	accountsWithOutsideTransfers := []common.Address{}
   181  
   182  	for _, account := range c.accounts {
   183  		acc, err := c.accountsDB.GetAccountByAddress(nodetypes.Address(account))
   184  		if err != nil {
   185  			return err
   186  		}
   187  		if mnemonicWasNotShown {
   188  			if acc.AddressWasNotShown {
   189  				log.Info("skip findNewBlocksCommand, mnemonic has not been shown and the address has not been shared yet", "address", account)
   190  				continue
   191  			}
   192  		}
   193  		if !mnemonicWasNotShown || acc.Type != accounts.AccountTypeGenerated {
   194  			accountsWithOutsideTransfers = append(accountsWithOutsideTransfers, account)
   195  		}
   196  
   197  		accountsToCheck = append(accountsToCheck, account)
   198  	}
   199  
   200  	if len(accountsToCheck) == 0 {
   201  		return nil
   202  	}
   203  
   204  	headNum, accountsWithDetectedChanges, err := c.detectTransfers(parent, accountsToCheck)
   205  	if err != nil {
   206  		log.Error("findNewBlocksCommand error on transfer detection", "error", err, "chain", c.chainClient.NetworkID())
   207  		return err
   208  	}
   209  
   210  	c.blockChainState.SetLastBlockNumber(c.chainClient.NetworkID(), headNum.Uint64())
   211  
   212  	if len(accountsWithDetectedChanges) != 0 {
   213  		log.Debug("findNewBlocksCommand detected accounts with changes, proceeding", "accounts", accountsWithDetectedChanges, "from", c.fromBlockNumber)
   214  		err = c.findAndSaveEthBlocks(parent, c.fromBlockNumber, headNum, accountsToCheck)
   215  		if err != nil {
   216  			return err
   217  		}
   218  	} else if c.iteration%c.nonceCheckIntervalIterations == 0 && len(accountsWithOutsideTransfers) > 0 {
   219  		log.Debug("findNewBlocksCommand nonce check", "accounts", accountsWithOutsideTransfers)
   220  		accountsWithNonceChanges, err := c.detectNonceChange(parent, headNum, accountsWithOutsideTransfers)
   221  		if err != nil {
   222  			return err
   223  		}
   224  
   225  		if len(accountsWithNonceChanges) > 0 {
   226  			log.Debug("findNewBlocksCommand detected nonce diff", "accounts", accountsWithNonceChanges)
   227  			for account, from := range accountsWithNonceChanges {
   228  				err = c.findAndSaveEthBlocks(parent, from, headNum, []common.Address{account})
   229  				if err != nil {
   230  					return err
   231  				}
   232  			}
   233  		}
   234  
   235  		for _, account := range accountsToCheck {
   236  			if _, ok := accountsWithNonceChanges[account]; ok {
   237  				continue
   238  			}
   239  			err := c.markEthBlockRangeChecked(account, &BlockRange{nil, c.fromBlockNumber, headNum})
   240  			if err != nil {
   241  				return err
   242  			}
   243  		}
   244  	}
   245  
   246  	if len(accountsWithDetectedChanges) != 0 || c.iteration%c.logsCheckIntervalIterations == 0 {
   247  		from := c.fromBlockNumber
   248  		if c.logsCheckLastKnownBlock != nil {
   249  			from = c.logsCheckLastKnownBlock
   250  		}
   251  		err = c.findAndSaveTokenBlocks(parent, from, headNum)
   252  		if err != nil {
   253  			return err
   254  		}
   255  		c.logsCheckLastKnownBlock = headNum
   256  	}
   257  	c.fromBlockNumber = headNum
   258  	c.iteration++
   259  
   260  	return nil
   261  }
   262  
   263  func (c *findNewBlocksCommand) findAndSaveEthBlocks(parent context.Context, fromNum, headNum *big.Int, accounts []common.Address) error {
   264  	// Check ETH transfers for each account independently
   265  	mnemonicWasNotShown, err := c.accountsDB.GetMnemonicWasNotShown()
   266  	if err != nil {
   267  		return err
   268  	}
   269  
   270  	for _, account := range accounts {
   271  		if mnemonicWasNotShown {
   272  			acc, err := c.accountsDB.GetAccountByAddress(nodetypes.Address(account))
   273  			if err != nil {
   274  				return err
   275  			}
   276  			if acc.AddressWasNotShown {
   277  				log.Info("skip findNewBlocksCommand, mnemonic has not been shown and the address has not been shared yet", "address", account)
   278  				continue
   279  			}
   280  		}
   281  
   282  		log.Debug("start findNewBlocksCommand", "account", account, "chain", c.chainClient.NetworkID(), "noLimit", c.noLimit, "from", fromNum, "to", headNum)
   283  
   284  		headers, startBlockNum, err := c.findBlocksWithEthTransfers(parent, account, fromNum, headNum)
   285  		if err != nil {
   286  			return err
   287  		}
   288  
   289  		if len(headers) > 0 {
   290  			log.Debug("findNewBlocksCommand saving headers", "len", len(headers), "lastBlockNumber", headNum,
   291  				"balance", c.balanceCacher.Cache().GetBalance(account, c.chainClient.NetworkID(), headNum),
   292  				"nonce", c.balanceCacher.Cache().GetNonce(account, c.chainClient.NetworkID(), headNum))
   293  
   294  			err := c.db.SaveBlocks(c.chainClient.NetworkID(), headers)
   295  			if err != nil {
   296  				return err
   297  			}
   298  
   299  			c.blocksFound(headers)
   300  		}
   301  
   302  		err = c.markEthBlockRangeChecked(account, &BlockRange{startBlockNum, fromNum, headNum})
   303  		if err != nil {
   304  			return err
   305  		}
   306  
   307  		log.Debug("end findNewBlocksCommand", "account", account, "chain", c.chainClient.NetworkID(), "noLimit", c.noLimit, "from", fromNum, "to", headNum)
   308  	}
   309  
   310  	return nil
   311  }
   312  
   313  func (c *findNewBlocksCommand) findAndSaveTokenBlocks(parent context.Context, fromNum, headNum *big.Int) error {
   314  	// Check token transfers for all accounts.
   315  	// Each account's last checked block can be different, so we can get duplicated headers,
   316  	// so we need to deduplicate them
   317  	const incomingOnly = false
   318  	erc20Headers, err := c.fastIndexErc20(parent, fromNum, headNum, incomingOnly)
   319  	if err != nil {
   320  		log.Error("findNewBlocksCommand fastIndexErc20", "err", err, "account", c.accounts, "chain", c.chainClient.NetworkID())
   321  		return err
   322  	}
   323  
   324  	if len(erc20Headers) > 0 {
   325  		log.Debug("findNewBlocksCommand saving headers", "len", len(erc20Headers), "from", fromNum, "to", headNum)
   326  
   327  		// get not loaded headers from DB for all accs and blocks
   328  		preLoadedTransactions, err := c.db.GetTransactionsToLoad(c.chainClient.NetworkID(), common.Address{}, nil)
   329  		if err != nil {
   330  			return err
   331  		}
   332  
   333  		tokenBlocksFiltered := filterNewPreloadedTransactions(erc20Headers, preLoadedTransactions)
   334  
   335  		err = c.db.SaveBlocks(c.chainClient.NetworkID(), tokenBlocksFiltered)
   336  		if err != nil {
   337  			return err
   338  		}
   339  
   340  		c.blocksFound(tokenBlocksFiltered)
   341  	}
   342  
   343  	return c.markTokenBlockRangeChecked(c.accounts, fromNum, headNum)
   344  }
   345  
   346  func (c *findBlocksCommand) markTokenBlockRangeChecked(accounts []common.Address, from, to *big.Int) error {
   347  	log.Debug("markTokenBlockRangeChecked", "chain", c.chainClient.NetworkID(), "from", from.Uint64(), "to", to.Uint64())
   348  
   349  	for _, account := range accounts {
   350  		err := c.blockRangeDAO.updateTokenRange(c.chainClient.NetworkID(), account, &BlockRange{FirstKnown: from, LastKnown: to})
   351  		if err != nil {
   352  			log.Error("findNewBlocksCommand upsertTokenRange", "error", err)
   353  			return err
   354  		}
   355  	}
   356  
   357  	return nil
   358  }
   359  
   360  func filterNewPreloadedTransactions(erc20Headers []*DBHeader, preLoadedTransfers []*PreloadedTransaction) []*DBHeader {
   361  	var uniqueErc20Headers []*DBHeader
   362  	for _, header := range erc20Headers {
   363  		loaded := false
   364  		for _, transfer := range preLoadedTransfers {
   365  			if header.PreloadedTransactions[0].ID == transfer.ID {
   366  				loaded = true
   367  				break
   368  			}
   369  		}
   370  
   371  		if !loaded {
   372  			uniqueErc20Headers = append(uniqueErc20Headers, header)
   373  		}
   374  	}
   375  
   376  	return uniqueErc20Headers
   377  }
   378  
   379  func (c *findNewBlocksCommand) findBlocksWithEthTransfers(parent context.Context, account common.Address, fromOrig, toOrig *big.Int) (headers []*DBHeader, startBlockNum *big.Int, err error) {
   380  	log.Debug("start findNewBlocksCommand::findBlocksWithEthTransfers", "account", account, "chain", c.chainClient.NetworkID(), "noLimit", c.noLimit, "from", c.fromBlockNumber, "to", c.toBlockNumber)
   381  
   382  	rangeSize := big.NewInt(int64(c.defaultNodeBlockChunkSize))
   383  
   384  	from, to := new(big.Int).Set(fromOrig), new(big.Int).Set(toOrig)
   385  
   386  	// Limit the range size to DefaultNodeBlockChunkSize
   387  	if new(big.Int).Sub(to, from).Cmp(rangeSize) > 0 {
   388  		from.Sub(to, rangeSize)
   389  	}
   390  
   391  	for {
   392  		if from.Cmp(to) == 0 {
   393  			log.Debug("findNewBlocksCommand empty range", "from", from, "to", to)
   394  			break
   395  		}
   396  
   397  		fromBlock := &Block{Number: from}
   398  
   399  		var newFromBlock *Block
   400  		var ethHeaders []*DBHeader
   401  		newFromBlock, ethHeaders, startBlockNum, err = c.fastIndex(parent, account, c.balanceCacher, fromBlock, to)
   402  		if err != nil {
   403  			log.Error("findNewBlocksCommand checkRange fastIndex", "err", err, "account", account,
   404  				"chain", c.chainClient.NetworkID())
   405  			return nil, nil, err
   406  		}
   407  		log.Debug("findNewBlocksCommand checkRange", "chainID", c.chainClient.NetworkID(), "account", account,
   408  			"startBlock", startBlockNum, "newFromBlock", newFromBlock.Number, "toBlockNumber", to, "noLimit", c.noLimit)
   409  
   410  		headers = append(headers, ethHeaders...)
   411  
   412  		if startBlockNum != nil && startBlockNum.Cmp(from) >= 0 {
   413  			log.Debug("Checked all ranges, stop execution", "startBlock", startBlockNum, "from", from, "to", to)
   414  			break
   415  		}
   416  
   417  		nextFrom, nextTo := nextRange(c.defaultNodeBlockChunkSize, newFromBlock.Number, fromOrig)
   418  
   419  		if nextFrom.Cmp(from) == 0 && nextTo.Cmp(to) == 0 {
   420  			log.Debug("findNewBlocksCommand empty next range", "from", from, "to", to)
   421  			break
   422  		}
   423  
   424  		from = nextFrom
   425  		to = nextTo
   426  	}
   427  
   428  	log.Debug("end findNewBlocksCommand::findBlocksWithEthTransfers", "account", account, "chain", c.chainClient.NetworkID(), "noLimit", c.noLimit)
   429  
   430  	return headers, startBlockNum, nil
   431  }
   432  
   433  // TODO NewFindBlocksCommand
   434  type findBlocksCommand struct {
   435  	accounts                  []common.Address
   436  	db                        *Database
   437  	accountsDB                *accounts.Database
   438  	blockRangeDAO             BlockRangeDAOer
   439  	chainClient               chain.ClientInterface
   440  	balanceCacher             balance.Cacher
   441  	feed                      *event.Feed
   442  	noLimit                   bool
   443  	tokenManager              *token.Manager
   444  	fromBlockNumber           *big.Int
   445  	logsCheckLastKnownBlock   *big.Int
   446  	toBlockNumber             *big.Int
   447  	blocksLoadedCh            chan<- []*DBHeader
   448  	defaultNodeBlockChunkSize int
   449  
   450  	// Not to be set by the caller
   451  	resFromBlock           *Block
   452  	startBlockNumber       *big.Int
   453  	reachedETHHistoryStart bool
   454  }
   455  
   456  func (c *findBlocksCommand) Runner(interval ...time.Duration) async.Runner {
   457  	intvl := findBlocksRetryInterval
   458  	if len(interval) > 0 {
   459  		intvl = interval[0]
   460  	}
   461  	return async.FiniteCommandWithErrorCounter{
   462  		FiniteCommand: async.FiniteCommand{
   463  			Interval: intvl,
   464  			Runable:  c.Run,
   465  		},
   466  		ErrorCounter: async.NewErrorCounter(3, "findBlocksCommand"),
   467  	}
   468  }
   469  
   470  func (c *findBlocksCommand) Command(interval ...time.Duration) async.Command {
   471  	return c.Runner(interval...).Run
   472  }
   473  
   474  type ERC20BlockRange struct {
   475  	from *big.Int
   476  	to   *big.Int
   477  }
   478  
   479  func (c *findBlocksCommand) ERC20ScanByBalance(parent context.Context, account common.Address, fromBlock, toBlock *big.Int, token common.Address) ([]ERC20BlockRange, error) {
   480  	var err error
   481  	batchSize := getErc20BatchSize(c.chainClient.NetworkID())
   482  	ranges := [][]*big.Int{{fromBlock, toBlock}}
   483  	foundRanges := []ERC20BlockRange{}
   484  	cache := map[int64]*big.Int{}
   485  	for {
   486  		nextRanges := [][]*big.Int{}
   487  		for _, blockRange := range ranges {
   488  			from, to := blockRange[0], blockRange[1]
   489  			fromBalance, ok := cache[from.Int64()]
   490  			if !ok {
   491  				fromBalance, err = c.tokenManager.GetTokenBalanceAt(parent, c.chainClient, account, token, from)
   492  				if err != nil {
   493  					return nil, err
   494  				}
   495  
   496  				if fromBalance == nil {
   497  					fromBalance = big.NewInt(0)
   498  				}
   499  				cache[from.Int64()] = fromBalance
   500  			}
   501  
   502  			toBalance, ok := cache[to.Int64()]
   503  			if !ok {
   504  				toBalance, err = c.tokenManager.GetTokenBalanceAt(parent, c.chainClient, account, token, to)
   505  				if err != nil {
   506  					return nil, err
   507  				}
   508  				if toBalance == nil {
   509  					toBalance = big.NewInt(0)
   510  				}
   511  				cache[to.Int64()] = toBalance
   512  			}
   513  
   514  			if fromBalance.Cmp(toBalance) != 0 {
   515  				diff := new(big.Int).Sub(to, from)
   516  				if diff.Cmp(batchSize) <= 0 {
   517  					foundRanges = append(foundRanges, ERC20BlockRange{from, to})
   518  					continue
   519  				}
   520  
   521  				halfOfDiff := new(big.Int).Div(diff, big.NewInt(2))
   522  				mid := new(big.Int).Add(from, halfOfDiff)
   523  
   524  				nextRanges = append(nextRanges, []*big.Int{from, mid})
   525  				nextRanges = append(nextRanges, []*big.Int{mid, to})
   526  			}
   527  		}
   528  
   529  		if len(nextRanges) == 0 {
   530  			break
   531  		}
   532  
   533  		ranges = nextRanges
   534  	}
   535  
   536  	return foundRanges, nil
   537  }
   538  
   539  func (c *findBlocksCommand) checkERC20Tail(parent context.Context, account common.Address) ([]*DBHeader, error) {
   540  	log.Debug("checkERC20Tail", "account", account, "to block", c.startBlockNumber, "from", c.resFromBlock.Number)
   541  	tokens, err := c.tokenManager.GetTokens(c.chainClient.NetworkID())
   542  	if err != nil {
   543  		return nil, err
   544  	}
   545  	addresses := make([]common.Address, len(tokens))
   546  	for i, token := range tokens {
   547  		addresses[i] = token.Address
   548  	}
   549  
   550  	from := new(big.Int).Sub(c.resFromBlock.Number, big.NewInt(1))
   551  
   552  	clients := make(map[uint64]chain.ClientInterface, 1)
   553  	clients[c.chainClient.NetworkID()] = c.chainClient
   554  	atBlocks := make(map[uint64]*big.Int, 1)
   555  	atBlocks[c.chainClient.NetworkID()] = from
   556  	balances, err := c.tokenManager.GetBalancesAtByChain(parent, clients, []common.Address{account}, addresses, atBlocks)
   557  	if err != nil {
   558  		return nil, err
   559  	}
   560  
   561  	foundRanges := []ERC20BlockRange{}
   562  	for token, balance := range balances[c.chainClient.NetworkID()][account] {
   563  		bigintBalance := big.NewInt(balance.ToInt().Int64())
   564  		if bigintBalance.Cmp(big.NewInt(0)) <= 0 {
   565  			continue
   566  		}
   567  		result, err := c.ERC20ScanByBalance(parent, account, big.NewInt(0), from, token)
   568  		if err != nil {
   569  			return nil, err
   570  		}
   571  
   572  		foundRanges = append(foundRanges, result...)
   573  	}
   574  
   575  	uniqRanges := []ERC20BlockRange{}
   576  	rangesMap := map[string]bool{}
   577  	for _, rangeItem := range foundRanges {
   578  		key := rangeItem.from.String() + "-" + rangeItem.to.String()
   579  		if _, ok := rangesMap[key]; !ok {
   580  			rangesMap[key] = true
   581  			uniqRanges = append(uniqRanges, rangeItem)
   582  		}
   583  	}
   584  
   585  	foundHeaders := []*DBHeader{}
   586  	for _, rangeItem := range uniqRanges {
   587  		headers, err := c.fastIndexErc20(parent, rangeItem.from, rangeItem.to, true)
   588  		if err != nil {
   589  			return nil, err
   590  		}
   591  		foundHeaders = append(foundHeaders, headers...)
   592  	}
   593  
   594  	return foundHeaders, nil
   595  }
   596  
   597  func (c *findBlocksCommand) Run(parent context.Context) (err error) {
   598  	log.Debug("start findBlocksCommand", "accounts", c.accounts, "chain", c.chainClient.NetworkID(), "noLimit", c.noLimit, "from", c.fromBlockNumber, "to", c.toBlockNumber)
   599  
   600  	account := c.accounts[0] // For now this command supports only 1 account
   601  	mnemonicWasNotShown, err := c.accountsDB.GetMnemonicWasNotShown()
   602  	if err != nil {
   603  		return err
   604  	}
   605  
   606  	if mnemonicWasNotShown {
   607  		account, err := c.accountsDB.GetAccountByAddress(nodetypes.BytesToAddress(account.Bytes()))
   608  		if err != nil {
   609  			return err
   610  		}
   611  		if account.AddressWasNotShown {
   612  			log.Info("skip findBlocksCommand, mnemonic has not been shown and the address has not been shared yet", "address", account)
   613  			return nil
   614  		}
   615  	}
   616  
   617  	rangeSize := big.NewInt(int64(c.defaultNodeBlockChunkSize))
   618  	from, to := new(big.Int).Set(c.fromBlockNumber), new(big.Int).Set(c.toBlockNumber)
   619  
   620  	// Limit the range size to DefaultNodeBlockChunkSize
   621  	if new(big.Int).Sub(to, from).Cmp(rangeSize) > 0 {
   622  		from.Sub(to, rangeSize)
   623  	}
   624  
   625  	for {
   626  		if from.Cmp(to) == 0 {
   627  			log.Debug("findBlocksCommand empty range", "from", from, "to", to)
   628  			break
   629  		}
   630  
   631  		var headers []*DBHeader
   632  		if c.reachedETHHistoryStart {
   633  			if c.fromBlockNumber.Cmp(zero) == 0 && c.startBlockNumber != nil && c.startBlockNumber.Cmp(zero) == 1 {
   634  				headers, err = c.checkERC20Tail(parent, account)
   635  				if err != nil {
   636  					log.Error("findBlocksCommand checkERC20Tail", "err", err, "account", account, "chain", c.chainClient.NetworkID())
   637  					break
   638  				}
   639  			}
   640  		} else {
   641  			headers, err = c.checkRange(parent, from, to)
   642  			if err != nil {
   643  				break
   644  			}
   645  		}
   646  
   647  		if len(headers) > 0 {
   648  			log.Debug("findBlocksCommand saving headers", "len", len(headers), "lastBlockNumber", to,
   649  				"balance", c.balanceCacher.Cache().GetBalance(account, c.chainClient.NetworkID(), to),
   650  				"nonce", c.balanceCacher.Cache().GetNonce(account, c.chainClient.NetworkID(), to))
   651  
   652  			err = c.db.SaveBlocks(c.chainClient.NetworkID(), headers)
   653  			if err != nil {
   654  				break
   655  			}
   656  
   657  			c.blocksFound(headers)
   658  		}
   659  
   660  		if c.reachedETHHistoryStart {
   661  			err = c.markTokenBlockRangeChecked([]common.Address{account}, big.NewInt(0), to)
   662  			if err != nil {
   663  				break
   664  			}
   665  			log.Debug("findBlocksCommand reached first ETH transfer and checked erc20 tail", "chain", c.chainClient.NetworkID(), "account", account)
   666  			break
   667  		}
   668  
   669  		err = c.markEthBlockRangeChecked(account, &BlockRange{c.startBlockNumber, c.resFromBlock.Number, to})
   670  		if err != nil {
   671  			break
   672  		}
   673  
   674  		err = c.markTokenBlockRangeChecked([]common.Address{account}, c.resFromBlock.Number, to)
   675  		if err != nil {
   676  			break
   677  		}
   678  
   679  		// if we have found first ETH block and we have not reached the start of ETH history yet
   680  		if c.startBlockNumber != nil && c.fromBlockNumber.Cmp(from) == -1 {
   681  			log.Debug("ERC20 tail should be checked", "initial from", c.fromBlockNumber, "actual from", from, "first ETH block", c.startBlockNumber)
   682  			c.reachedETHHistoryStart = true
   683  			continue
   684  		}
   685  
   686  		if c.startBlockNumber != nil && c.startBlockNumber.Cmp(from) >= 0 {
   687  			log.Debug("Checked all ranges, stop execution", "startBlock", c.startBlockNumber, "from", from, "to", to)
   688  			break
   689  		}
   690  
   691  		nextFrom, nextTo := nextRange(c.defaultNodeBlockChunkSize, c.resFromBlock.Number, c.fromBlockNumber)
   692  
   693  		if nextFrom.Cmp(from) == 0 && nextTo.Cmp(to) == 0 {
   694  			log.Debug("findBlocksCommand empty next range", "from", from, "to", to)
   695  			break
   696  		}
   697  
   698  		from = nextFrom
   699  		to = nextTo
   700  	}
   701  
   702  	log.Debug("end findBlocksCommand", "account", account, "chain", c.chainClient.NetworkID(), "noLimit", c.noLimit, "err", err)
   703  
   704  	return err
   705  }
   706  
   707  func (c *findBlocksCommand) blocksFound(headers []*DBHeader) {
   708  	c.blocksLoadedCh <- headers
   709  }
   710  
   711  func (c *findBlocksCommand) markEthBlockRangeChecked(account common.Address, blockRange *BlockRange) error {
   712  	log.Debug("upsert block range", "Start", blockRange.Start, "FirstKnown", blockRange.FirstKnown, "LastKnown", blockRange.LastKnown,
   713  		"chain", c.chainClient.NetworkID(), "account", account)
   714  
   715  	err := c.blockRangeDAO.upsertEthRange(c.chainClient.NetworkID(), account, blockRange)
   716  	if err != nil {
   717  		log.Error("findBlocksCommand upsertRange", "error", err)
   718  		return err
   719  	}
   720  
   721  	return nil
   722  }
   723  
   724  func (c *findBlocksCommand) checkRange(parent context.Context, from *big.Int, to *big.Int) (
   725  	foundHeaders []*DBHeader, err error) {
   726  
   727  	account := c.accounts[0]
   728  	fromBlock := &Block{Number: from}
   729  
   730  	newFromBlock, ethHeaders, startBlock, err := c.fastIndex(parent, account, c.balanceCacher, fromBlock, to)
   731  	if err != nil {
   732  		log.Error("findBlocksCommand checkRange fastIndex", "err", err, "account", account,
   733  			"chain", c.chainClient.NetworkID())
   734  		return nil, err
   735  	}
   736  	log.Debug("findBlocksCommand checkRange", "chainID", c.chainClient.NetworkID(), "account", account,
   737  		"startBlock", startBlock, "newFromBlock", newFromBlock.Number, "toBlockNumber", to, "noLimit", c.noLimit)
   738  
   739  	// There could be incoming ERC20 transfers which don't change the balance
   740  	// and nonce of ETH account, so we keep looking for them
   741  	erc20Headers, err := c.fastIndexErc20(parent, newFromBlock.Number, to, false)
   742  	if err != nil {
   743  		log.Error("findBlocksCommand checkRange fastIndexErc20", "err", err, "account", account, "chain", c.chainClient.NetworkID())
   744  		return nil, err
   745  	}
   746  
   747  	allHeaders := append(ethHeaders, erc20Headers...)
   748  
   749  	if len(allHeaders) > 0 {
   750  		foundHeaders = uniqueHeaderPerBlockHash(allHeaders)
   751  	}
   752  
   753  	c.resFromBlock = newFromBlock
   754  	c.startBlockNumber = startBlock
   755  
   756  	log.Debug("end findBlocksCommand checkRange", "chainID", c.chainClient.NetworkID(), "account", account,
   757  		"c.startBlock", c.startBlockNumber, "newFromBlock", newFromBlock.Number,
   758  		"toBlockNumber", to, "c.resFromBlock", c.resFromBlock.Number)
   759  
   760  	return
   761  }
   762  
   763  func loadBlockRangeInfo(chainID uint64, account common.Address, blockDAO BlockRangeDAOer) (
   764  	*ethTokensBlockRanges, error) {
   765  
   766  	blockRange, _, err := blockDAO.getBlockRange(chainID, account)
   767  	if err != nil {
   768  		log.Error("failed to load block ranges from database", "chain", chainID, "account", account,
   769  			"error", err)
   770  		return nil, err
   771  	}
   772  
   773  	return blockRange, nil
   774  }
   775  
   776  // Returns if all blocks are loaded, which means that start block (beginning of account history)
   777  // has been found and all block headers saved to the DB
   778  func areAllHistoryBlocksLoaded(blockInfo *BlockRange) bool {
   779  	if blockInfo != nil && blockInfo.FirstKnown != nil &&
   780  		((blockInfo.Start != nil && blockInfo.Start.Cmp(blockInfo.FirstKnown) >= 0) ||
   781  			blockInfo.FirstKnown.Cmp(zero) == 0) {
   782  		return true
   783  	}
   784  
   785  	return false
   786  }
   787  
   788  func areAllHistoryBlocksLoadedForAddress(blockRangeDAO BlockRangeDAOer, chainID uint64,
   789  	address common.Address) (bool, error) {
   790  
   791  	blockRange, _, err := blockRangeDAO.getBlockRange(chainID, address)
   792  	if err != nil {
   793  		log.Error("findBlocksCommand getBlockRange", "error", err)
   794  		return false, err
   795  	}
   796  
   797  	return areAllHistoryBlocksLoaded(blockRange.eth) && areAllHistoryBlocksLoaded(blockRange.tokens), nil
   798  }
   799  
   800  // run fast indexing for every accont up to canonical chain head minus safety depth.
   801  // every account will run it from last synced header.
   802  func (c *findBlocksCommand) fastIndex(ctx context.Context, account common.Address, bCacher balance.Cacher,
   803  	fromBlock *Block, toBlockNumber *big.Int) (resultingFrom *Block, headers []*DBHeader,
   804  	startBlock *big.Int, err error) {
   805  
   806  	log.Debug("fast index started", "chainID", c.chainClient.NetworkID(), "account", account,
   807  		"from", fromBlock.Number, "to", toBlockNumber)
   808  
   809  	start := time.Now()
   810  	group := async.NewGroup(ctx)
   811  
   812  	command := &ethHistoricalCommand{
   813  		chainClient:   c.chainClient,
   814  		balanceCacher: bCacher,
   815  		address:       account,
   816  		feed:          c.feed,
   817  		from:          fromBlock,
   818  		to:            toBlockNumber,
   819  		noLimit:       c.noLimit,
   820  		threadLimit:   SequentialThreadLimit,
   821  	}
   822  	group.Add(command.Command())
   823  
   824  	select {
   825  	case <-ctx.Done():
   826  		err = ctx.Err()
   827  		log.Debug("fast indexer ctx Done", "error", err)
   828  		return
   829  	case <-group.WaitAsync():
   830  		if command.error != nil {
   831  			err = command.error
   832  			return
   833  		}
   834  		resultingFrom = &Block{Number: command.resultingFrom}
   835  		headers = command.foundHeaders
   836  		startBlock = command.startBlock
   837  		log.Debug("fast indexer finished", "chainID", c.chainClient.NetworkID(), "account", account, "in", time.Since(start),
   838  			"startBlock", command.startBlock, "resultingFrom", resultingFrom.Number, "headers", len(headers))
   839  		return
   840  	}
   841  }
   842  
   843  // run fast indexing for every accont up to canonical chain head minus safety depth.
   844  // every account will run it from last synced header.
   845  func (c *findBlocksCommand) fastIndexErc20(ctx context.Context, fromBlockNumber *big.Int,
   846  	toBlockNumber *big.Int, incomingOnly bool) ([]*DBHeader, error) {
   847  
   848  	start := time.Now()
   849  	group := async.NewGroup(ctx)
   850  
   851  	erc20 := &erc20HistoricalCommand{
   852  		erc20:        NewERC20TransfersDownloader(c.chainClient, c.accounts, types.LatestSignerForChainID(c.chainClient.ToBigInt()), incomingOnly),
   853  		chainClient:  c.chainClient,
   854  		feed:         c.feed,
   855  		from:         fromBlockNumber,
   856  		to:           toBlockNumber,
   857  		foundHeaders: []*DBHeader{},
   858  	}
   859  	group.Add(erc20.Command())
   860  
   861  	select {
   862  	case <-ctx.Done():
   863  		return nil, ctx.Err()
   864  	case <-group.WaitAsync():
   865  		headers := erc20.foundHeaders
   866  		log.Debug("fast indexer Erc20 finished", "chainID", c.chainClient.NetworkID(),
   867  			"in", time.Since(start), "headers", len(headers))
   868  		return headers, nil
   869  	}
   870  }
   871  
   872  // Start transfers loop to load transfers for new blocks
   873  func (c *loadBlocksAndTransfersCommand) startTransfersLoop(ctx context.Context) {
   874  	c.incLoops()
   875  	go func() {
   876  		defer func() {
   877  			c.decLoops()
   878  		}()
   879  
   880  		log.Debug("loadTransfersLoop start", "chain", c.chainClient.NetworkID())
   881  
   882  		for {
   883  			select {
   884  			case <-ctx.Done():
   885  				log.Debug("startTransfersLoop done", "chain", c.chainClient.NetworkID(), "error", ctx.Err())
   886  				return
   887  			case dbHeaders := <-c.blocksLoadedCh:
   888  				log.Debug("loadTransfersOnDemand transfers received", "chain", c.chainClient.NetworkID(), "headers", len(dbHeaders))
   889  
   890  				blocksByAddress := map[common.Address][]*big.Int{}
   891  				// iterate over headers and group them by address
   892  				for _, dbHeader := range dbHeaders {
   893  					blocksByAddress[dbHeader.Address] = append(blocksByAddress[dbHeader.Address], dbHeader.Number)
   894  				}
   895  
   896  				go func() {
   897  					_ = loadTransfers(ctx, c.blockDAO, c.db, c.chainClient, noBlockLimit,
   898  						blocksByAddress, c.transactionManager, c.pendingTxManager, c.tokenManager, c.feed)
   899  				}()
   900  			}
   901  		}
   902  	}()
   903  }
   904  
   905  func newLoadBlocksAndTransfersCommand(accounts []common.Address, db *Database, accountsDB *accounts.Database,
   906  	blockDAO *BlockDAO, blockRangesSeqDAO BlockRangeDAOer, chainClient chain.ClientInterface, feed *event.Feed,
   907  	transactionManager *TransactionManager, pendingTxManager *transactions.PendingTxTracker,
   908  	tokenManager *token.Manager, balanceCacher balance.Cacher, omitHistory bool,
   909  	blockChainState *blockchainstate.BlockChainState) *loadBlocksAndTransfersCommand {
   910  
   911  	return &loadBlocksAndTransfersCommand{
   912  		accounts:           accounts,
   913  		db:                 db,
   914  		blockRangeDAO:      blockRangesSeqDAO,
   915  		accountsDB:         accountsDB,
   916  		blockDAO:           blockDAO,
   917  		chainClient:        chainClient,
   918  		feed:               feed,
   919  		balanceCacher:      balanceCacher,
   920  		transactionManager: transactionManager,
   921  		pendingTxManager:   pendingTxManager,
   922  		tokenManager:       tokenManager,
   923  		blocksLoadedCh:     make(chan []*DBHeader, 100),
   924  		omitHistory:        omitHistory,
   925  		contractMaker:      tokenManager.ContractMaker,
   926  		blockChainState:    blockChainState,
   927  	}
   928  }
   929  
   930  type loadBlocksAndTransfersCommand struct {
   931  	accounts      []common.Address
   932  	db            *Database
   933  	accountsDB    *accounts.Database
   934  	blockRangeDAO BlockRangeDAOer
   935  	blockDAO      *BlockDAO
   936  	chainClient   chain.ClientInterface
   937  	feed          *event.Feed
   938  	balanceCacher balance.Cacher
   939  	// nonArchivalRPCNode bool // TODO Make use of it
   940  	transactionManager *TransactionManager
   941  	pendingTxManager   *transactions.PendingTxTracker
   942  	tokenManager       *token.Manager
   943  	blocksLoadedCh     chan []*DBHeader
   944  	omitHistory        bool
   945  	contractMaker      *contracts.ContractMaker
   946  	blockChainState    *blockchainstate.BlockChainState
   947  
   948  	// Not to be set by the caller
   949  	transfersLoaded map[common.Address]bool // For event RecentHistoryReady to be sent only once per account during app lifetime
   950  	loops           atomic.Int32
   951  }
   952  
   953  func (c *loadBlocksAndTransfersCommand) incLoops() {
   954  	c.loops.Add(1)
   955  }
   956  
   957  func (c *loadBlocksAndTransfersCommand) decLoops() {
   958  	c.loops.Add(-1)
   959  }
   960  
   961  func (c *loadBlocksAndTransfersCommand) isStarted() bool {
   962  	return c.loops.Load() > 0
   963  }
   964  
   965  func (c *loadBlocksAndTransfersCommand) Run(parent context.Context) (err error) {
   966  	log.Debug("start load all transfers command", "chain", c.chainClient.NetworkID(), "accounts", c.accounts)
   967  
   968  	// Finite processes (to be restarted on error, but stopped on success or context cancel):
   969  	// fetching transfers for loaded blocks
   970  	// fetching history blocks
   971  
   972  	// Infinite processes (to be restarted on error), but stopped on context cancel:
   973  	// fetching new blocks
   974  	// fetching transfers for new blocks
   975  
   976  	ctx := parent
   977  	finiteGroup := async.NewAtomicGroup(ctx)
   978  	finiteGroup.SetName("finiteGroup")
   979  	defer func() {
   980  		finiteGroup.Stop()
   981  		finiteGroup.Wait()
   982  	}()
   983  
   984  	blockRanges, err := c.blockRangeDAO.getBlockRanges(c.chainClient.NetworkID(), c.accounts)
   985  	if err != nil {
   986  		return err
   987  	}
   988  
   989  	firstScan := false
   990  	var headNum *big.Int
   991  	for _, address := range c.accounts {
   992  		blockRange, ok := blockRanges[address]
   993  		if !ok || blockRange.tokens.LastKnown == nil {
   994  			firstScan = true
   995  			break
   996  		}
   997  
   998  		if headNum == nil || blockRange.tokens.LastKnown.Cmp(headNum) < 0 {
   999  			headNum = blockRange.tokens.LastKnown
  1000  		}
  1001  	}
  1002  
  1003  	fromNum := big.NewInt(0)
  1004  	if firstScan {
  1005  		headNum, err = getHeadBlockNumber(ctx, c.chainClient)
  1006  		if err != nil {
  1007  			return err
  1008  		}
  1009  	}
  1010  
  1011  	// It will start loadTransfersCommand which will run until all transfers from DB are loaded or any one failed to load
  1012  	err = c.startFetchingTransfersForLoadedBlocks(finiteGroup)
  1013  	if err != nil {
  1014  		log.Error("loadBlocksAndTransfersCommand fetchTransfersForLoadedBlocks", "error", err)
  1015  		return err
  1016  	}
  1017  
  1018  	if !c.isStarted() {
  1019  		c.startTransfersLoop(ctx)
  1020  		c.startFetchingNewBlocks(ctx, c.accounts, headNum, c.blocksLoadedCh)
  1021  	}
  1022  
  1023  	// It will start findBlocksCommands which will run until success when all blocks are loaded
  1024  	err = c.fetchHistoryBlocks(finiteGroup, c.accounts, fromNum, headNum, c.blocksLoadedCh)
  1025  	if err != nil {
  1026  		log.Error("loadBlocksAndTransfersCommand fetchHistoryBlocks", "error", err)
  1027  		return err
  1028  	}
  1029  
  1030  	select {
  1031  	case <-ctx.Done():
  1032  		log.Debug("loadBlocksAndTransfers command cancelled", "chain", c.chainClient.NetworkID(), "accounts", c.accounts, "error", ctx.Err())
  1033  	case <-finiteGroup.WaitAsync():
  1034  		err = finiteGroup.Error() // if there was an error, rerun the command
  1035  		log.Debug("end loadBlocksAndTransfers command", "chain", c.chainClient.NetworkID(), "accounts", c.accounts, "error", err, "group", finiteGroup.Name())
  1036  	}
  1037  
  1038  	return err
  1039  }
  1040  
  1041  func (c *loadBlocksAndTransfersCommand) Runner(interval ...time.Duration) async.Runner {
  1042  	// 30s - default interval for Infura's delay returned in error. That should increase chances
  1043  	// for request to succeed with the next attempt for now until we have a proper retry mechanism
  1044  	intvl := 30 * time.Second
  1045  	if len(interval) > 0 {
  1046  		intvl = interval[0]
  1047  	}
  1048  
  1049  	return async.FiniteCommand{
  1050  		Interval: intvl,
  1051  		Runable:  c.Run,
  1052  	}
  1053  }
  1054  
  1055  func (c *loadBlocksAndTransfersCommand) Command(interval ...time.Duration) async.Command {
  1056  	return c.Runner(interval...).Run
  1057  }
  1058  
  1059  func (c *loadBlocksAndTransfersCommand) fetchHistoryBlocks(group *async.AtomicGroup, accounts []common.Address, fromNum, toNum *big.Int, blocksLoadedCh chan []*DBHeader) (err error) {
  1060  	for _, account := range accounts {
  1061  		err = c.fetchHistoryBlocksForAccount(group, account, fromNum, toNum, c.blocksLoadedCh)
  1062  		if err != nil {
  1063  			return err
  1064  		}
  1065  	}
  1066  	return nil
  1067  }
  1068  
  1069  func (c *loadBlocksAndTransfersCommand) fetchHistoryBlocksForAccount(group *async.AtomicGroup, account common.Address, fromNum, toNum *big.Int, blocksLoadedCh chan []*DBHeader) error {
  1070  
  1071  	log.Debug("fetchHistoryBlocks start", "chainID", c.chainClient.NetworkID(), "account", account, "omit", c.omitHistory)
  1072  
  1073  	if c.omitHistory {
  1074  		blockRange := &ethTokensBlockRanges{eth: &BlockRange{nil, big.NewInt(0), toNum}, tokens: &BlockRange{nil, big.NewInt(0), toNum}}
  1075  		err := c.blockRangeDAO.upsertRange(c.chainClient.NetworkID(), account, blockRange)
  1076  		log.Error("fetchHistoryBlocks upsertRange", "error", err)
  1077  		return err
  1078  	}
  1079  
  1080  	blockRange, err := loadBlockRangeInfo(c.chainClient.NetworkID(), account, c.blockRangeDAO)
  1081  	if err != nil {
  1082  		log.Error("fetchHistoryBlocks loadBlockRangeInfo", "error", err)
  1083  		return err
  1084  	}
  1085  
  1086  	ranges := [][]*big.Int{}
  1087  	// There are 2 history intervals:
  1088  	// 1) from 0 to FirstKnown
  1089  	// 2) from LastKnown to `toNum`` (head)
  1090  	// If we blockRange is nil, we need to load all blocks from `fromNum` to `toNum`
  1091  	// As current implementation checks ETH first then tokens, tokens ranges maybe behind ETH ranges in
  1092  	// cases when block searching was interrupted, so we use tokens ranges
  1093  	if blockRange.tokens.LastKnown != nil || blockRange.tokens.FirstKnown != nil {
  1094  		if blockRange.tokens.LastKnown != nil && toNum.Cmp(blockRange.tokens.LastKnown) > 0 {
  1095  			ranges = append(ranges, []*big.Int{blockRange.tokens.LastKnown, toNum})
  1096  		}
  1097  
  1098  		if blockRange.tokens.FirstKnown != nil {
  1099  			if fromNum.Cmp(blockRange.tokens.FirstKnown) < 0 {
  1100  				ranges = append(ranges, []*big.Int{fromNum, blockRange.tokens.FirstKnown})
  1101  			} else {
  1102  				if !c.transfersLoaded[account] {
  1103  					transfersLoaded, err := c.areAllTransfersLoaded(account)
  1104  					if err != nil {
  1105  						return err
  1106  					}
  1107  
  1108  					if transfersLoaded {
  1109  						if c.transfersLoaded == nil {
  1110  							c.transfersLoaded = make(map[common.Address]bool)
  1111  						}
  1112  						c.transfersLoaded[account] = true
  1113  						c.notifyHistoryReady(account)
  1114  					}
  1115  				}
  1116  			}
  1117  		}
  1118  	} else {
  1119  		ranges = append(ranges, []*big.Int{fromNum, toNum})
  1120  	}
  1121  
  1122  	if len(ranges) > 0 {
  1123  		storage := chain.NewLimitsDBStorage(c.db.client)
  1124  		limiter := chain.NewRequestLimiter(storage)
  1125  		chainClient, _ := createChainClientWithLimiter(c.chainClient, account, limiter)
  1126  		if chainClient == nil {
  1127  			chainClient = c.chainClient
  1128  		}
  1129  
  1130  		for _, rangeItem := range ranges {
  1131  			log.Debug("range item", "r", rangeItem, "n", c.chainClient.NetworkID(), "a", account)
  1132  
  1133  			fbc := &findBlocksCommand{
  1134  				accounts:                  []common.Address{account},
  1135  				db:                        c.db,
  1136  				accountsDB:                c.accountsDB,
  1137  				blockRangeDAO:             c.blockRangeDAO,
  1138  				chainClient:               chainClient,
  1139  				balanceCacher:             c.balanceCacher,
  1140  				feed:                      c.feed,
  1141  				noLimit:                   false,
  1142  				fromBlockNumber:           rangeItem[0],
  1143  				toBlockNumber:             rangeItem[1],
  1144  				tokenManager:              c.tokenManager,
  1145  				blocksLoadedCh:            blocksLoadedCh,
  1146  				defaultNodeBlockChunkSize: DefaultNodeBlockChunkSize,
  1147  			}
  1148  			group.Add(fbc.Command())
  1149  		}
  1150  	}
  1151  
  1152  	return nil
  1153  }
  1154  
  1155  func (c *loadBlocksAndTransfersCommand) startFetchingNewBlocks(ctx context.Context, addresses []common.Address, fromNum *big.Int, blocksLoadedCh chan<- []*DBHeader) {
  1156  	log.Debug("startFetchingNewBlocks start", "chainID", c.chainClient.NetworkID(), "accounts", addresses)
  1157  
  1158  	c.incLoops()
  1159  	go func() {
  1160  		defer func() {
  1161  			c.decLoops()
  1162  		}()
  1163  
  1164  		newBlocksCmd := &findNewBlocksCommand{
  1165  			findBlocksCommand: &findBlocksCommand{
  1166  				accounts:                  addresses,
  1167  				db:                        c.db,
  1168  				accountsDB:                c.accountsDB,
  1169  				blockRangeDAO:             c.blockRangeDAO,
  1170  				chainClient:               c.chainClient,
  1171  				balanceCacher:             c.balanceCacher,
  1172  				feed:                      c.feed,
  1173  				noLimit:                   false,
  1174  				fromBlockNumber:           fromNum,
  1175  				tokenManager:              c.tokenManager,
  1176  				blocksLoadedCh:            blocksLoadedCh,
  1177  				defaultNodeBlockChunkSize: DefaultNodeBlockChunkSize,
  1178  			},
  1179  			contractMaker:                c.contractMaker,
  1180  			blockChainState:              c.blockChainState,
  1181  			nonceCheckIntervalIterations: nonceCheckIntervalIterations,
  1182  			logsCheckIntervalIterations:  logsCheckIntervalIterations,
  1183  		}
  1184  		group := async.NewGroup(ctx)
  1185  		group.Add(newBlocksCmd.Command())
  1186  
  1187  		// No need to wait for the group since it is infinite
  1188  		<-ctx.Done()
  1189  
  1190  		log.Debug("startFetchingNewBlocks end", "chainID", c.chainClient.NetworkID(), "accounts", addresses, "error", ctx.Err())
  1191  	}()
  1192  }
  1193  
  1194  func (c *loadBlocksAndTransfersCommand) getBlocksToLoad() (map[common.Address][]*big.Int, error) {
  1195  	blocksMap := make(map[common.Address][]*big.Int)
  1196  	for _, account := range c.accounts {
  1197  		blocks, err := c.blockDAO.GetBlocksToLoadByAddress(c.chainClient.NetworkID(), account, numberOfBlocksCheckedPerIteration)
  1198  		if err != nil {
  1199  			log.Error("loadBlocksAndTransfersCommand GetBlocksToLoadByAddress", "error", err)
  1200  			return nil, err
  1201  		}
  1202  
  1203  		if len(blocks) == 0 {
  1204  			log.Debug("fetchTransfers no blocks to load", "chainID", c.chainClient.NetworkID(), "account", account)
  1205  			continue
  1206  		}
  1207  
  1208  		blocksMap[account] = blocks
  1209  	}
  1210  
  1211  	if len(blocksMap) == 0 {
  1212  		log.Debug("fetchTransfers no blocks to load", "chainID", c.chainClient.NetworkID())
  1213  	}
  1214  
  1215  	return blocksMap, nil
  1216  }
  1217  
  1218  func (c *loadBlocksAndTransfersCommand) startFetchingTransfersForLoadedBlocks(group *async.AtomicGroup) error {
  1219  
  1220  	log.Debug("fetchTransfers start", "chainID", c.chainClient.NetworkID(), "accounts", c.accounts)
  1221  
  1222  	blocksMap, err := c.getBlocksToLoad()
  1223  	if err != nil {
  1224  		return err
  1225  	}
  1226  
  1227  	go func() {
  1228  		txCommand := &loadTransfersCommand{
  1229  			accounts:           c.accounts,
  1230  			db:                 c.db,
  1231  			blockDAO:           c.blockDAO,
  1232  			chainClient:        c.chainClient,
  1233  			transactionManager: c.transactionManager,
  1234  			pendingTxManager:   c.pendingTxManager,
  1235  			tokenManager:       c.tokenManager,
  1236  			blocksByAddress:    blocksMap,
  1237  			feed:               c.feed,
  1238  		}
  1239  
  1240  		group.Add(txCommand.Command())
  1241  		log.Debug("fetchTransfers end", "chainID", c.chainClient.NetworkID(), "accounts", c.accounts)
  1242  	}()
  1243  
  1244  	return nil
  1245  }
  1246  
  1247  func (c *loadBlocksAndTransfersCommand) notifyHistoryReady(account common.Address) {
  1248  	if c.feed != nil {
  1249  		c.feed.Send(walletevent.Event{
  1250  			Type:     EventRecentHistoryReady,
  1251  			Accounts: []common.Address{account},
  1252  			ChainID:  c.chainClient.NetworkID(),
  1253  		})
  1254  	}
  1255  }
  1256  
  1257  func (c *loadBlocksAndTransfersCommand) areAllTransfersLoaded(account common.Address) (bool, error) {
  1258  	allBlocksLoaded, err := areAllHistoryBlocksLoadedForAddress(c.blockRangeDAO, c.chainClient.NetworkID(), account)
  1259  	if err != nil {
  1260  		log.Error("loadBlockAndTransfersCommand allHistoryBlocksLoaded", "error", err)
  1261  		return false, err
  1262  	}
  1263  
  1264  	if allBlocksLoaded {
  1265  		headers, err := c.blockDAO.GetBlocksToLoadByAddress(c.chainClient.NetworkID(), account, 1)
  1266  		if err != nil {
  1267  			log.Error("loadBlocksAndTransfersCommand GetFirstSavedBlock", "error", err)
  1268  			return false, err
  1269  		}
  1270  
  1271  		if len(headers) == 0 {
  1272  			return true, nil
  1273  		}
  1274  	}
  1275  
  1276  	return false, nil
  1277  }
  1278  
  1279  // TODO - make it a common method for every service that wants head block number, that will cache the latest block
  1280  // and updates it on timeout
  1281  func getHeadBlockNumber(parent context.Context, chainClient chain.ClientInterface) (*big.Int, error) {
  1282  	ctx, cancel := context.WithTimeout(parent, 3*time.Second)
  1283  	head, err := chainClient.HeaderByNumber(ctx, nil)
  1284  	cancel()
  1285  	if err != nil {
  1286  		log.Error("getHeadBlockNumber", "error", err)
  1287  		return nil, err
  1288  	}
  1289  
  1290  	return head.Number, err
  1291  }
  1292  
  1293  func nextRange(maxRangeSize int, prevFrom, zeroBlockNumber *big.Int) (*big.Int, *big.Int) {
  1294  	log.Debug("next range start", "from", prevFrom, "zeroBlockNumber", zeroBlockNumber)
  1295  
  1296  	rangeSize := big.NewInt(int64(maxRangeSize))
  1297  
  1298  	to := big.NewInt(0).Set(prevFrom)
  1299  	from := big.NewInt(0).Sub(to, rangeSize)
  1300  	if from.Cmp(zeroBlockNumber) < 0 {
  1301  		from = new(big.Int).Set(zeroBlockNumber)
  1302  	}
  1303  
  1304  	log.Debug("next range end", "from", from, "to", to, "zeroBlockNumber", zeroBlockNumber)
  1305  
  1306  	return from, to
  1307  }
  1308  
  1309  func accountLimiterTag(account common.Address) string {
  1310  	return transferHistoryTag + "_" + account.String()
  1311  }
  1312  
  1313  func createChainClientWithLimiter(client chain.ClientInterface, account common.Address, limiter chain.RequestLimiter) (chain.ClientInterface, error) {
  1314  	// Each account has its own limit and a global limit for all accounts
  1315  	accountTag := accountLimiterTag(account)
  1316  	chainClient := chain.ClientWithTag(client, accountTag, transferHistoryTag)
  1317  
  1318  	// Check if limit is already reached, then skip the comamnd
  1319  	if allow, err := limiter.Allow(accountTag); !allow {
  1320  		log.Info("fetchHistoryBlocksForAccount limit reached", "account", account, "chain", chainClient.NetworkID(), "error", err)
  1321  		return nil, err
  1322  	}
  1323  
  1324  	if allow, err := limiter.Allow(transferHistoryTag); !allow {
  1325  		log.Info("fetchHistoryBlocksForAccount common limit reached", "chain", chainClient.NetworkID(), "error", err)
  1326  		return nil, err
  1327  	}
  1328  
  1329  	limit, _ := limiter.GetLimit(accountTag)
  1330  	if limit == nil {
  1331  		err := limiter.SetLimit(accountTag, transferHistoryLimitPerAccount, chain.LimitInfinitely)
  1332  		if err != nil {
  1333  			log.Error("fetchHistoryBlocksForAccount SetLimit", "error", err, "accountTag", accountTag)
  1334  		}
  1335  	}
  1336  
  1337  	// Here total limit per day is overwriten on each app start, that still saves us RPC calls, but allows to proceed
  1338  	// after app restart if the limit was reached. Currently there is no way to reset the limit from UI
  1339  	err := limiter.SetLimit(transferHistoryTag, transferHistoryLimit, transferHistoryLimitPeriod)
  1340  	if err != nil {
  1341  		log.Error("fetchHistoryBlocksForAccount SetLimit", "error", err, "groupTag", transferHistoryTag)
  1342  	}
  1343  	chainClient.SetLimiter(limiter)
  1344  
  1345  	return chainClient, nil
  1346  }