github.com/diadata-org/diadata@v1.4.593/pkg/dia/scraper/exchange-scrapers/APIScraper.go (about)

     1  package scrapers
     2  
     3  import (
     4  	"io"
     5  
     6  	"github.com/diadata-org/diadata/pkg/dia"
     7  	models "github.com/diadata-org/diadata/pkg/model"
     8  )
     9  
    10  // The collector kills a scraper after @watchdogDelayXXX seconds of inactivity
    11  const (
    12  // TODO use this with test
    13  // watchdogDelay3Mins = 3 * 60
    14  // watchdogDelayShort = 10 * 60
    15  // watchdogDelay      = 20 * 60
    16  // watchdogDelayLong  = 120 * 60
    17  )
    18  
    19  // empty type used for signaling
    20  type nothing struct{}
    21  
    22  var (
    23  	Exchanges          = make(map[string]dia.Exchange)
    24  	ExchangeDuplicates = make(map[string]dia.Exchange)
    25  	Blockchains        map[string]dia.BlockChain
    26  	chainConfigs       map[string]dia.ChainConfig
    27  )
    28  
    29  var evmID map[string]string
    30  
    31  func init() {
    32  
    33  	relDB, err := models.NewRelDataStore()
    34  	if err != nil {
    35  		log.Fatal("get rel datastore: ", err)
    36  	}
    37  
    38  	exchanges, err := relDB.GetAllExchanges()
    39  	if err != nil {
    40  		log.Fatal("get all exchanges: ", err)
    41  	}
    42  	for _, exchange := range exchanges {
    43  		Exchanges[exchange.Name] = exchange
    44  	}
    45  
    46  	evmID = make(map[string]string)
    47  	evmID["137"] = dia.POLYGON
    48  	evmID["1"] = dia.ETHEREUM
    49  	evmID["250"] = dia.FANTOM
    50  	evmID["56"] = dia.BINANCESMARTCHAIN
    51  	evmID["43114"] = dia.BINANCESMARTCHAIN
    52  	evmID["1284"] = dia.MOONBEAM
    53  	evmID["1285"] = dia.MOONRIVER
    54  	evmID["42161"] = dia.ARBITRUM
    55  	evmID["43114"] = dia.AVALANCHE
    56  
    57  	chains, err := relDB.GetAllBlockchains(false)
    58  	if err != nil {
    59  		log.Fatal("get all chains: ", err)
    60  	}
    61  	Blockchains = make(map[string]dia.BlockChain)
    62  	for _, chain := range chains {
    63  		Blockchains[chain.Name] = chain
    64  	}
    65  
    66  	chainconfigurations, err := relDB.GetAllChainConfig()
    67  	if err != nil {
    68  		log.Fatal("get all chains: ", err)
    69  	}
    70  	chainConfigs = make(map[string]dia.ChainConfig)
    71  	for _, chainconfig := range chainconfigurations {
    72  		chainConfigs[chainconfig.ChainID] = chainconfig
    73  	}
    74  
    75  	ExchangeDuplicates[dia.Binance2Exchange] = dia.Exchange{Name: "Binance2", Centralized: true, WatchdogDelay: 300}
    76  	ExchangeDuplicates[dia.Binance3Exchange] = dia.Exchange{Name: "Binance3", Centralized: true, WatchdogDelay: 300}
    77  	ExchangeDuplicates[dia.BKEX2Exchange] = dia.Exchange{Name: "BKEX2", Centralized: true, WatchdogDelay: 1200}
    78  
    79  }
    80  
    81  // APIScraper provides common methods needed to get Trade information from
    82  // exchange APIs.
    83  type APIScraper interface {
    84  	io.Closer
    85  	// ScrapePair returns a PairScraper that continuously scrapes trades for a
    86  	// single pair from this APIScraper
    87  	ScrapePair(pair dia.ExchangePair) (PairScraper, error)
    88  	// FetchAvailablePairs returns a list with all trading pairs available on
    89  	// the exchange associated to the APIScraper. The format is such that it can
    90  	// be used by the corr. pairScraper in order to fetch trades.
    91  	FetchAvailablePairs() ([]dia.ExchangePair, error)
    92  
    93  	// FillSymbolData collects information associated to the symbol ticker of an
    94  	// asset traded on the exchange associated to the APIScraper.
    95  	// Ideally, data is returned as close to original (blockchain) notation as possible.
    96  	// This is only needed for CEX. For DEX the trade can be filled.
    97  	FillSymbolData(symbol string) (dia.Asset, error)
    98  
    99  	NormalizePair(pair dia.ExchangePair) (dia.ExchangePair, error)
   100  	// Channel returns a channel that can be used to receive trades
   101  	Channel() chan *dia.Trade
   102  }
   103  
   104  // PairScraper receives trades for a single pc.ExchangePair from a single exchange.
   105  type PairScraper interface {
   106  	io.Closer
   107  	// Error returns an error when the channel Channel() is closed
   108  	// and nil otherwise
   109  	Error() error
   110  
   111  	// Pair returns the pair this scraper is subscribed to
   112  	Pair() dia.ExchangePair
   113  }
   114  
   115  // NewAPIScraper returns an API scraper for @exchange. If scrape==true it actually does
   116  // scraping. Otherwise can be used for pairdiscovery.
   117  func NewAPIScraper(exchange string, scrape bool, key string, secret string, relDB *models.RelDB) APIScraper {
   118  	switch exchange {
   119  	case dia.BitstampExchange:
   120  		return NewBitstampScraper(Exchanges[dia.BitstampExchange], scrape, relDB)
   121  	case dia.BinanceExchange:
   122  		return NewBinanceScraper(key, secret, Exchanges[dia.BinanceExchange], Exchanges[dia.BinanceExchange].Name, scrape, relDB)
   123  	case dia.Binance2Exchange:
   124  		return NewBinanceScraper(key, secret, Exchanges[dia.BinanceExchange], dia.Binance2Exchange, scrape, relDB)
   125  	case dia.Binance3Exchange:
   126  		return NewBinanceScraper(key, secret, Exchanges[dia.BinanceExchange], dia.Binance3Exchange, scrape, relDB)
   127  	case dia.BinanceExchangeUS:
   128  		return NewBinanceScraperUS(key, secret, Exchanges[dia.BinanceExchangeUS], scrape, relDB)
   129  	case dia.BitfinexExchange:
   130  		return NewBitfinexScraper(key, secret, Exchanges[dia.BitfinexExchange], scrape, relDB)
   131  	case dia.CoinBaseExchange:
   132  		return NewCoinBaseScraper(Exchanges[dia.CoinBaseExchange], scrape, relDB)
   133  	case dia.KrakenExchange:
   134  		return NewKrakenScraper(key, secret, Exchanges[dia.KrakenExchange], scrape, relDB)
   135  	case dia.OKExExchange:
   136  		return NewOKExScraper(Exchanges[dia.OKExExchange], scrape, relDB)
   137  	case dia.CryptoDotComExchange:
   138  		return NewCryptoDotComScraper(Exchanges[dia.CryptoDotComExchange], scrape, relDB)
   139  	case dia.HuobiExchange:
   140  		return NewHuobiScraper(Exchanges[dia.HuobiExchange], scrape, relDB)
   141  	case dia.GateIOExchange:
   142  		return NewGateIOScraper(Exchanges[dia.GateIOExchange], scrape, relDB)
   143  	case dia.BancorExchange:
   144  		return NewBancorScraper(Exchanges[dia.BancorExchange], scrape)
   145  	case dia.UniswapExchange:
   146  		return NewUniswapScraper(Exchanges[dia.UniswapExchange], scrape, relDB)
   147  	case dia.UniswapExchangeBase:
   148  		return NewUniswapScraper(Exchanges[dia.UniswapExchangeBase], scrape, relDB)
   149  	case dia.PanCakeSwap:
   150  		return NewUniswapScraper(Exchanges[dia.PanCakeSwap], scrape, relDB)
   151  	case dia.PanCakeSwapExchangeV3:
   152  		return NewUniswapV3Scraper(Exchanges[dia.PanCakeSwapExchangeV3], scrape, relDB)
   153  	case dia.SushiSwapExchange:
   154  		return NewUniswapScraper(Exchanges[dia.SushiSwapExchange], scrape, relDB)
   155  	case dia.SushiSwapExchangePolygon:
   156  		return NewUniswapScraper(Exchanges[dia.SushiSwapExchangePolygon], scrape, relDB)
   157  	case dia.SushiSwapExchangeArbitrum:
   158  		return NewUniswapScraper(Exchanges[dia.SushiSwapExchangeArbitrum], scrape, relDB)
   159  	case dia.SushiSwapExchangeFantom:
   160  		return NewUniswapScraper(Exchanges[dia.SushiSwapExchangeFantom], scrape, relDB)
   161  	case dia.CamelotExchange:
   162  		return NewUniswapScraper(Exchanges[dia.CamelotExchange], scrape, relDB)
   163  	case dia.CamelotExchangeV3:
   164  		return NewUniswapV3Scraper(Exchanges[dia.CamelotExchangeV3], scrape, relDB)
   165  	case dia.CurveFIExchange:
   166  		return NewCurveFIScraper(Exchanges[dia.CurveFIExchange], scrape, relDB)
   167  	case dia.CurveFIExchangeFantom:
   168  		return NewCurveFIScraper(Exchanges[dia.CurveFIExchangeFantom], scrape, relDB)
   169  	case dia.CurveFIExchangeMoonbeam:
   170  		return NewCurveFIScraper(Exchanges[dia.CurveFIExchangeMoonbeam], scrape, relDB)
   171  	case dia.CurveFIExchangePolygon:
   172  		return NewCurveFIScraper(Exchanges[dia.CurveFIExchangePolygon], scrape, relDB)
   173  	case dia.CurveFIExchangeArbitrum:
   174  		return NewCurveFIScraper(Exchanges[dia.CurveFIExchangeArbitrum], scrape, relDB)
   175  	case dia.BalancerV2Exchange:
   176  		return NewBalancerV2Scraper(Exchanges[dia.BalancerV2Exchange], scrape, relDB)
   177  	case dia.BalancerV2ExchangeArbitrum:
   178  		return NewBalancerV2Scraper(Exchanges[dia.BalancerV2ExchangeArbitrum], scrape, relDB)
   179  	case dia.BalancerV2ExchangePolygon:
   180  		return NewBalancerV2Scraper(Exchanges[dia.BalancerV2ExchangePolygon], scrape, relDB)
   181  	case dia.BalancerV3Exchange:
   182  		return NewBalancerV3Scraper(Exchanges[dia.BalancerV3Exchange], scrape, relDB)
   183  	case dia.BeetsExchange:
   184  		return NewBalancerV2Scraper(Exchanges[dia.BeetsExchange], scrape, relDB)
   185  	case dia.KuCoinExchange:
   186  		return NewKuCoinScraper(key, secret, Exchanges[dia.KuCoinExchange], scrape, relDB)
   187  	case dia.BitMartExchange:
   188  		return NewBitMartScraper(Exchanges[dia.BitMartExchange], scrape, relDB)
   189  	case dia.BitMaxExchange:
   190  		return NewBitMaxScraper(Exchanges[dia.BitMaxExchange], scrape, relDB)
   191  	case dia.MEXCExchange:
   192  		return NewMEXCScraper(Exchanges[dia.MEXCExchange], scrape, relDB)
   193  	case dia.BKEXExchange:
   194  		return NewBKEXScraper(Exchanges[dia.BKEXExchange], dia.BKEXExchange, scrape, relDB)
   195  	case dia.BKEX2Exchange:
   196  		return NewBKEXScraper(Exchanges[dia.BKEXExchange], dia.BKEX2Exchange, scrape, relDB)
   197  	case dia.UniswapExchangeV3:
   198  		return NewUniswapV3Scraper(Exchanges[dia.UniswapExchangeV3], scrape, relDB)
   199  	case dia.UniswapExchangeV3Base:
   200  		return NewUniswapV3Scraper(Exchanges[dia.UniswapExchangeV3Base], scrape, relDB)
   201  	case dia.UniswapExchangeV3Celo:
   202  		return NewUniswapV3Scraper(Exchanges[dia.UniswapExchangeV3Celo], scrape, relDB)
   203  	case dia.DfynNetwork:
   204  		return NewUniswapScraper(Exchanges[dia.DfynNetwork], scrape, relDB)
   205  	case dia.UbeswapExchange:
   206  		return NewUniswapScraper(Exchanges[dia.UbeswapExchange], scrape, relDB)
   207  	case dia.NileV1Exchange:
   208  		return NewUniswapScraper(Exchanges[dia.NileV1Exchange], scrape, relDB)
   209  	case dia.RamsesV1Exchange:
   210  		return NewUniswapScraper(Exchanges[dia.RamsesV1Exchange], scrape, relDB)
   211  	case dia.NileV2Exchange:
   212  		return NewUniswapV3Scraper(Exchanges[dia.NileV2Exchange], scrape, relDB)
   213  	case dia.RamsesV2Exchange:
   214  		return NewUniswapV3Scraper(Exchanges[dia.RamsesV2Exchange], scrape, relDB)
   215  	case dia.UniswapExchangeV3Polygon:
   216  		return NewUniswapV3Scraper(Exchanges[dia.UniswapExchangeV3Polygon], scrape, relDB)
   217  	case dia.UniswapExchangeV3Arbitrum:
   218  		return NewUniswapV3Scraper(Exchanges[dia.UniswapExchangeV3Arbitrum], scrape, relDB)
   219  	case dia.UniswapExchangeV4:
   220  		return NewUniswapV4Scraper(Exchanges[dia.UniswapExchangeV4], scrape, relDB)
   221  
   222  	case dia.HuckleberryExchange:
   223  		return NewUniswapScraper(Exchanges[dia.HuckleberryExchange], scrape, relDB)
   224  	case dia.TraderJoeExchange:
   225  		return NewUniswapScraper(Exchanges[dia.TraderJoeExchange], scrape, relDB)
   226  	case dia.PangolinExchange:
   227  		return NewUniswapScraper(Exchanges[dia.PangolinExchange], scrape, relDB)
   228  	case dia.PlatypusExchange:
   229  		return NewPlatypusScraper(Exchanges[dia.PlatypusExchange], scrape, relDB)
   230  	case dia.SpookyswapExchange:
   231  		return NewUniswapScraper(Exchanges[dia.SpookyswapExchange], scrape, relDB)
   232  	case dia.QuickswapExchange:
   233  		return NewUniswapScraper(Exchanges[dia.QuickswapExchange], scrape, relDB)
   234  	case dia.SpiritswapExchange:
   235  		return NewUniswapScraper(Exchanges[dia.SpiritswapExchange], scrape, relDB)
   236  	case dia.SolarbeamExchange:
   237  		return NewUniswapScraper(Exchanges[dia.SolarbeamExchange], scrape, relDB)
   238  	case dia.TrisolarisExchange:
   239  		return NewUniswapScraper(Exchanges[dia.TrisolarisExchange], scrape, relDB)
   240  	case dia.ByBitExchange:
   241  		return NewByBitScraper(Exchanges[dia.ByBitExchange], scrape, relDB)
   242  	case dia.OrcaExchange:
   243  		return NewOrcaScraper(Exchanges[dia.OrcaExchange], scrape)
   244  	case dia.AnyswapExchange:
   245  		return NewAnyswapScraper(Exchanges[dia.AnyswapExchange], scrape, relDB)
   246  	case dia.NetswapExchange:
   247  		return NewUniswapScraper(Exchanges[dia.NetswapExchange], scrape, relDB)
   248  	case dia.BitMexExchange:
   249  		return NewBitMexScraper(Exchanges[dia.BitMexExchange], scrape, relDB)
   250  	case dia.TethysExchange:
   251  		return NewUniswapScraper(Exchanges[dia.TethysExchange], scrape, relDB)
   252  	case dia.HermesExchange:
   253  		return NewUniswapScraper(Exchanges[dia.HermesExchange], scrape, relDB)
   254  	case dia.OmniDexExchange:
   255  		return NewUniswapScraper(Exchanges[dia.OmniDexExchange], scrape, relDB)
   256  	case dia.DiffusionExchange:
   257  		return NewUniswapScraper(Exchanges[dia.DiffusionExchange], scrape, relDB)
   258  	case dia.ApeswapExchange:
   259  		return NewUniswapScraper(Exchanges[dia.ApeswapExchange], scrape, relDB)
   260  	case dia.BiswapExchange:
   261  		return NewUniswapScraper(Exchanges[dia.BiswapExchange], scrape, relDB)
   262  	case dia.ArthswapExchange:
   263  		return NewUniswapScraper(Exchanges[dia.ArthswapExchange], scrape, relDB)
   264  	case dia.StellaswapExchange:
   265  		return NewUniswapScraper(Exchanges[dia.StellaswapExchange], scrape, relDB)
   266  	case dia.WanswapExchange:
   267  		return NewUniswapScraper(Exchanges[dia.WanswapExchange], scrape, relDB)
   268  	case dia.TraderJoeExchangeV2_1:
   269  		return NewTraderJoeScraper(Exchanges[dia.TraderJoeExchangeV2_1], scrape, relDB)
   270  	case dia.TraderJoeExchangeV2_1Arbitrum:
   271  		return NewTraderJoeScraper(Exchanges[dia.TraderJoeExchangeV2_1Arbitrum], scrape, relDB)
   272  	case dia.TraderJoeExchangeV2_1Avalanche:
   273  		return NewTraderJoeScraper(Exchanges[dia.TraderJoeExchangeV2_1Avalanche], scrape, relDB)
   274  	case dia.TraderJoeExchangeV2_1BNB:
   275  		return NewTraderJoeScraper(Exchanges[dia.TraderJoeExchangeV2_1BNB], scrape, relDB)
   276  	case dia.TraderJoeExchangeV2_2Avalanche:
   277  		return NewTraderJoeScraper(Exchanges[dia.TraderJoeExchangeV2_2Avalanche], scrape, relDB)
   278  	case dia.OsmosisExchange:
   279  		return NewOsmosisScraper(Exchanges[dia.OsmosisExchange], scrape, relDB)
   280  	case dia.ZenlinkswapExchange:
   281  		return NewZenlinkScraper(Exchanges[dia.ZenlinkswapExchange], scrape)
   282  	case dia.ZenlinkswapExchangeBifrostPolkadot:
   283  		return NewZenlinkScraper(Exchanges[dia.ZenlinkswapExchangeBifrostPolkadot], scrape)
   284  	case dia.VelodromeExchange:
   285  		return NewVelodromeScraper(Exchanges[dia.VelodromeExchange], scrape, relDB)
   286  	case dia.VelodromeExchangeSwellchain:
   287  		return NewVelodromeScraper(Exchanges[dia.VelodromeExchangeSwellchain], scrape, relDB)
   288  	case dia.AerodromeV1Exchange:
   289  		return NewVelodromeScraper(Exchanges[dia.AerodromeV1Exchange], scrape, relDB)
   290  	case dia.MaverickExchange:
   291  		return NewMaverickScraper(Exchanges[dia.MaverickExchange], scrape, relDB)
   292  	case dia.PearlfiExchangeTestnet:
   293  		return NewUniswapV3Scraper(Exchanges[dia.PearlfiExchangeTestnet], scrape, relDB)
   294  	case dia.PearlfiExchange:
   295  		return NewUniswapV3Scraper(Exchanges[dia.PearlfiExchange], scrape, relDB)
   296  	case dia.PearlfiStableswapExchange:
   297  		return NewUniswapScraper(Exchanges[dia.PearlfiStableswapExchange], scrape, relDB)
   298  	case dia.ThenaExchange:
   299  		return NewUniswapScraper(Exchanges[dia.ThenaExchange], scrape, relDB)
   300  	case dia.ThenaV3Exchange:
   301  		return NewUniswapV3Scraper(Exchanges[dia.ThenaV3Exchange], scrape, relDB)
   302  	// case dia.FinageForex:
   303  	// 	return NewFinageForexScraper(Exchanges[dia.FinageForex], scrape, relDB, key, secret)
   304  	case dia.BitgetExchange:
   305  		return NewBitgetScraper(Exchanges[dia.BitgetExchange], scrape, relDB)
   306  	case dia.AerodromeSlipstreamExchange:
   307  		return NewUniswapV3Scraper(Exchanges[dia.AerodromeSlipstreamExchange], scrape, relDB)
   308  	case dia.VelodromeSlipstreamExchange:
   309  		return NewUniswapV3Scraper(Exchanges[dia.VelodromeSlipstreamExchange], scrape, relDB)
   310  	case dia.CoinExExchange:
   311  		return NewCoinExScraper(Exchanges[dia.CoinExExchange], scrape, relDB)
   312  		// case dia.FinageForex:
   313  		// 	return NewFinageForexScraper(Exchanges[dia.FinageForex], scrape, relDB, key, secret)
   314  
   315  	case dia.MultiChain:
   316  		return NewBridgeSwapScraper(Exchanges[dia.MultiChain], scrape, relDB)
   317  
   318  	case "Influx":
   319  		return NewInfluxScraper(scrape)
   320  
   321  	case "UniswapHistory":
   322  		return NewUniswapHistoryScraper(Exchanges[dia.UniswapExchange], scrape, relDB)
   323  	case dia.AyinExchange:
   324  		return NewAyinScraper(Exchanges[dia.AyinExchange], scrape, relDB)
   325  	case dia.BitflowExchange:
   326  		return NewBitflowScraper(Exchanges[dia.BitflowExchange], scrape, relDB)
   327  	case dia.VelarExchange:
   328  		return NewVelarScraper(Exchanges[dia.VelarExchange], scrape, relDB)
   329  	case dia.BifrostExchange:
   330  		return NewBifrostScraper(Exchanges[dia.BifrostExchange], scrape, relDB)
   331  	case dia.HydrationExchange:
   332  		return NewHydrationScraper(Exchanges[dia.HydrationExchange], scrape, relDB)
   333  	default:
   334  		return nil
   335  	}
   336  
   337  }