github.com/trezor/blockbook@v0.4.1-0.20240328132726-e9a08582ee2c/fiat/coingecko.go (about) 1 package fiat 2 3 import ( 4 "encoding/json" 5 "fmt" 6 "io" 7 "net/http" 8 "net/url" 9 "os" 10 "strconv" 11 "strings" 12 "time" 13 14 "github.com/golang/glog" 15 "github.com/linxGnu/grocksdb" 16 "github.com/trezor/blockbook/common" 17 "github.com/trezor/blockbook/db" 18 ) 19 20 const ( 21 DefaultHTTPTimeout = 15 * time.Second 22 DefaultThrottleDelayMs = 100 // 100 ms delay between requests 23 ) 24 25 // Coingecko is a structure that implements RatesDownloaderInterface 26 type Coingecko struct { 27 url string 28 apiKey string 29 coin string 30 platformIdentifier string 31 platformVsCurrency string 32 allowedVsCurrencies map[string]struct{} 33 httpTimeout time.Duration 34 throttlingDelay time.Duration 35 timeFormat string 36 httpClient *http.Client 37 db *db.RocksDB 38 updatingCurrent bool 39 updatingTokens bool 40 metrics *common.Metrics 41 } 42 43 // simpleSupportedVSCurrencies https://api.coingecko.com/api/v3/simple/supported_vs_currencies 44 type simpleSupportedVSCurrencies []string 45 46 type coinsListItem struct { 47 ID string `json:"id"` 48 Symbol string `json:"symbol"` 49 Name string `json:"name"` 50 Platforms map[string]string `json:"platforms"` 51 } 52 53 // coinList https://api.coingecko.com/api/v3/coins/list 54 type coinList []coinsListItem 55 56 type marketPoint [2]float64 57 type marketChartPrices struct { 58 Prices []marketPoint `json:"prices"` 59 } 60 61 // NewCoinGeckoDownloader creates a coingecko structure that implements the RatesDownloaderInterface 62 func NewCoinGeckoDownloader(db *db.RocksDB, url string, coin string, platformIdentifier string, platformVsCurrency string, allowedVsCurrencies string, timeFormat string, metrics *common.Metrics, throttleDown bool) RatesDownloaderInterface { 63 throttlingDelayMs := 0 // No delay by default 64 if throttleDown { 65 throttlingDelayMs = DefaultThrottleDelayMs 66 } 67 68 allowedVsCurrenciesMap := getAllowedVsCurrenciesMap(allowedVsCurrencies) 69 70 apiKey := os.Getenv("COINGECKO_API_KEY") 71 72 // use default address if not overridden, with respect to existence of apiKey 73 if url == "" { 74 if apiKey != "" { 75 url = "https://pro-api.coingecko.com/api/v3/" 76 } else { 77 url = "https://api.coingecko.com/api/v3" 78 } 79 } 80 glog.Info("Coingecko downloader url ", url) 81 82 return &Coingecko{ 83 url: url, 84 apiKey: apiKey, 85 coin: coin, 86 platformIdentifier: platformIdentifier, 87 platformVsCurrency: platformVsCurrency, 88 allowedVsCurrencies: allowedVsCurrenciesMap, 89 httpTimeout: DefaultHTTPTimeout, 90 timeFormat: timeFormat, 91 httpClient: &http.Client{ 92 Timeout: DefaultHTTPTimeout, 93 }, 94 db: db, 95 throttlingDelay: time.Duration(throttlingDelayMs) * time.Millisecond, 96 metrics: metrics, 97 } 98 } 99 100 // getAllowedVsCurrenciesMap returns a map of allowed vs currencies 101 func getAllowedVsCurrenciesMap(currenciesString string) map[string]struct{} { 102 allowedVsCurrenciesMap := make(map[string]struct{}) 103 if len(currenciesString) > 0 { 104 for _, c := range strings.Split(strings.ToLower(currenciesString), ",") { 105 allowedVsCurrenciesMap[c] = struct{}{} 106 } 107 } 108 return allowedVsCurrenciesMap 109 } 110 111 // doReq HTTP client 112 func doReq(req *http.Request, client *http.Client) ([]byte, error) { 113 resp, err := client.Do(req) 114 if err != nil { 115 return nil, err 116 } 117 defer resp.Body.Close() 118 body, err := io.ReadAll(resp.Body) 119 if err != nil { 120 return nil, err 121 } 122 if resp.StatusCode != 200 { 123 return nil, fmt.Errorf("%s", body) 124 } 125 return body, nil 126 } 127 128 // makeReq HTTP request helper - will retry the call after 1 minute on error 129 func (cg *Coingecko) makeReq(url string, endpoint string) ([]byte, error) { 130 for { 131 // glog.Infof("Coingecko makeReq %v", url) 132 req, err := http.NewRequest("GET", url, nil) 133 if err != nil { 134 return nil, err 135 } 136 req.Header.Set("Content-Type", "application/json") 137 if cg.apiKey != "" { 138 req.Header.Set("x-cg-pro-api-key", cg.apiKey) 139 } 140 resp, err := doReq(req, cg.httpClient) 141 if err == nil { 142 if cg.metrics != nil { 143 cg.metrics.CoingeckoRequests.With(common.Labels{"endpoint": endpoint, "status": "success"}).Inc() 144 } 145 return resp, err 146 } 147 if err.Error() != "error code: 1015" && !strings.Contains(strings.ToLower(err.Error()), "exceeded the rate limit") && !strings.Contains(strings.ToLower(err.Error()), "throttled") { 148 if cg.metrics != nil { 149 cg.metrics.CoingeckoRequests.With(common.Labels{"endpoint": endpoint, "status": "error"}).Inc() 150 } 151 glog.Errorf("Coingecko makeReq %v error %v", url, err) 152 return nil, err 153 } 154 if cg.metrics != nil { 155 cg.metrics.CoingeckoRequests.With(common.Labels{"endpoint": endpoint, "status": "throttle"}).Inc() 156 } 157 // if there is a throttling error, wait 60 seconds and retry 158 glog.Warningf("Coingecko makeReq %v error %v, will retry in 60 seconds", url, err) 159 time.Sleep(60 * time.Second) 160 } 161 } 162 163 // SimpleSupportedVSCurrencies /simple/supported_vs_currencies 164 func (cg *Coingecko) simpleSupportedVSCurrencies() (simpleSupportedVSCurrencies, error) { 165 url := cg.url + "/simple/supported_vs_currencies" 166 resp, err := cg.makeReq(url, "supported_vs_currencies") 167 if err != nil { 168 return nil, err 169 } 170 var data simpleSupportedVSCurrencies 171 err = json.Unmarshal(resp, &data) 172 if err != nil { 173 return nil, err 174 } 175 if len(cg.allowedVsCurrencies) == 0 { 176 return data, nil 177 } 178 filtered := make([]string, 0, len(cg.allowedVsCurrencies)) 179 for _, c := range data { 180 if _, found := cg.allowedVsCurrencies[c]; found { 181 filtered = append(filtered, c) 182 } 183 } 184 return filtered, nil 185 } 186 187 // SimplePrice /simple/price Multiple ID and Currency (ids, vs_currencies) 188 func (cg *Coingecko) simplePrice(ids []string, vsCurrencies []string) (*map[string]map[string]float32, error) { 189 params := url.Values{} 190 idsParam := strings.Join(ids, ",") 191 vsCurrenciesParam := strings.Join(vsCurrencies, ",") 192 193 params.Add("ids", idsParam) 194 params.Add("vs_currencies", vsCurrenciesParam) 195 196 url := fmt.Sprintf("%s/simple/price?%s", cg.url, params.Encode()) 197 resp, err := cg.makeReq(url, "simple/price") 198 if err != nil { 199 return nil, err 200 } 201 202 t := make(map[string]map[string]float32) 203 err = json.Unmarshal(resp, &t) 204 if err != nil { 205 return nil, err 206 } 207 208 return &t, nil 209 } 210 211 // CoinsList /coins/list 212 func (cg *Coingecko) coinsList() (coinList, error) { 213 params := url.Values{} 214 platform := "false" 215 if cg.platformIdentifier != "" { 216 platform = "true" 217 } 218 params.Add("include_platform", platform) 219 url := fmt.Sprintf("%s/coins/list?%s", cg.url, params.Encode()) 220 resp, err := cg.makeReq(url, "coins/list") 221 if err != nil { 222 return nil, err 223 } 224 225 var data coinList 226 err = json.Unmarshal(resp, &data) 227 if err != nil { 228 return nil, err 229 } 230 return data, nil 231 } 232 233 // coinMarketChart /coins/{id}/market_chart?vs_currency={usd, eur, jpy, etc.}&days={1,14,30,max} 234 func (cg *Coingecko) coinMarketChart(id string, vs_currency string, days string, daily bool) (*marketChartPrices, error) { 235 if len(id) == 0 || len(vs_currency) == 0 || len(days) == 0 { 236 return nil, fmt.Errorf("id, vs_currency, and days is required") 237 } 238 239 params := url.Values{} 240 if daily { 241 params.Add("interval", "daily") 242 } 243 params.Add("vs_currency", vs_currency) 244 params.Add("days", days) 245 246 url := fmt.Sprintf("%s/coins/%s/market_chart?%s", cg.url, id, params.Encode()) 247 resp, err := cg.makeReq(url, "market_chart") 248 if err != nil { 249 return nil, err 250 } 251 252 m := marketChartPrices{} 253 err = json.Unmarshal(resp, &m) 254 if err != nil { 255 return &m, err 256 } 257 258 return &m, nil 259 } 260 261 var vsCurrencies []string 262 var platformIds []string 263 var platformIdsToTokens map[string]string 264 265 func (cg *Coingecko) platformIds() error { 266 if cg.platformIdentifier == "" { 267 return nil 268 } 269 cl, err := cg.coinsList() 270 if err != nil { 271 return err 272 } 273 idsMap := make(map[string]string, 64) 274 ids := make([]string, 0, 64) 275 for i := range cl { 276 id, found := cl[i].Platforms[cg.platformIdentifier] 277 if found && id != "" { 278 idsMap[cl[i].ID] = id 279 ids = append(ids, cl[i].ID) 280 } 281 } 282 platformIds = ids 283 platformIdsToTokens = idsMap 284 return nil 285 } 286 287 // CurrentTickers returns the latest exchange rates 288 func (cg *Coingecko) CurrentTickers() (*common.CurrencyRatesTicker, error) { 289 cg.updatingCurrent = true 290 defer func() { cg.updatingCurrent = false }() 291 292 var newTickers = common.CurrencyRatesTicker{} 293 294 if vsCurrencies == nil { 295 vs, err := cg.simpleSupportedVSCurrencies() 296 if err != nil { 297 return nil, err 298 } 299 vsCurrencies = vs 300 } 301 prices, err := cg.simplePrice([]string{cg.coin}, vsCurrencies) 302 if err != nil || prices == nil { 303 return nil, err 304 } 305 newTickers.Rates = make(map[string]float32, len((*prices)[cg.coin])) 306 for t, v := range (*prices)[cg.coin] { 307 newTickers.Rates[t] = v 308 } 309 310 if cg.platformIdentifier != "" && cg.platformVsCurrency != "" { 311 if platformIdsToTokens == nil { 312 err = cg.platformIds() 313 if err != nil { 314 return nil, err 315 } 316 } 317 newTickers.TokenRates = make(map[string]float32) 318 from := 0 319 const maxRequestLen = 6000 320 requestLen := 0 321 for to := 0; to < len(platformIds); to++ { 322 requestLen += len(platformIds[to]) + 3 // 3 characters for the comma separator %2C 323 if requestLen > maxRequestLen || to+1 >= len(platformIds) { 324 tokenPrices, err := cg.simplePrice(platformIds[from:to+1], []string{cg.platformVsCurrency}) 325 if err != nil || tokenPrices == nil { 326 return nil, err 327 } 328 for id, v := range *tokenPrices { 329 t, found := platformIdsToTokens[id] 330 if found { 331 newTickers.TokenRates[t] = v[cg.platformVsCurrency] 332 } 333 } 334 from = to + 1 335 requestLen = 0 336 } 337 } 338 } 339 newTickers.Timestamp = time.Now().UTC() 340 return &newTickers, nil 341 } 342 343 func (cg *Coingecko) getHighGranularityTickers(days string) (*[]common.CurrencyRatesTicker, error) { 344 mc, err := cg.coinMarketChart(cg.coin, highGranularityVsCurrency, days, false) 345 if err != nil { 346 return nil, err 347 } 348 if len(mc.Prices) < 2 { 349 return nil, nil 350 } 351 // ignore the last point, it is not in granularity 352 tickers := make([]common.CurrencyRatesTicker, len(mc.Prices)-1) 353 for i, p := range mc.Prices[:len(mc.Prices)-1] { 354 var timestamp uint 355 timestamp = uint(p[0]) 356 if timestamp > 100000000000 { 357 // convert timestamp from milliseconds to seconds 358 timestamp /= 1000 359 } 360 rate := float32(p[1]) 361 u := time.Unix(int64(timestamp), 0).UTC() 362 ticker := common.CurrencyRatesTicker{ 363 Timestamp: u, 364 Rates: make(map[string]float32), 365 } 366 ticker.Rates[highGranularityVsCurrency] = rate 367 tickers[i] = ticker 368 } 369 return &tickers, nil 370 } 371 372 // HourlyTickers returns the array of the exchange rates in hourly granularity 373 func (cg *Coingecko) HourlyTickers() (*[]common.CurrencyRatesTicker, error) { 374 return cg.getHighGranularityTickers("90") 375 } 376 377 // HourlyTickers returns the array of the exchange rates in five minutes granularity 378 func (cg *Coingecko) FiveMinutesTickers() (*[]common.CurrencyRatesTicker, error) { 379 return cg.getHighGranularityTickers("1") 380 } 381 382 func (cg *Coingecko) getHistoricalTicker(tickersToUpdate map[uint]*common.CurrencyRatesTicker, coinId string, vsCurrency string, token string) (bool, error) { 383 lastTicker, err := cg.db.FiatRatesFindLastTicker(vsCurrency, token) 384 if err != nil { 385 return false, err 386 } 387 var days string 388 if lastTicker == nil { 389 days = "max" 390 } else { 391 diff := time.Since(lastTicker.Timestamp) 392 d := int(diff / (24 * 3600 * 1000000000)) 393 if d == 0 { // nothing to do, the last ticker exist 394 return false, nil 395 } 396 days = strconv.Itoa(d) 397 } 398 mc, err := cg.coinMarketChart(coinId, vsCurrency, days, true) 399 if err != nil { 400 return false, err 401 } 402 warningLogged := false 403 for _, p := range mc.Prices { 404 var timestamp uint 405 timestamp = uint(p[0]) 406 if timestamp > 100000000000 { 407 // convert timestamp from milliseconds to seconds 408 timestamp /= 1000 409 } 410 rate := float32(p[1]) 411 if timestamp%(24*3600) == 0 && timestamp != 0 && rate != 0 { // process only tickers for the whole day with non 0 value 412 var found bool 413 var ticker *common.CurrencyRatesTicker 414 if ticker, found = tickersToUpdate[timestamp]; !found { 415 u := time.Unix(int64(timestamp), 0).UTC() 416 ticker, err = cg.db.FiatRatesGetTicker(&u) 417 if err != nil { 418 return false, err 419 } 420 if ticker == nil { 421 if token != "" { // if the base currency is not found in DB, do not create ticker for the token 422 if !warningLogged { 423 glog.Warningf("No base currency ticker for date %v for token %s", u, token) 424 warningLogged = true 425 } 426 continue 427 } 428 ticker = &common.CurrencyRatesTicker{ 429 Timestamp: u, 430 Rates: make(map[string]float32), 431 } 432 } 433 tickersToUpdate[timestamp] = ticker 434 } 435 if token == "" { 436 ticker.Rates[vsCurrency] = rate 437 } else { 438 if ticker.TokenRates == nil { 439 ticker.TokenRates = make(map[string]float32) 440 } 441 ticker.TokenRates[token] = rate 442 } 443 } 444 } 445 return true, nil 446 } 447 448 func (cg *Coingecko) storeTickers(tickersToUpdate map[uint]*common.CurrencyRatesTicker) error { 449 if len(tickersToUpdate) > 0 { 450 wb := grocksdb.NewWriteBatch() 451 defer wb.Destroy() 452 for _, v := range tickersToUpdate { 453 if err := cg.db.FiatRatesStoreTicker(wb, v); err != nil { 454 return err 455 } 456 } 457 if err := cg.db.WriteBatch(wb); err != nil { 458 return err 459 } 460 } 461 return nil 462 } 463 464 func (cg *Coingecko) throttleHistoricalDownload() { 465 // long delay next request to avoid throttling if downloading current tickers at the same time 466 delay := 1 467 if cg.updatingCurrent { 468 delay = 600 469 } 470 time.Sleep(cg.throttlingDelay * time.Duration(delay)) 471 } 472 473 // UpdateHistoricalTickers gets historical tickers for the main crypto currency 474 func (cg *Coingecko) UpdateHistoricalTickers() error { 475 tickersToUpdate := make(map[uint]*common.CurrencyRatesTicker) 476 477 // reload vs_currencies 478 vs, err := cg.simpleSupportedVSCurrencies() 479 if err != nil { 480 return err 481 } 482 vsCurrencies = vs 483 484 for _, currency := range vsCurrencies { 485 // get historical rates for each currency 486 var err error 487 var req bool 488 if req, err = cg.getHistoricalTicker(tickersToUpdate, cg.coin, currency, ""); err != nil { 489 // report error and continue, Coingecko may return error like "Could not find coin with the given id" 490 // the rates will be updated next run 491 glog.Errorf("getHistoricalTicker %s-%s %v", cg.coin, currency, err) 492 } 493 if req { 494 cg.throttleHistoricalDownload() 495 } 496 } 497 498 return cg.storeTickers(tickersToUpdate) 499 } 500 501 // UpdateHistoricalTokenTickers gets historical tickers for the tokens 502 func (cg *Coingecko) UpdateHistoricalTokenTickers() error { 503 if cg.updatingTokens { 504 return nil 505 } 506 cg.updatingTokens = true 507 defer func() { cg.updatingTokens = false }() 508 tickersToUpdate := make(map[uint]*common.CurrencyRatesTicker) 509 510 if cg.platformIdentifier != "" && cg.platformVsCurrency != "" { 511 // reload platform ids 512 if err := cg.platformIds(); err != nil { 513 return err 514 } 515 glog.Infof("Coingecko returned %d %s tokens ", len(platformIds), cg.coin) 516 count := 0 517 // get token historical rates 518 for tokenId, token := range platformIdsToTokens { 519 var err error 520 var req bool 521 if req, err = cg.getHistoricalTicker(tickersToUpdate, tokenId, cg.platformVsCurrency, token); err != nil { 522 // report error and continue, Coingecko may return error like "Could not find coin with the given id" 523 // the rates will be updated next run 524 glog.Errorf("getHistoricalTicker %s-%s %v", tokenId, cg.platformVsCurrency, err) 525 } 526 count++ 527 if count%100 == 0 { 528 err := cg.storeTickers(tickersToUpdate) 529 if err != nil { 530 return err 531 } 532 tickersToUpdate = make(map[uint]*common.CurrencyRatesTicker) 533 glog.Infof("Coingecko updated %d of %d token tickers", count, len(platformIds)) 534 } 535 if req { 536 cg.throttleHistoricalDownload() 537 } 538 } 539 } 540 541 return cg.storeTickers(tickersToUpdate) 542 }