github.com/cryptohub-digital/blockbook-fork@v0.0.0-20230713133354-673c927af7f1/fiat/coingecko.go (about) 1 package fiat 2 3 import ( 4 "encoding/json" 5 "fmt" 6 "io" 7 "net/http" 8 "net/url" 9 "os" 10 "strconv" 11 "strings" 12 "time" 13 14 "github.com/cryptohub-digital/blockbook-fork/common" 15 "github.com/cryptohub-digital/blockbook-fork/db" 16 "github.com/golang/glog" 17 "github.com/linxGnu/grocksdb" 18 ) 19 20 // Coingecko is a structure that implements RatesDownloaderInterface 21 type Coingecko struct { 22 url string 23 apiKey string 24 coin string 25 platformIdentifier string 26 platformVsCurrency string 27 allowedVsCurrencies map[string]struct{} 28 httpTimeout time.Duration 29 throttlingDelay time.Duration 30 timeFormat string 31 httpClient *http.Client 32 db *db.RocksDB 33 updatingCurrent bool 34 updatingTokens bool 35 metrics *common.Metrics 36 } 37 38 // simpleSupportedVSCurrencies https://api.coingecko.com/api/v3/simple/supported_vs_currencies 39 type simpleSupportedVSCurrencies []string 40 41 type coinsListItem struct { 42 ID string `json:"id"` 43 Symbol string `json:"symbol"` 44 Name string `json:"name"` 45 Platforms map[string]string `json:"platforms"` 46 } 47 48 // coinList https://api.coingecko.com/api/v3/coins/list 49 type coinList []coinsListItem 50 51 type marketPoint [2]float64 52 type marketChartPrices struct { 53 Prices []marketPoint `json:"prices"` 54 } 55 56 // NewCoinGeckoDownloader creates a coingecko structure that implements the RatesDownloaderInterface 57 func NewCoinGeckoDownloader(db *db.RocksDB, url string, coin string, platformIdentifier string, platformVsCurrency string, allowedVsCurrencies string, timeFormat string, metrics *common.Metrics, throttleDown bool) RatesDownloaderInterface { 58 var throttlingDelayMs int 59 if throttleDown { 60 throttlingDelayMs = 100 61 } 62 httpTimeout := 15 * time.Second 63 allowedVsCurrenciesMap := make(map[string]struct{}) 64 if len(allowedVsCurrencies) > 0 { 65 for _, c := range strings.Split(strings.ToLower(allowedVsCurrencies), ",") { 66 allowedVsCurrenciesMap[c] = struct{}{} 67 } 68 } 69 70 apiKey := os.Getenv("COINGECKO_API_KEY") 71 72 // use default address if not overridden, with respect to existence of apiKey 73 if url == "" { 74 if apiKey != "" { 75 url = "https://pro-api.coingecko.com/api/v3/" 76 } else { 77 url = "https://api.coingecko.com/api/v3" 78 } 79 } 80 glog.Info("Coingecko downloader url ", url) 81 82 return &Coingecko{ 83 url: url, 84 apiKey: apiKey, 85 coin: coin, 86 platformIdentifier: platformIdentifier, 87 platformVsCurrency: platformVsCurrency, 88 allowedVsCurrencies: allowedVsCurrenciesMap, 89 httpTimeout: httpTimeout, 90 timeFormat: timeFormat, 91 httpClient: &http.Client{ 92 Timeout: httpTimeout, 93 }, 94 db: db, 95 throttlingDelay: time.Duration(throttlingDelayMs) * time.Millisecond, 96 metrics: metrics, 97 } 98 } 99 100 // doReq HTTP client 101 func doReq(req *http.Request, client *http.Client) ([]byte, error) { 102 resp, err := client.Do(req) 103 if err != nil { 104 return nil, err 105 } 106 defer resp.Body.Close() 107 body, err := io.ReadAll(resp.Body) 108 if err != nil { 109 return nil, err 110 } 111 if resp.StatusCode != 200 { 112 return nil, fmt.Errorf("%s", body) 113 } 114 return body, nil 115 } 116 117 // makeReq HTTP request helper - will retry the call after 1 minute on error 118 func (cg *Coingecko) makeReq(url string, endpoint string) ([]byte, error) { 119 for { 120 // glog.Infof("Coingecko makeReq %v", url) 121 req, err := http.NewRequest("GET", url, nil) 122 if err != nil { 123 return nil, err 124 } 125 req.Header.Set("Content-Type", "application/json") 126 if cg.apiKey != "" { 127 req.Header.Set("x-cg-pro-api-key", cg.apiKey) 128 } 129 resp, err := doReq(req, cg.httpClient) 130 if err == nil { 131 if cg.metrics != nil { 132 cg.metrics.CoingeckoRequests.With(common.Labels{"endpoint": endpoint, "status": "success"}).Inc() 133 } 134 return resp, err 135 } 136 if err.Error() != "error code: 1015" && !strings.Contains(strings.ToLower(err.Error()), "exceeded the rate limit") { 137 if cg.metrics != nil { 138 cg.metrics.CoingeckoRequests.With(common.Labels{"endpoint": endpoint, "status": "error"}).Inc() 139 } 140 glog.Errorf("Coingecko makeReq %v error %v", url, err) 141 return nil, err 142 } 143 if cg.metrics != nil { 144 cg.metrics.CoingeckoRequests.With(common.Labels{"endpoint": endpoint, "status": "throttle"}).Inc() 145 } 146 // if there is a throttling error, wait 60 seconds and retry 147 glog.Warningf("Coingecko makeReq %v error %v, will retry in 60 seconds", url, err) 148 time.Sleep(60 * time.Second) 149 } 150 } 151 152 // SimpleSupportedVSCurrencies /simple/supported_vs_currencies 153 func (cg *Coingecko) simpleSupportedVSCurrencies() (simpleSupportedVSCurrencies, error) { 154 url := cg.url + "/simple/supported_vs_currencies" 155 resp, err := cg.makeReq(url, "supported_vs_currencies") 156 if err != nil { 157 return nil, err 158 } 159 var data simpleSupportedVSCurrencies 160 err = json.Unmarshal(resp, &data) 161 if err != nil { 162 return nil, err 163 } 164 if len(cg.allowedVsCurrencies) == 0 { 165 return data, nil 166 } 167 filtered := make([]string, 0, len(cg.allowedVsCurrencies)) 168 for _, c := range data { 169 if _, found := cg.allowedVsCurrencies[c]; found { 170 filtered = append(filtered, c) 171 } 172 } 173 return filtered, nil 174 } 175 176 // SimplePrice /simple/price Multiple ID and Currency (ids, vs_currencies) 177 func (cg *Coingecko) simplePrice(ids []string, vsCurrencies []string) (*map[string]map[string]float32, error) { 178 params := url.Values{} 179 idsParam := strings.Join(ids, ",") 180 vsCurrenciesParam := strings.Join(vsCurrencies, ",") 181 182 params.Add("ids", idsParam) 183 params.Add("vs_currencies", vsCurrenciesParam) 184 185 url := fmt.Sprintf("%s/simple/price?%s", cg.url, params.Encode()) 186 resp, err := cg.makeReq(url, "simple/price") 187 if err != nil { 188 return nil, err 189 } 190 191 t := make(map[string]map[string]float32) 192 err = json.Unmarshal(resp, &t) 193 if err != nil { 194 return nil, err 195 } 196 197 return &t, nil 198 } 199 200 // CoinsList /coins/list 201 func (cg *Coingecko) coinsList() (coinList, error) { 202 params := url.Values{} 203 platform := "false" 204 if cg.platformIdentifier != "" { 205 platform = "true" 206 } 207 params.Add("include_platform", platform) 208 url := fmt.Sprintf("%s/coins/list?%s", cg.url, params.Encode()) 209 resp, err := cg.makeReq(url, "coins/list") 210 if err != nil { 211 return nil, err 212 } 213 214 var data coinList 215 err = json.Unmarshal(resp, &data) 216 if err != nil { 217 return nil, err 218 } 219 return data, nil 220 } 221 222 // coinMarketChart /coins/{id}/market_chart?vs_currency={usd, eur, jpy, etc.}&days={1,14,30,max} 223 func (cg *Coingecko) coinMarketChart(id string, vs_currency string, days string, daily bool) (*marketChartPrices, error) { 224 if len(id) == 0 || len(vs_currency) == 0 || len(days) == 0 { 225 return nil, fmt.Errorf("id, vs_currency, and days is required") 226 } 227 228 params := url.Values{} 229 if daily { 230 params.Add("interval", "daily") 231 } 232 params.Add("vs_currency", vs_currency) 233 params.Add("days", days) 234 235 url := fmt.Sprintf("%s/coins/%s/market_chart?%s", cg.url, id, params.Encode()) 236 resp, err := cg.makeReq(url, "market_chart") 237 if err != nil { 238 return nil, err 239 } 240 241 m := marketChartPrices{} 242 err = json.Unmarshal(resp, &m) 243 if err != nil { 244 return &m, err 245 } 246 247 return &m, nil 248 } 249 250 var vsCurrencies []string 251 var platformIds []string 252 var platformIdsToTokens map[string]string 253 254 func (cg *Coingecko) platformIds() error { 255 if cg.platformIdentifier == "" { 256 return nil 257 } 258 cl, err := cg.coinsList() 259 if err != nil { 260 return err 261 } 262 idsMap := make(map[string]string, 64) 263 ids := make([]string, 0, 64) 264 for i := range cl { 265 id, found := cl[i].Platforms[cg.platformIdentifier] 266 if found && id != "" { 267 idsMap[cl[i].ID] = id 268 ids = append(ids, cl[i].ID) 269 } 270 } 271 platformIds = ids 272 platformIdsToTokens = idsMap 273 return nil 274 } 275 276 // CurrentTickers returns the latest exchange rates 277 func (cg *Coingecko) CurrentTickers() (*common.CurrencyRatesTicker, error) { 278 cg.updatingCurrent = true 279 defer func() { cg.updatingCurrent = false }() 280 281 var newTickers = common.CurrencyRatesTicker{} 282 283 if vsCurrencies == nil { 284 vs, err := cg.simpleSupportedVSCurrencies() 285 if err != nil { 286 return nil, err 287 } 288 vsCurrencies = vs 289 } 290 prices, err := cg.simplePrice([]string{cg.coin}, vsCurrencies) 291 if err != nil || prices == nil { 292 return nil, err 293 } 294 newTickers.Rates = make(map[string]float32, len((*prices)[cg.coin])) 295 for t, v := range (*prices)[cg.coin] { 296 newTickers.Rates[t] = v 297 } 298 299 if cg.platformIdentifier != "" && cg.platformVsCurrency != "" { 300 if platformIdsToTokens == nil { 301 err = cg.platformIds() 302 if err != nil { 303 return nil, err 304 } 305 } 306 newTickers.TokenRates = make(map[string]float32) 307 from := 0 308 const maxRequestLen = 6000 309 requestLen := 0 310 for to := 0; to < len(platformIds); to++ { 311 requestLen += len(platformIds[to]) + 3 // 3 characters for the comma separator %2C 312 if requestLen > maxRequestLen || to+1 >= len(platformIds) { 313 tokenPrices, err := cg.simplePrice(platformIds[from:to+1], []string{cg.platformVsCurrency}) 314 if err != nil || tokenPrices == nil { 315 return nil, err 316 } 317 for id, v := range *tokenPrices { 318 t, found := platformIdsToTokens[id] 319 if found { 320 newTickers.TokenRates[t] = v[cg.platformVsCurrency] 321 } 322 } 323 from = to + 1 324 requestLen = 0 325 } 326 } 327 } 328 newTickers.Timestamp = time.Now().UTC() 329 return &newTickers, nil 330 } 331 332 func (cg *Coingecko) getHighGranularityTickers(days string) (*[]common.CurrencyRatesTicker, error) { 333 mc, err := cg.coinMarketChart(cg.coin, highGranularityVsCurrency, days, false) 334 if err != nil { 335 return nil, err 336 } 337 if len(mc.Prices) < 2 { 338 return nil, nil 339 } 340 // ignore the last point, it is not in granularity 341 tickers := make([]common.CurrencyRatesTicker, len(mc.Prices)-1) 342 for i, p := range mc.Prices[:len(mc.Prices)-1] { 343 var timestamp uint 344 timestamp = uint(p[0]) 345 if timestamp > 100000000000 { 346 // convert timestamp from milliseconds to seconds 347 timestamp /= 1000 348 } 349 rate := float32(p[1]) 350 u := time.Unix(int64(timestamp), 0).UTC() 351 ticker := common.CurrencyRatesTicker{ 352 Timestamp: u, 353 Rates: make(map[string]float32), 354 } 355 ticker.Rates[highGranularityVsCurrency] = rate 356 tickers[i] = ticker 357 } 358 return &tickers, nil 359 } 360 361 // HourlyTickers returns the array of the exchange rates in hourly granularity 362 func (cg *Coingecko) HourlyTickers() (*[]common.CurrencyRatesTicker, error) { 363 return cg.getHighGranularityTickers("90") 364 } 365 366 // HourlyTickers returns the array of the exchange rates in five minutes granularity 367 func (cg *Coingecko) FiveMinutesTickers() (*[]common.CurrencyRatesTicker, error) { 368 return cg.getHighGranularityTickers("1") 369 } 370 371 func (cg *Coingecko) getHistoricalTicker(tickersToUpdate map[uint]*common.CurrencyRatesTicker, coinId string, vsCurrency string, token string) (bool, error) { 372 lastTicker, err := cg.db.FiatRatesFindLastTicker(vsCurrency, token) 373 if err != nil { 374 return false, err 375 } 376 var days string 377 if lastTicker == nil { 378 days = "max" 379 } else { 380 diff := time.Since(lastTicker.Timestamp) 381 d := int(diff / (24 * 3600 * 1000000000)) 382 if d == 0 { // nothing to do, the last ticker exist 383 return false, nil 384 } 385 days = strconv.Itoa(d) 386 } 387 mc, err := cg.coinMarketChart(coinId, vsCurrency, days, true) 388 if err != nil { 389 return false, err 390 } 391 warningLogged := false 392 for _, p := range mc.Prices { 393 var timestamp uint 394 timestamp = uint(p[0]) 395 if timestamp > 100000000000 { 396 // convert timestamp from milliseconds to seconds 397 timestamp /= 1000 398 } 399 rate := float32(p[1]) 400 if timestamp%(24*3600) == 0 && timestamp != 0 && rate != 0 { // process only tickers for the whole day with non 0 value 401 var found bool 402 var ticker *common.CurrencyRatesTicker 403 if ticker, found = tickersToUpdate[timestamp]; !found { 404 u := time.Unix(int64(timestamp), 0).UTC() 405 ticker, err = cg.db.FiatRatesGetTicker(&u) 406 if err != nil { 407 return false, err 408 } 409 if ticker == nil { 410 if token != "" { // if the base currency is not found in DB, do not create ticker for the token 411 if !warningLogged { 412 glog.Warningf("No base currency ticker for date %v for token %s", u, token) 413 warningLogged = true 414 } 415 continue 416 } 417 ticker = &common.CurrencyRatesTicker{ 418 Timestamp: u, 419 Rates: make(map[string]float32), 420 } 421 } 422 tickersToUpdate[timestamp] = ticker 423 } 424 if token == "" { 425 ticker.Rates[vsCurrency] = rate 426 } else { 427 if ticker.TokenRates == nil { 428 ticker.TokenRates = make(map[string]float32) 429 } 430 ticker.TokenRates[token] = rate 431 } 432 } 433 } 434 return true, nil 435 } 436 437 func (cg *Coingecko) storeTickers(tickersToUpdate map[uint]*common.CurrencyRatesTicker) error { 438 if len(tickersToUpdate) > 0 { 439 wb := grocksdb.NewWriteBatch() 440 defer wb.Destroy() 441 for _, v := range tickersToUpdate { 442 if err := cg.db.FiatRatesStoreTicker(wb, v); err != nil { 443 return err 444 } 445 } 446 if err := cg.db.WriteBatch(wb); err != nil { 447 return err 448 } 449 } 450 return nil 451 } 452 453 func (cg *Coingecko) throttleHistoricalDownload() { 454 // long delay next request to avoid throttling if downloading current tickers at the same time 455 delay := 1 456 if cg.updatingCurrent { 457 delay = 600 458 } 459 time.Sleep(cg.throttlingDelay * time.Duration(delay)) 460 } 461 462 // UpdateHistoricalTickers gets historical tickers for the main crypto currency 463 func (cg *Coingecko) UpdateHistoricalTickers() error { 464 tickersToUpdate := make(map[uint]*common.CurrencyRatesTicker) 465 466 // reload vs_currencies 467 vs, err := cg.simpleSupportedVSCurrencies() 468 if err != nil { 469 return err 470 } 471 vsCurrencies = vs 472 473 for _, currency := range vsCurrencies { 474 // get historical rates for each currency 475 var err error 476 var req bool 477 if req, err = cg.getHistoricalTicker(tickersToUpdate, cg.coin, currency, ""); err != nil { 478 // report error and continue, Coingecko may return error like "Could not find coin with the given id" 479 // the rates will be updated next run 480 glog.Errorf("getHistoricalTicker %s-%s %v", cg.coin, currency, err) 481 } 482 if req { 483 cg.throttleHistoricalDownload() 484 } 485 } 486 487 return cg.storeTickers(tickersToUpdate) 488 } 489 490 // UpdateHistoricalTokenTickers gets historical tickers for the tokens 491 func (cg *Coingecko) UpdateHistoricalTokenTickers() error { 492 if cg.updatingTokens { 493 return nil 494 } 495 cg.updatingTokens = true 496 defer func() { cg.updatingTokens = false }() 497 tickersToUpdate := make(map[uint]*common.CurrencyRatesTicker) 498 499 if cg.platformIdentifier != "" && cg.platformVsCurrency != "" { 500 // reload platform ids 501 if err := cg.platformIds(); err != nil { 502 return err 503 } 504 glog.Infof("Coingecko returned %d %s tokens ", len(platformIds), cg.coin) 505 count := 0 506 // get token historical rates 507 for tokenId, token := range platformIdsToTokens { 508 var err error 509 var req bool 510 if req, err = cg.getHistoricalTicker(tickersToUpdate, tokenId, cg.platformVsCurrency, token); err != nil { 511 // report error and continue, Coingecko may return error like "Could not find coin with the given id" 512 // the rates will be updated next run 513 glog.Errorf("getHistoricalTicker %s-%s %v", tokenId, cg.platformVsCurrency, err) 514 } 515 count++ 516 if count%100 == 0 { 517 err := cg.storeTickers(tickersToUpdate) 518 if err != nil { 519 return err 520 } 521 tickersToUpdate = make(map[uint]*common.CurrencyRatesTicker) 522 glog.Infof("Coingecko updated %d of %d token tickers", count, len(platformIds)) 523 } 524 if req { 525 cg.throttleHistoricalDownload() 526 } 527 } 528 } 529 530 return cg.storeTickers(tickersToUpdate) 531 }