github.com/trezor/blockbook@v0.4.1-0.20240328132726-e9a08582ee2c/fiat/fiat_rates.go (about) 1 package fiat 2 3 import ( 4 "encoding/json" 5 "errors" 6 "fmt" 7 "math/rand" 8 "strings" 9 "sync" 10 "time" 11 12 "github.com/golang/glog" 13 "github.com/trezor/blockbook/common" 14 "github.com/trezor/blockbook/db" 15 ) 16 17 const currentTickersKey = "CurrentTickers" 18 const hourlyTickersKey = "HourlyTickers" 19 const fiveMinutesTickersKey = "FiveMinutesTickers" 20 21 const highGranularityVsCurrency = "usd" 22 23 const secondsInDay = 24 * 60 * 60 24 const secondsInHour = 60 * 60 25 const secondsInFiveMinutes = 5 * 60 26 27 // OnNewFiatRatesTicker is used to send notification about a new FiatRates ticker 28 type OnNewFiatRatesTicker func(ticker *common.CurrencyRatesTicker) 29 30 // RatesDownloaderInterface provides method signatures for a specific fiat rates downloader 31 type RatesDownloaderInterface interface { 32 CurrentTickers() (*common.CurrencyRatesTicker, error) 33 HourlyTickers() (*[]common.CurrencyRatesTicker, error) 34 FiveMinutesTickers() (*[]common.CurrencyRatesTicker, error) 35 UpdateHistoricalTickers() error 36 UpdateHistoricalTokenTickers() error 37 } 38 39 // FiatRates is used to fetch and refresh fiat rates 40 type FiatRates struct { 41 Enabled bool 42 periodSeconds int64 43 db *db.RocksDB 44 timeFormat string 45 callbackOnNewTicker OnNewFiatRatesTicker 46 downloader RatesDownloaderInterface 47 downloadTokens bool 48 provider string 49 allowedVsCurrencies string 50 mux sync.RWMutex 51 currentTicker *common.CurrencyRatesTicker 52 hourlyTickers map[int64]*common.CurrencyRatesTicker 53 hourlyTickersFrom int64 54 hourlyTickersTo int64 55 fiveMinutesTickers map[int64]*common.CurrencyRatesTicker 56 fiveMinutesTickersFrom int64 57 fiveMinutesTickersTo int64 58 dailyTickers map[int64]*common.CurrencyRatesTicker 59 dailyTickersFrom int64 60 dailyTickersTo int64 61 } 62 63 // NewFiatRates initializes the FiatRates handler 64 func NewFiatRates(db *db.RocksDB, config *common.Config, metrics *common.Metrics, callback OnNewFiatRatesTicker) (*FiatRates, error) { 65 66 var fr = &FiatRates{ 67 provider: config.FiatRates, 68 allowedVsCurrencies: config.FiatRatesVsCurrencies, 69 } 70 71 if config.FiatRates == "" || config.FiatRatesParams == "" { 72 glog.Infof("FiatRates config is empty, not downloading fiat rates") 73 fr.Enabled = false 74 return fr, nil 75 } 76 77 type fiatRatesParams struct { 78 URL string `json:"url"` 79 Coin string `json:"coin"` 80 PlatformIdentifier string `json:"platformIdentifier"` 81 PlatformVsCurrency string `json:"platformVsCurrency"` 82 PeriodSeconds int64 `json:"periodSeconds"` 83 } 84 rdParams := &fiatRatesParams{} 85 err := json.Unmarshal([]byte(config.FiatRatesParams), &rdParams) 86 if err != nil { 87 return nil, err 88 } 89 if rdParams.PeriodSeconds == 0 { 90 return nil, errors.New("missing parameters") 91 } 92 fr.timeFormat = "02-01-2006" // Layout string for FiatRates date formatting (DD-MM-YYYY) 93 fr.periodSeconds = rdParams.PeriodSeconds // Time period for syncing the latest market data 94 if fr.periodSeconds < 60 { // minimum is one minute 95 fr.periodSeconds = 60 96 } 97 fr.db = db 98 fr.callbackOnNewTicker = callback 99 fr.downloadTokens = rdParams.PlatformIdentifier != "" && rdParams.PlatformVsCurrency != "" 100 if fr.downloadTokens { 101 common.TickerRecalculateTokenRate = strings.ToLower(db.GetInternalState().CoinShortcut) != rdParams.PlatformVsCurrency 102 common.TickerTokenVsCurrency = rdParams.PlatformVsCurrency 103 } 104 is := fr.db.GetInternalState() 105 if fr.provider == "coingecko" { 106 throttle := true 107 if callback == nil { 108 // a small hack - in tests the callback is not used, therefore there is no delay slowing down the test 109 throttle = false 110 } 111 fr.downloader = NewCoinGeckoDownloader(db, rdParams.URL, rdParams.Coin, rdParams.PlatformIdentifier, rdParams.PlatformVsCurrency, fr.allowedVsCurrencies, fr.timeFormat, metrics, throttle) 112 if is != nil { 113 is.HasFiatRates = true 114 is.HasTokenFiatRates = fr.downloadTokens 115 fr.Enabled = true 116 117 if err := fr.loadDailyTickers(); err != nil { 118 return nil, err 119 } 120 121 currentTickers, err := db.FiatRatesGetSpecialTickers(currentTickersKey) 122 if err != nil { 123 glog.Error("FiatRatesDownloader: get CurrentTickers from DB error ", err) 124 } 125 if currentTickers != nil && len(*currentTickers) > 0 { 126 fr.currentTicker = &(*currentTickers)[0] 127 } 128 129 hourlyTickers, err := db.FiatRatesGetSpecialTickers(hourlyTickersKey) 130 if err != nil { 131 glog.Error("FiatRatesDownloader: get HourlyTickers from DB error ", err) 132 } 133 fr.hourlyTickers, fr.hourlyTickersFrom, fr.hourlyTickersTo = fr.tickersToMap(hourlyTickers, secondsInHour) 134 135 fiveMinutesTickers, err := db.FiatRatesGetSpecialTickers(fiveMinutesTickersKey) 136 if err != nil { 137 glog.Error("FiatRatesDownloader: get FiveMinutesTickers from DB error ", err) 138 } 139 fr.fiveMinutesTickers, fr.fiveMinutesTickersFrom, fr.fiveMinutesTickersTo = fr.tickersToMap(fiveMinutesTickers, secondsInFiveMinutes) 140 141 } 142 } else { 143 return nil, fmt.Errorf("unknown provider %q", fr.provider) 144 } 145 fr.logTickersInfo() 146 return fr, nil 147 } 148 149 // GetCurrentTicker returns current ticker 150 func (fr *FiatRates) GetCurrentTicker(vsCurrency string, token string) *common.CurrencyRatesTicker { 151 fr.mux.RLock() 152 currentTicker := fr.currentTicker 153 fr.mux.RUnlock() 154 if currentTicker != nil && common.IsSuitableTicker(currentTicker, vsCurrency, token) { 155 return currentTicker 156 } 157 return nil 158 } 159 160 // getTokenTickersForTimestamps returns tickers for slice of timestamps, that contain requested vsCurrency and token 161 func (fr *FiatRates) getTokenTickersForTimestamps(timestamps []int64, vsCurrency string, token string) (*[]*common.CurrencyRatesTicker, error) { 162 currentTicker := fr.GetCurrentTicker("", token) 163 tickers := make([]*common.CurrencyRatesTicker, len(timestamps)) 164 var prevTicker *common.CurrencyRatesTicker 165 var prevTs int64 166 var err error 167 for i, t := range timestamps { 168 // check if the token is available in the current ticker - if not, return nil ticker instead of wasting time in costly DB searches 169 if currentTicker != nil { 170 var ticker *common.CurrencyRatesTicker 171 date := time.Unix(t, 0) 172 // if previously found ticker is newer than this one (token tickers may not be in DB for every day), skip search in DB 173 if prevTicker != nil && t >= prevTs && !date.After(prevTicker.Timestamp) { 174 ticker = prevTicker 175 prevTs = t 176 } else { 177 ticker, err = fr.db.FiatRatesFindTicker(&date, vsCurrency, token) 178 if err != nil { 179 return nil, err 180 } 181 prevTicker = ticker 182 prevTs = t 183 } 184 // if ticker not found in DB, use current ticker 185 if ticker == nil { 186 tickers[i] = currentTicker 187 prevTicker = currentTicker 188 prevTs = t 189 } else { 190 tickers[i] = ticker 191 } 192 } 193 } 194 return &tickers, nil 195 } 196 197 // GetTickersForTimestamps returns tickers for slice of timestamps, that contain requested vsCurrency and token 198 func (fr *FiatRates) GetTickersForTimestamps(timestamps []int64, vsCurrency string, token string) (*[]*common.CurrencyRatesTicker, error) { 199 if !fr.Enabled { 200 return nil, nil 201 } 202 // token rates are not in memory, them load from DB 203 if token != "" { 204 return fr.getTokenTickersForTimestamps(timestamps, vsCurrency, token) 205 } 206 fr.mux.RLock() 207 defer fr.mux.RUnlock() 208 tickers := make([]*common.CurrencyRatesTicker, len(timestamps)) 209 var prevTicker *common.CurrencyRatesTicker 210 var prevTs int64 211 for i, t := range timestamps { 212 dailyTs := ceilUnix(t, secondsInDay) 213 // use higher granularity only for non daily timestamps 214 if t != dailyTs { 215 if t >= fr.fiveMinutesTickersFrom && t <= fr.fiveMinutesTickersTo { 216 if ticker, found := fr.fiveMinutesTickers[ceilUnix(t, secondsInFiveMinutes)]; found && ticker != nil { 217 if common.IsSuitableTicker(ticker, vsCurrency, token) { 218 tickers[i] = ticker 219 continue 220 } 221 } 222 } 223 if t >= fr.hourlyTickersFrom && t <= fr.hourlyTickersTo { 224 if ticker, found := fr.hourlyTickers[ceilUnix(t, secondsInHour)]; found && ticker != nil { 225 if common.IsSuitableTicker(ticker, vsCurrency, token) { 226 tickers[i] = ticker 227 continue 228 } 229 } 230 } 231 } 232 if prevTicker != nil && t >= prevTs && t <= prevTicker.Timestamp.Unix() { 233 tickers[i] = prevTicker 234 continue 235 } else { 236 var found bool 237 if dailyTs < fr.dailyTickersFrom { 238 dailyTs = fr.dailyTickersFrom 239 } 240 var ticker *common.CurrencyRatesTicker 241 for ; dailyTs <= fr.dailyTickersTo; dailyTs += secondsInDay { 242 if ticker, found = fr.dailyTickers[dailyTs]; found && ticker != nil { 243 if common.IsSuitableTicker(ticker, vsCurrency, token) { 244 tickers[i] = ticker 245 prevTicker = ticker 246 prevTs = t 247 break 248 } else { 249 found = false 250 } 251 } 252 } 253 if !found { 254 tickers[i] = fr.currentTicker 255 prevTicker = fr.currentTicker 256 prevTs = t 257 } 258 } 259 } 260 return &tickers, nil 261 } 262 func (fr *FiatRates) logTickersInfo() { 263 glog.Infof("fiat rates %s handler, %d (%s - %s) daily tickers, %d (%s - %s) hourly tickers, %d (%s - %s) 5 minute tickers", fr.provider, 264 len(fr.dailyTickers), time.Unix(fr.dailyTickersFrom, 0).Format("2006-01-02"), time.Unix(fr.dailyTickersTo, 0).Format("2006-01-02"), 265 len(fr.hourlyTickers), time.Unix(fr.hourlyTickersFrom, 0).Format("2006-01-02 15:04"), time.Unix(fr.hourlyTickersTo, 0).Format("2006-01-02 15:04"), 266 len(fr.fiveMinutesTickers), time.Unix(fr.fiveMinutesTickersFrom, 0).Format("2006-01-02 15:04"), time.Unix(fr.fiveMinutesTickersTo, 0).Format("2006-01-02 15:04")) 267 } 268 269 func roundTimeUnix(t time.Time, granularity int64) int64 { 270 return roundUnix(t.UTC().Unix(), granularity) 271 } 272 273 func roundUnix(t int64, granularity int64) int64 { 274 unix := t + (granularity >> 1) 275 return unix - unix%granularity 276 } 277 278 func ceilUnix(t int64, granularity int64) int64 { 279 unix := t + (granularity - 1) 280 return unix - unix%granularity 281 } 282 283 // loadDailyTickers loads daily tickers to cache 284 func (fr *FiatRates) loadDailyTickers() error { 285 fr.mux.Lock() 286 defer fr.mux.Unlock() 287 fr.dailyTickers = make(map[int64]*common.CurrencyRatesTicker) 288 err := fr.db.FiatRatesGetAllTickers(func(ticker *common.CurrencyRatesTicker) error { 289 normalizedTime := roundTimeUnix(ticker.Timestamp, secondsInDay) 290 if normalizedTime == fr.dailyTickersFrom { 291 // there are multiple tickers on the first day, use only the first one 292 return nil 293 } 294 // remove token rates from cache to save memory (tickers with token rates are hundreds of kb big) 295 ticker.TokenRates = nil 296 if len(fr.dailyTickers) > 0 { 297 // check that there is a ticker for every day, if missing, set it from current value if missing 298 prevTime := normalizedTime 299 for { 300 prevTime -= secondsInDay 301 if _, found := fr.dailyTickers[prevTime]; found { 302 break 303 } 304 fr.dailyTickers[prevTime] = ticker 305 } 306 } else { 307 fr.dailyTickersFrom = normalizedTime 308 } 309 fr.dailyTickers[normalizedTime] = ticker 310 fr.dailyTickersTo = normalizedTime 311 return nil 312 }) 313 return err 314 } 315 316 // setCurrentTicker sets current ticker 317 func (fr *FiatRates) setCurrentTicker(t *common.CurrencyRatesTicker) { 318 fr.mux.Lock() 319 defer fr.mux.Unlock() 320 fr.currentTicker = t 321 fr.db.FiatRatesStoreSpecialTickers(currentTickersKey, &[]common.CurrencyRatesTicker{*t}) 322 } 323 324 func (fr *FiatRates) tickersToMap(tickers *[]common.CurrencyRatesTicker, granularitySeconds int64) (map[int64]*common.CurrencyRatesTicker, int64, int64) { 325 if tickers == nil || len(*tickers) == 0 { 326 return make(map[int64]*common.CurrencyRatesTicker), 0, 0 327 } 328 m := make(map[int64]*common.CurrencyRatesTicker, len(*tickers)) 329 from := int64(0) 330 to := int64(0) 331 for i := range *tickers { 332 ticker := (*tickers)[i] 333 normalizedTime := roundTimeUnix(ticker.Timestamp, granularitySeconds) 334 dailyTime := roundTimeUnix(ticker.Timestamp, secondsInDay) 335 dailyTicker, found := fr.dailyTickers[dailyTime] 336 if !found { 337 // if not found in historical tickers, use current ticker 338 dailyTicker = fr.currentTicker 339 } 340 if dailyTicker != nil { 341 // high granularity tickers are loaded only in one currency, add other currencies based on daily rate between fiat currencies 342 vsRate, foundVs := ticker.Rates[highGranularityVsCurrency] 343 dailyVsRate, foundDaily := dailyTicker.Rates[highGranularityVsCurrency] 344 if foundDaily && dailyVsRate != 0 && foundVs && vsRate != 0 { 345 for currency, rate := range dailyTicker.Rates { 346 if currency != highGranularityVsCurrency { 347 ticker.Rates[currency] = vsRate * rate / dailyVsRate 348 } 349 } 350 } 351 } 352 if len(m) > 0 { 353 if normalizedTime == from { 354 // there are multiple normalized tickers for the first entry, skip 355 continue 356 } 357 // check that there is a ticker for each period, set it from current value if missing 358 prevTime := normalizedTime 359 for { 360 prevTime -= granularitySeconds 361 if _, found := m[prevTime]; found { 362 break 363 } 364 m[prevTime] = &ticker 365 } 366 } else { 367 from = normalizedTime 368 } 369 m[normalizedTime] = &ticker 370 to = normalizedTime 371 } 372 return m, from, to 373 } 374 375 // setCurrentTicker sets hourly tickers 376 func (fr *FiatRates) setHourlyTickers(t *[]common.CurrencyRatesTicker) { 377 fr.db.FiatRatesStoreSpecialTickers(hourlyTickersKey, t) 378 fr.mux.Lock() 379 defer fr.mux.Unlock() 380 fr.hourlyTickers, fr.hourlyTickersFrom, fr.hourlyTickersTo = fr.tickersToMap(t, secondsInHour) 381 } 382 383 // setCurrentTicker sets hourly tickers 384 func (fr *FiatRates) setFiveMinutesTickers(t *[]common.CurrencyRatesTicker) { 385 fr.db.FiatRatesStoreSpecialTickers(fiveMinutesTickersKey, t) 386 fr.mux.Lock() 387 defer fr.mux.Unlock() 388 fr.fiveMinutesTickers, fr.fiveMinutesTickersFrom, fr.fiveMinutesTickersTo = fr.tickersToMap(t, secondsInFiveMinutes) 389 } 390 391 // RunDownloader periodically downloads current (every 15 minutes) and historical (once a day) tickers 392 func (fr *FiatRates) RunDownloader() error { 393 glog.Infof("Starting %v FiatRates downloader...", fr.provider) 394 var lastHistoricalTickers time.Time 395 is := fr.db.GetInternalState() 396 tickerFromIs := fr.GetCurrentTicker("", "") 397 firstRun := true 398 for { 399 unix := time.Now().Unix() 400 next := unix + fr.periodSeconds 401 next -= next % fr.periodSeconds 402 // skip waiting for the period for the first run if there are no tickerFromIs or they are too old 403 if !firstRun || (tickerFromIs != nil && next-tickerFromIs.Timestamp.Unix() < fr.periodSeconds) { 404 // wait for the next run with a slight random value to avoid too many request at the same time 405 next += int64(rand.Intn(12)) 406 time.Sleep(time.Duration(next-unix) * time.Second) 407 } 408 firstRun = false 409 currentTicker, err := fr.downloader.CurrentTickers() 410 if err != nil || currentTicker == nil { 411 glog.Error("FiatRatesDownloader: CurrentTickers error ", err) 412 } else { 413 fr.setCurrentTicker(currentTicker) 414 glog.Info("FiatRatesDownloader: CurrentTickers updated") 415 if fr.callbackOnNewTicker != nil { 416 fr.callbackOnNewTicker(currentTicker) 417 } 418 } 419 hourlyTickers, err := fr.downloader.HourlyTickers() 420 if err != nil || hourlyTickers == nil { 421 glog.Error("FiatRatesDownloader: HourlyTickers error ", err) 422 } else { 423 fr.setHourlyTickers(hourlyTickers) 424 glog.Info("FiatRatesDownloader: HourlyTickers updated") 425 } 426 fiveMinutesTickers, err := fr.downloader.FiveMinutesTickers() 427 if err != nil || fiveMinutesTickers == nil { 428 glog.Error("FiatRatesDownloader: FiveMinutesTickers error ", err) 429 } else { 430 fr.setFiveMinutesTickers(fiveMinutesTickers) 431 glog.Info("FiatRatesDownloader: FiveMinutesTickers updated") 432 } 433 now := time.Now().UTC() 434 // once a day, 1 hour after UTC midnight (to let the provider prepare historical rates) update historical tickers 435 if (now.YearDay() != lastHistoricalTickers.YearDay() || now.Year() != lastHistoricalTickers.Year()) && now.Hour() > 0 { 436 err = fr.downloader.UpdateHistoricalTickers() 437 if err != nil { 438 glog.Error("FiatRatesDownloader: UpdateHistoricalTickers error ", err) 439 } else { 440 lastHistoricalTickers = time.Now().UTC() 441 if err = fr.loadDailyTickers(); err != nil { 442 glog.Error("FiatRatesDownloader: loadDailyTickers error ", err) 443 } else { 444 ticker, found := fr.dailyTickers[fr.dailyTickersTo] 445 if !found || ticker == nil { 446 glog.Error("FiatRatesDownloader: dailyTickers not loaded") 447 } else { 448 glog.Infof("FiatRatesDownloader: UpdateHistoricalTickers finished, last ticker from %v", ticker.Timestamp) 449 fr.logTickersInfo() 450 if is != nil { 451 is.HistoricalFiatRatesTime = ticker.Timestamp 452 } 453 } 454 } 455 if fr.downloadTokens { 456 // UpdateHistoricalTokenTickers in a goroutine, it can take quite some time as there are many tokens 457 go func() { 458 err := fr.downloader.UpdateHistoricalTokenTickers() 459 if err != nil { 460 glog.Error("FiatRatesDownloader: UpdateHistoricalTokenTickers error ", err) 461 } else { 462 glog.Info("FiatRatesDownloader: UpdateHistoricalTokenTickers finished") 463 if is != nil { 464 is.HistoricalTokenFiatRatesTime = time.Now().UTC() 465 } 466 } 467 }() 468 } 469 } 470 } 471 } 472 }