github.com/bchainhub/blockbook@v0.3.2/fiat/fiat_rates.go (about) 1 package fiat 2 3 import ( 4 "blockbook/db" 5 "encoding/json" 6 "errors" 7 "fmt" 8 "reflect" 9 "time" 10 11 "github.com/golang/glog" 12 ) 13 14 // OnNewFiatRatesTicker is used to send notification about a new FiatRates ticker 15 type OnNewFiatRatesTicker func(ticker *db.CurrencyRatesTicker) 16 17 // RatesDownloaderInterface provides method signatures for specific fiat rates downloaders 18 type RatesDownloaderInterface interface { 19 getTicker(timestamp *time.Time) (*db.CurrencyRatesTicker, error) 20 marketDataExists(timestamp *time.Time) (bool, error) 21 } 22 23 // RatesDownloader stores FiatRates API parameters 24 type RatesDownloader struct { 25 periodSeconds time.Duration 26 db *db.RocksDB 27 startTime *time.Time // a starting timestamp for tests to be deterministic (time.Now() for production) 28 timeFormat string 29 callbackOnNewTicker OnNewFiatRatesTicker 30 downloader RatesDownloaderInterface 31 } 32 33 // NewFiatRatesDownloader initiallizes the downloader for FiatRates API. 34 // If the startTime is nil, the downloader will start from the beginning. 35 func NewFiatRatesDownloader(db *db.RocksDB, apiType string, params string, startTime *time.Time, callback OnNewFiatRatesTicker) (*RatesDownloader, error) { 36 var rd = &RatesDownloader{} 37 type fiatRatesParams struct { 38 URL string `json:"url"` 39 Coin string `json:"coin"` 40 PeriodSeconds int `json:"periodSeconds"` 41 } 42 rdParams := &fiatRatesParams{} 43 err := json.Unmarshal([]byte(params), &rdParams) 44 if err != nil { 45 return nil, err 46 } 47 if rdParams.URL == "" || rdParams.PeriodSeconds == 0 { 48 return nil, errors.New("Missing parameters") 49 } 50 rd.timeFormat = "02-01-2006" // Layout string for FiatRates date formatting (DD-MM-YYYY) 51 rd.periodSeconds = time.Duration(rdParams.PeriodSeconds) * time.Second // Time period for syncing the latest market data 52 rd.db = db 53 rd.callbackOnNewTicker = callback 54 if startTime == nil { 55 timeNow := time.Now().UTC() 56 rd.startTime = &timeNow 57 } else { 58 rd.startTime = startTime // If startTime is nil, time.Now() will be used 59 } 60 if apiType == "coingecko" { 61 rd.downloader = NewCoinGeckoDownloader(rdParams.URL, rdParams.Coin, rd.timeFormat) 62 } else { 63 return nil, fmt.Errorf("NewFiatRatesDownloader: incorrect API type %q", apiType) 64 } 65 return rd, nil 66 } 67 68 // Run starts the FiatRates downloader. If there are tickers available, it continues from the last record. 69 // If there are no tickers, it finds the earliest market data available on API and downloads historical data. 70 // When historical data is downloaded, it continues to fetch the latest ticker prices. 71 func (rd *RatesDownloader) Run() error { 72 var timestamp *time.Time 73 74 // Check if there are any tickers stored in database 75 glog.Infof("Finding last available ticker...") 76 ticker, err := rd.db.FiatRatesFindLastTicker() 77 if err != nil { 78 glog.Errorf("RatesDownloader FindTicker error: %v", err) 79 return err 80 } 81 82 if ticker == nil { 83 // If no tickers found, start downloading from the beginning 84 glog.Infof("No tickers found! Looking up the earliest market data available on API and downloading from there.") 85 timestamp, err = rd.findEarliestMarketData() 86 if err != nil { 87 glog.Errorf("Error looking up earliest market data: %v", err) 88 return err 89 } 90 } else { 91 // If found, continue downloading data from the next day of the last available record 92 glog.Infof("Last available ticker: %v", ticker.Timestamp) 93 timestamp = ticker.Timestamp 94 } 95 err = rd.syncHistorical(timestamp) 96 if err != nil { 97 glog.Errorf("RatesDownloader syncHistorical error: %v", err) 98 return err 99 } 100 if err := rd.syncLatest(); err != nil { 101 glog.Errorf("RatesDownloader syncLatest error: %v", err) 102 return err 103 } 104 return nil 105 } 106 107 // FindEarliestMarketData uses binary search to find the oldest market data available on API. 108 func (rd *RatesDownloader) findEarliestMarketData() (*time.Time, error) { 109 minDateString := "03-01-2009" 110 minDate, err := time.Parse(rd.timeFormat, minDateString) 111 if err != nil { 112 glog.Error("Error parsing date: ", err) 113 return nil, err 114 } 115 maxDate := rd.startTime.Add(time.Duration(-24) * time.Hour) // today's historical tickers may not be ready yet, so set to yesterday 116 currentDate := maxDate 117 for { 118 var dataExists bool = false 119 for { 120 dataExists, err = rd.downloader.marketDataExists(¤tDate) 121 if err != nil { 122 glog.Errorf("Error checking if market data exists for date %v. Error: %v. Retrying in %v seconds.", currentDate, err, rd.periodSeconds) 123 timer := time.NewTimer(rd.periodSeconds) 124 <-timer.C 125 } 126 break 127 } 128 dateDiff := currentDate.Sub(minDate) 129 if dataExists { 130 if dateDiff < time.Hour*24 { 131 maxDate := time.Date(maxDate.Year(), maxDate.Month(), maxDate.Day(), 0, 0, 0, 0, maxDate.Location()) // truncate time to day 132 return &maxDate, nil 133 } 134 maxDate = currentDate 135 currentDate = currentDate.Add(-1 * dateDiff / 2) 136 } else { 137 minDate = currentDate 138 currentDate = currentDate.Add(maxDate.Sub(currentDate) / 2) 139 } 140 } 141 } 142 143 // syncLatest downloads the latest FiatRates data every rd.PeriodSeconds 144 func (rd *RatesDownloader) syncLatest() error { 145 timer := time.NewTimer(rd.periodSeconds) 146 var lastTickerRates map[string]float64 147 sameTickerCounter := 0 148 for { 149 ticker, err := rd.downloader.getTicker(nil) 150 if err != nil { 151 // Do not exit on GET error, log it, wait and try again 152 glog.Errorf("syncLatest GetData error: %v", err) 153 <-timer.C 154 timer.Reset(rd.periodSeconds) 155 continue 156 } 157 158 if sameTickerCounter < 5 && reflect.DeepEqual(ticker.Rates, lastTickerRates) { 159 // If rates are the same as previous, do not store them 160 glog.Infof("syncLatest: ticker rates for %v are the same as previous, skipping...", ticker.Timestamp) 161 <-timer.C 162 timer.Reset(rd.periodSeconds) 163 sameTickerCounter++ 164 continue 165 } 166 lastTickerRates = ticker.Rates 167 sameTickerCounter = 0 168 169 glog.Infof("syncLatest: storing ticker for %v", ticker.Timestamp) 170 err = rd.db.FiatRatesStoreTicker(ticker) 171 if err != nil { 172 // If there's an error storing ticker (like missing rates), log it, wait and try again 173 glog.Errorf("syncLatest StoreTicker error: %v", err) 174 } else if rd.callbackOnNewTicker != nil { 175 rd.callbackOnNewTicker(ticker) 176 } 177 <-timer.C 178 timer.Reset(rd.periodSeconds) 179 } 180 } 181 182 // syncHistorical downloads all the historical data since the specified timestamp till today, 183 // then continues to download the latest rates 184 func (rd *RatesDownloader) syncHistorical(timestamp *time.Time) error { 185 period := time.Duration(1) * time.Second 186 timer := time.NewTimer(period) 187 for { 188 if rd.startTime.Sub(*timestamp) < time.Duration(time.Hour*24) { 189 break 190 } 191 192 ticker, err := rd.downloader.getTicker(timestamp) 193 if err != nil { 194 // Do not exit on GET error, log it, wait and try again 195 glog.Errorf("syncHistorical GetData error: %v", err) 196 <-timer.C 197 timer.Reset(rd.periodSeconds) 198 continue 199 } 200 201 glog.Infof("syncHistorical: storing ticker for %v", ticker.Timestamp) 202 err = rd.db.FiatRatesStoreTicker(ticker) 203 if err != nil { 204 // If there's an error storing ticker (like missing rates), log it and continue to the next day 205 glog.Errorf("syncHistorical error storing ticker for %v: %v", timestamp, err) 206 } 207 208 *timestamp = timestamp.Add(time.Hour * 24) // go to the next day 209 210 <-timer.C 211 timer.Reset(period) 212 } 213 return nil 214 }