github.com/diadata-org/diadata@v1.4.593/pkg/dia/scraper/historical-scrapers/CoinmarketcapScraper.go (about)

     1  package historicalscrapers
     2  
     3  import (
     4  	"encoding/json"
     5  	"errors"
     6  	"fmt"
     7  	"time"
     8  
     9  	log "github.com/sirupsen/logrus"
    10  
    11  	"net/http"
    12  
    13  	"github.com/diadata-org/diadata/pkg/dia"
    14  	models "github.com/diadata-org/diadata/pkg/model"
    15  	"github.com/diadata-org/diadata/pkg/utils"
    16  )
    17  
    18  const (
    19  	cmcDateLimit = time.Duration(89 * 24 * time.Hour)
    20  )
    21  
    22  type cmcQuote struct {
    23  	Quote struct {
    24  		Price     float64   `json:"price"`
    25  		Timestamp time.Time `json:"timestamp"`
    26  	} `json:"USD"`
    27  }
    28  
    29  type cmcFullQuote struct {
    30  	FullQuote cmcQuote  `json:"quote"`
    31  	Timestamp time.Time `json:"timestamp"`
    32  }
    33  
    34  type cmcQuotes struct {
    35  	Quotes []cmcFullQuote `json:"quotes"`
    36  	Symbol string         `json:"symbol"`
    37  }
    38  
    39  type cmcResponse struct {
    40  	Data cmcQuotes `json:"data"`
    41  }
    42  
    43  type CoinmarketcapScraper struct {
    44  	datastore        *models.DB
    45  	rdb              *models.RelDB
    46  	quotationChannel chan models.AssetQuotation
    47  	doneChannel      chan bool
    48  	dateLimit        time.Duration
    49  }
    50  
    51  func NewCoinmarketcapScraper(rdb *models.RelDB, datastore *models.DB) (cmcScraper CoinmarketcapScraper) {
    52  	cmcScraper.doneChannel = make(chan bool)
    53  	cmcScraper.quotationChannel = make(chan models.AssetQuotation)
    54  	cmcScraper.dateLimit = cmcDateLimit
    55  	cmcScraper.datastore = datastore
    56  	cmcScraper.rdb = rdb
    57  
    58  	go func() {
    59  		cmcScraper.FetchQuotations()
    60  	}()
    61  	return cmcScraper
    62  }
    63  
    64  func (s CoinmarketcapScraper) FetchQuotations() {
    65  
    66  	log.Info("Starting historical quotes scraper for Coinmarketcap")
    67  
    68  	ethAsset := dia.Asset{
    69  		Symbol:     "ETH",
    70  		Name:       "Ethereum",
    71  		Address:    "0x0000000000000000000000000000000000000000",
    72  		Decimals:   18,
    73  		Blockchain: "Ethereum",
    74  	}
    75  
    76  	cmcAPIKey := utils.Getenv("CMC_API_KEY", "")
    77  
    78  	// Get the timestamp of the last recorded quotation
    79  	endtime := time.Now()
    80  	starttime := endtime.Add(-s.dateLimit)
    81  	lastTimestamp, err := s.rdb.GetLastHistoricalQuotationTimestamp(ethAsset)
    82  	if err != nil {
    83  		log.Infof("No last time found for %s. Initialize with historical data.", ethAsset.Symbol)
    84  		quotes, err := fetchCMCQuotations(ethAsset.Symbol, starttime, endtime, cmcAPIKey)
    85  		if err != nil {
    86  			log.Errorf("Failed to fetch quotation from CMC API: %v", err)
    87  		}
    88  
    89  		for _, quote := range quotes.Data.Quotes {
    90  			historicalQuote := models.AssetQuotation{
    91  				Asset:  ethAsset,
    92  				Price:  quote.FullQuote.Quote.Price,
    93  				Source: "Coinmarketcap",
    94  				Time:   quote.FullQuote.Quote.Timestamp,
    95  			}
    96  			s.quotationChannel <- historicalQuote
    97  		}
    98  		s.doneChannel <- true
    99  	} else {
   100  		// TO DO: Otherwise try to fetch quotations from our DB. If too far in the past, fetch again from CMC.
   101  		for timestamp := lastTimestamp.Add(1 * time.Hour); timestamp.Before(endtime); timestamp = timestamp.Add(1 * time.Hour) {
   102  			quote, err := s.datastore.GetAssetQuotation(ethAsset, dia.CRYPTO_ZERO_UNIX_TIME, timestamp)
   103  			if err != nil {
   104  				log.Error("get asset quotation: ", err)
   105  			}
   106  
   107  			historicalQuote := models.AssetQuotation{
   108  				Asset:  ethAsset,
   109  				Price:  quote.Price,
   110  				Source: quote.Source,
   111  				Time:   quote.Time,
   112  			}
   113  
   114  			s.quotationChannel <- historicalQuote
   115  		}
   116  	}
   117  }
   118  
   119  func fetchCMCQuotations(assetSymbol string, starttime time.Time, endtime time.Time, apiKey string) (response cmcResponse, err error) {
   120  	log.Infof("starttime -- endtime: %v -- %v ", starttime, endtime)
   121  
   122  	url := fmt.Sprintf("https://pro-api.coinmarketcap.com/v1/cryptocurrency/quotes/historical?symbol=%s&interval=hourly&time_start=%d&time_end=%d", assetSymbol, starttime.Unix(), endtime.Unix())
   123  	req, err := http.NewRequest("GET", url, nil)
   124  	if err != nil {
   125  		return
   126  	}
   127  
   128  	req.Header.Set("X-CMC_PRO_API_KEY", apiKey)
   129  
   130  	resp, statusCode, err := utils.HTTPRequest(req)
   131  	if err != nil {
   132  		return
   133  	}
   134  	if statusCode != http.StatusOK {
   135  		err = errors.New("non-200 status code from Coinmarketcap API")
   136  		return
   137  	}
   138  
   139  	if err := json.Unmarshal(resp, &response); err != nil {
   140  		log.Error("")
   141  	}
   142  
   143  	return
   144  }
   145  
   146  func (s CoinmarketcapScraper) QuoteChannel() chan models.AssetQuotation {
   147  	return s.quotationChannel
   148  }
   149  
   150  func (s CoinmarketcapScraper) Done() chan bool {
   151  	return s.doneChannel
   152  }