github.com/diadata-org/diadata@v1.4.593/pkg/dia/scraper/foreign-scrapers/CoingeckoScraper.go (about)

     1  package foreignscrapers
     2  
     3  // Please implement the scraping of coingecko quotations here.
     4  
     5  import (
     6  	"encoding/json"
     7  	"errors"
     8  	"strconv"
     9  	"strings"
    10  	"time"
    11  
    12  	models "github.com/diadata-org/diadata/pkg/model"
    13  	"github.com/diadata-org/diadata/pkg/utils"
    14  	log "github.com/sirupsen/logrus"
    15  )
    16  
    17  const (
    18  	source = "Coingecko"
    19  )
    20  
    21  var (
    22  	coingeckoUpdateSeconds int64
    23  )
    24  
    25  type CoingeckoCoin struct {
    26  	ID                 string  `json:"id"`
    27  	Symbol             string  `json:"symbol"`
    28  	Name               string  `json:"name"`
    29  	CurrentPrice       float64 `json:"current_price"`
    30  	Yesterday24hVolume float64 `json:"total_volume"`
    31  	LastUpdated        string  `json:"last_updated"`
    32  }
    33  
    34  type nothing struct{}
    35  
    36  type CoingeckoScraper struct {
    37  	ticker          *time.Ticker
    38  	foreignScrapper ForeignScraper
    39  	apiKey          string
    40  	apiSecret       string
    41  }
    42  
    43  func init() {
    44  	var err error
    45  	coingeckoUpdateSeconds, err = strconv.ParseInt(utils.Getenv("COINGECKO_UPDATE_SECONDS", "14400"), 10, 64)
    46  	if err != nil {
    47  		log.Error("Parse COINGECKO_UPDATE_SECONDS: ", err)
    48  	}
    49  }
    50  
    51  func NewCoingeckoScraper(datastore models.Datastore, apiKey string, apiSecret string) *CoingeckoScraper {
    52  
    53  	foreignScrapper := ForeignScraper{
    54  		shutdown:      make(chan nothing),
    55  		error:         nil,
    56  		datastore:     datastore,
    57  		chanQuotation: make(chan *models.ForeignQuotation),
    58  	}
    59  	s := &CoingeckoScraper{
    60  		ticker:          time.NewTicker(time.Duration(coingeckoUpdateSeconds) * time.Second),
    61  		foreignScrapper: foreignScrapper,
    62  		apiKey:          apiKey,
    63  		apiSecret:       apiSecret,
    64  	}
    65  	go s.mainLoop()
    66  
    67  	return s
    68  
    69  }
    70  
    71  // mainLoop runs in a goroutine until channel s is closed.
    72  func (scraper *CoingeckoScraper) mainLoop() {
    73  
    74  	// Initial run.
    75  	err := scraper.UpdateQuotation()
    76  	if err != nil {
    77  		log.Error(err)
    78  	}
    79  
    80  	// Update every @coingeckoUpdateSeconds.
    81  	for {
    82  		select {
    83  		case <-scraper.ticker.C:
    84  			err := scraper.UpdateQuotation()
    85  			if err != nil {
    86  				log.Error(err)
    87  			}
    88  		case <-scraper.foreignScrapper.shutdown: // user requested shutdown
    89  			log.Printf("Coingeckoscraper shutting down")
    90  			return
    91  		}
    92  	}
    93  
    94  }
    95  
    96  // Update retrieves new coin information from the coingecko API and stores it to influx
    97  func (scraper *CoingeckoScraper) UpdateQuotation() error {
    98  	log.Printf("Executing CoingeckoScraper update")
    99  
   100  	coins, err := scraper.getCoingeckoData()
   101  
   102  	if err != nil {
   103  		log.Errorln("Error getting data from coingecko", err)
   104  		return err
   105  	}
   106  
   107  	for _, coin := range coins {
   108  
   109  		// TO DO: normalize symbol instead of all upper
   110  		coin.Symbol = strings.ToUpper(coin.Symbol)
   111  
   112  		// Parse last updated timestamp
   113  		layout := "2006-01-02T15:04:05.000Z"
   114  		timestamp, err := time.Parse(layout, coin.LastUpdated)
   115  		if err != nil {
   116  			log.Errorln("error parsing time")
   117  		}
   118  
   119  		// Get yesterday's price from influx if available
   120  		priceYesterday, err := scraper.foreignScrapper.datastore.GetForeignPriceYesterday(coin.Symbol, source)
   121  		if err != nil {
   122  			priceYesterday = 0
   123  		}
   124  
   125  		foreignQuotation := models.ForeignQuotation{
   126  			Symbol:             coin.Symbol,
   127  			Name:               coin.Name,
   128  			Price:              coin.CurrentPrice,
   129  			PriceYesterday:     priceYesterday,
   130  			VolumeYesterdayUSD: coin.Yesterday24hVolume,
   131  			Source:             source,
   132  			Time:               timestamp,
   133  		}
   134  		scraper.foreignScrapper.chanQuotation <- &foreignQuotation
   135  	}
   136  	return nil
   137  
   138  }
   139  
   140  func (scraper *CoingeckoScraper) GetQuoteChannel() chan *models.ForeignQuotation {
   141  	return scraper.foreignScrapper.chanQuotation
   142  }
   143  
   144  func (scraper *CoingeckoScraper) getCoingeckoData() (coins []CoingeckoCoin, err error) {
   145  	var response []byte
   146  	if scraper.apiKey != "" {
   147  		baseString := "https://pro-api.coingecko.com/api/v3/coins/markets?vs_currency=usd&order=market_cap_desc&per_page=250&page=1&sparkline=false&x_cg_pro_api_key=" + scraper.apiKey
   148  		response, _, err = utils.GetRequest(baseString)
   149  
   150  	} else {
   151  		response, _, err = utils.GetRequest("https://api.coingecko.com/api/v3/coins/markets?vs_currency=usd&order=market_cap_desc&per_page=250&page=1&sparkline=false")
   152  	}
   153  	if err != nil {
   154  		return
   155  	}
   156  	err = json.Unmarshal(response, &coins)
   157  	if err != nil {
   158  		return
   159  	}
   160  	return
   161  }
   162  
   163  // Close closes any existing API connections
   164  func (scraper *CoingeckoScraper) Close() error {
   165  	if scraper.foreignScrapper.closed {
   166  		return errors.New("scraper already closed")
   167  	}
   168  	close(scraper.foreignScrapper.shutdown)
   169  	<-scraper.foreignScrapper.shutdownDone
   170  	scraper.foreignScrapper.errorLock.RLock()
   171  	defer scraper.foreignScrapper.errorLock.RUnlock()
   172  	return scraper.foreignScrapper.error
   173  }