github.com/diadata-org/diadata@v1.4.593/pkg/dia/scraper/foreign-scrapers/CoinMarketCapScraper.go

package foreignscrapers

import (
	"bufio"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	models "github.com/diadata-org/diadata/pkg/model"
	"github.com/diadata-org/diadata/pkg/utils"
	log "github.com/sirupsen/logrus"
)

const (
	coinMarketCapRefreshDelay = time.Second * 60 * 2
	coinMarketCapsource       = "CoinMarketCap"
)

type Quote struct {
	Price            float64 `json:"price"`
	Volume24H        float64 `json:"volume_24h"`
	PercentChange1H  float64 `json:"percent_change_1h"`
	PercentChange24H float64 `json:"percent_change_24h"`
	PercentChange7D  float64 `json:"percent_change_7d"`
	MarketCap        float64 `json:"market_cap"`
	LastUpdated      string  `json:"last_updated"`
}
type CoinMarketCapListing struct {
	Data []struct {
		ID                int              `json:"id"`
		Name              string           `json:"name"`
		Symbol            string           `json:"symbol"`
		Slug              string           `json:"slug"`
		CmcRank           int              `json:"cmc_rank"`
		NumMarketPairs    int              `json:"num_market_pairs"`
		CirculatingSupply float64          `json:"circulating_supply"`
		TotalSupply       float64          `json:"total_supply"`
		MaxSupply         float64          `json:"max_supply"`
		LastUpdated       string           `json:"last_updated"`
		DateAdded         string           `json:"date_added"`
		Tags              []string         `json:"tags"`
		Platform          interface{}      `json:"platform"`
		Quote             map[string]Quote `json:"quote"`
	} `json:"data"`
	Status struct {
		Timestamp    time.Time `json:"timestamp"`
		ErrorCode    int       `json:"error_code"`
		ErrorMessage string    `json:"error_message"`
		Elapsed      int       `json:"elapsed"`
		CreditCount  int       `json:"credit_count"`
	} `json:"status"`
}

type CoinMarketCapScraper struct {
	ticker          *time.Ticker
	foreignScrapper ForeignScraper
}

func NewCoinMarketCapScraper(datastore models.Datastore) *CoinMarketCapScraper {

	foreignScrapper := ForeignScraper{
		shutdown:      make(chan nothing),
		error:         nil,
		datastore:     datastore,
		chanQuotation: make(chan *models.ForeignQuotation),
	}
	s := &CoinMarketCapScraper{
		ticker:          time.NewTicker(coinMarketCapRefreshDelay),
		foreignScrapper: foreignScrapper,
	}
	go s.mainLoop()

	return s

}

// mainLoop runs in a goroutine until channel s is closed.
func (scraper *CoinMarketCapScraper) mainLoop() {
	for {
		select {
		case <-scraper.ticker.C:
			err := scraper.UpdateQuotation()
			if err != nil {
				log.Error(err)
			}
		case <-scraper.foreignScrapper.shutdown: // user requested shutdown
			log.Printf("CoinMarketCapscraper shutting down")
			return
		}
	}

}

// Update retrieves new coin information from the CoinMarketCap API and stores it to influx
func (scraper *CoinMarketCapScraper) UpdateQuotation() error {
	log.Printf("Executing CoinMarketCapScraper update")

	listing, err := getCoinMarketCapData()

	if err != nil {
		log.Errorln("Error getting data from CoinMarketCap", err)
		return err
	}

	for _, coin := range listing.Data {

		// TO DO: normalize symbol instead of all upper
		coin.Symbol = strings.ToUpper(coin.Symbol)

		// Parse last updated timestamp
		layout := "2006-01-02T15:04:05.000Z"
		timestamp, err := time.Parse(layout, coin.LastUpdated)
		if err != nil {
			log.Errorln("error parsing time")
		}

		// Get yesterday's price from influx if available
		priceYesterday, err := scraper.foreignScrapper.datastore.GetForeignPriceYesterday(coin.Symbol, source)
		if err != nil {
			priceYesterday = 0
		}
		usdQuote, ok := coin.Quote["USD"]
		if !ok {
			log.Warnf("Couldn't find usd price for coin: %v", coin.Symbol)
			continue
		}
		foreignQuotation := models.ForeignQuotation{
			Symbol:             coin.Symbol,
			Name:               coin.Name,
			Price:              usdQuote.Price,
			PriceYesterday:     priceYesterday,
			VolumeYesterdayUSD: usdQuote.Volume24H,
			Source:             coinMarketCapsource,
			Time:               timestamp,
		}
		scraper.foreignScrapper.chanQuotation <- &foreignQuotation
	}
	return nil

}

func (scraper *CoinMarketCapScraper) GetQuoteChannel() chan *models.ForeignQuotation {
	return scraper.foreignScrapper.chanQuotation
}

func getApiKey() string {
	if utils.Getenv("USE_ENV", "false") == "true" {
		return utils.Getenv("CMC_API_KEY", "")
	}
	var lines []string
	file, err := os.Open("/run/secrets/Coinmarketcap-API.key") // Read in key information
	if err != nil {
		log.Fatal(err)
	}
	defer func() {
		cerr := file.Close()
		if err == nil {
			err = cerr
		}
	}()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}
	if err = scanner.Err(); err != nil {
		log.Fatal(err)
	}
	if len(lines) != 1 {
		log.Fatal("Secrets file for coinmarketcap API key should have exactly one line")
	}
	return lines[0]
}

func getCoinMarketCapData() (listing CoinMarketCapListing, err error) {
	// There must be a pro coinmarketcap api key for this to work properly
	apiKey := getApiKey()

	req, err := http.NewRequestWithContext(context.Background(), "GET", "https://pro-api.coinmarketcap.com/v1/cryptocurrency/listings/latest", nil)
	if err != nil {
		log.Print(err)
		return
	}

	q := url.Values{}
	q.Add("start", "1")
	q.Add("limit", "200")
	q.Add("convert", "USD")
	q.Add("sort", "market_cap")
	q.Add("sort_dir", "desc")

	req.Header.Set("Accepts", "application/json")
	req.Header.Add("X-CMC_PRO_API_KEY", apiKey)
	req.URL.RawQuery = q.Encode()

	response, statusCode, err := utils.HTTPRequest(req)
	if err != nil {
		fmt.Println("Error sending request to server")
		os.Exit(1)
	}
	fmt.Println(statusCode)

	err = json.Unmarshal(response, &listing)
	if err != nil {
		return
	}
	return
}

/*
// closes all connected Scrapers. Must only be called from mainLoop
func (scraper *CoinMarketCapScraper) cleanup(err error) {

	scraper.foreignScrapper.errorLock.Lock()
	defer scraper.foreignScrapper.errorLock.Unlock()

	scraper.foreignScrapper.tickerRate.Stop()
	scraper.foreignScrapper.tickerState.Stop()

	if err != nil {
		scraper.foreignScrapper.error = err
	}
	scraper.foreignScrapper.closed = true

	close(scraper.foreignScrapper.shutdownDone) // signal that shutdown is complete
}
*/
// Close closes any existing API connections
func (scraper *CoinMarketCapScraper) Close() error {
	if scraper.foreignScrapper.closed {
		return errors.New("scraper already closed")
	}
	close(scraper.foreignScrapper.shutdown)
	<-scraper.foreignScrapper.shutdownDone
	scraper.foreignScrapper.errorLock.RLock()
	defer scraper.foreignScrapper.errorLock.RUnlock()
	return scraper.foreignScrapper.error
}