github.com/diadata-org/diadata@v1.4.593/internal/pkg/static-scrapers/historySOFR.go (about)

     1  package staticscrapers
     2  
     3  import (
     4  	"bytes"
     5  	"encoding/xml"
     6  	"fmt"
     7  	"os"
     8  	"strconv"
     9  	"time"
    10  
    11  	models "github.com/diadata-org/diadata/pkg/model"
    12  	utils "github.com/diadata-org/diadata/pkg/utils"
    13  	log "github.com/sirupsen/logrus"
    14  )
    15  
    16  type (
    17  	CmktRatesSecondaryFindByDateResponse struct {
    18  		XMLName           xml.Name             `xml:"mktRatesSecondaryFindByDateResponse,omitempty" json:"mktRatesSecondaryFindByDateResponse,omitempty"`
    19  		CmktRatesFindItem []*CmktRatesFindItem `xml:"mktRatesFindItem,omitempty" json:"mktRatesFindItem,omitempty"`
    20  	}
    21  
    22  	CmktRatesFindItem struct {
    23  		XMLName        xml.Name        `xml:"mktRatesFindItem,omitempty" json:"mktRatesFindItem,omitempty"`
    24  		CrateOperation *CrateOperation `xml:"rateOperation,omitempty" json:"rateOperation,omitempty"`
    25  	}
    26  
    27  	CrateOperation struct {
    28  		XMLName           xml.Name           `xml:"rateOperation,omitempty" json:"rateOperation,omitempty"`
    29  		CeffectiveDate    *CeffectiveDate    `xml:"effectiveDate,omitempty" json:"effectiveDate,omitempty"`
    30  		CinsertTimestamp  *CinsertTimestamp  `xml:"insertTimestamp,omitempty" json:"insertTimestamp,omitempty"`
    31  		Crate             *Crate             `xml:"rate,omitempty" json:"rate,omitempty"`
    32  		CrateType         *CrateType         `xml:"rateType,omitempty" json:"rateType,omitempty"`
    33  		CtradingVolume    *CtradingVolume    `xml:"tradingVolume,omitempty" json:"tradingVolume,omitempty"`
    34  		CbusinessEventLog *CbusinessEventLog `xml:"businessEventLog,omitempty" json:"businessEventLog,omitempty"`
    35  	}
    36  
    37  	CbusinessEventLog struct {
    38  		XMLName    xml.Name    `xml:"businessEventLog,omitempty" json:"businessEventLog,omitempty"`
    39  		CeventDate *CeventDate `xml:"eventDate,omitempty" json:"eventDate,omitempty"`
    40  	}
    41  
    42  	CeventDate struct {
    43  		XMLName xml.Name `xml:"eventDate,omitempty" json:"eventDate,omitempty"`
    44  		CEvDate string   `xml:",chardata" json:",omitempty"`
    45  	}
    46  
    47  	CeffectiveDate struct {
    48  		XMLName  xml.Name `xml:"effectiveDate,omitempty" json:"effectiveDate,omitempty"`
    49  		CEffDate string   `xml:",chardata" json:",omitempty"`
    50  	}
    51  
    52  	Crate struct {
    53  		XMLName xml.Name `xml:"rate,omitempty" json:"rate,omitempty"`
    54  		CValue  string   `xml:",chardata" json:",omitempty"`
    55  	}
    56  
    57  	CrateType struct {
    58  		XMLName xml.Name `xml:"rateType,omitempty" json:"rateType,omitempty"`
    59  		CType   string   `xml:",chardata" json:",omitempty"`
    60  	}
    61  
    62  	CtradingVolume struct {
    63  		XMLName xml.Name `xml:"tradingVolume,omitempty" json:"tradingVolume,omitempty"`
    64  		CVolume string   `xml:",chardata" json:",omitempty"`
    65  	}
    66  
    67  	CinsertTimestamp struct {
    68  		XMLName    xml.Name `xml:"insertTimestamp,omitempty" json:"insertTimestamp,omitempty"`
    69  		CTimestamp string   `xml:",chardata" json:",omitempty"`
    70  	}
    71  )
    72  
    73  // WriteHistoricSOFR makes a GET request to fetch the historic data of the SOFR index
    74  // and writes it into the redis database.
    75  func WriteHistoricSOFR(ds models.Datastore) error {
    76  	log.Printf("Writing historic SOFR data")
    77  
    78  	// Get rss from fed webpage
    79  	XMLdata, _, err := utils.GetRequest("https://apps.newyorkfed.org/api/mktrates/r3")
    80  
    81  	if err != nil {
    82  		return err
    83  	}
    84  
    85  	// Decode the body
    86  	rss := new(CmktRatesSecondaryFindByDateResponse)
    87  	buffer := bytes.NewBuffer(XMLdata)
    88  	decoded := xml.NewDecoder(buffer)
    89  	err = decoded.Decode(rss)
    90  
    91  	if err != nil {
    92  		fmt.Println(err)
    93  		os.Exit(1)
    94  	}
    95  
    96  	// A slice containing all historic data
    97  	histDataSlice := rss.CmktRatesFindItem
    98  	numData := len(histDataSlice)
    99  
   100  	for i := 0; i < numData; i++ {
   101  		var rate float64
   102  		var dateTime time.Time
   103  		var effDate time.Time
   104  		// Collect entries of InterestRate struct -----------------------------------
   105  		symbol := histDataSlice[i].CrateOperation.CrateType.CType
   106  
   107  		// Convert interest rate from string to float64
   108  		rate, err = strconv.ParseFloat(histDataSlice[i].CrateOperation.Crate.CValue, 64)
   109  		if err != nil {
   110  			fmt.Println(err)
   111  		}
   112  
   113  		// Convert time string to Time type in UTC and pass date (without daytime)
   114  		dateTime, err = time.Parse(time.RFC3339, histDataSlice[i].CrateOperation.CbusinessEventLog.CeventDate.CEvDate)
   115  
   116  		if err != nil {
   117  			log.Error(err)
   118  		} else {
   119  			dateTime = dateTime.Round(time.Second).UTC()
   120  		}
   121  
   122  		effDate, err = time.Parse("2006-01-02", histDataSlice[i].CrateOperation.CeffectiveDate.CEffDate)
   123  		if err != nil {
   124  			log.Error(err)
   125  		}
   126  
   127  		t := models.InterestRate{
   128  			Symbol:          symbol,
   129  			Value:           rate,
   130  			PublicationTime: dateTime,
   131  			EffectiveDate:   effDate,
   132  			Source:          "FED",
   133  		}
   134  
   135  		err = ds.SetInterestRate(&t)
   136  		if err != nil {
   137  			log.Error(err)
   138  		}
   139  
   140  	}
   141  
   142  	log.Info("Writing historic SOFR data complete.")
   143  
   144  	return err
   145  }