github.com/diadata-org/diadata@v1.4.593/internal/pkg/static-scrapers/historyESTER.go (about)

     1  package staticscrapers
     2  
     3  import (
     4  	"encoding/xml"
     5  	"fmt"
     6  	"io/ioutil"
     7  	"os"
     8  	"path/filepath"
     9  	"strconv"
    10  	"time"
    11  
    12  	models "github.com/diadata-org/diadata/pkg/model"
    13  	utils "github.com/diadata-org/diadata/pkg/utils"
    14  	log "github.com/sirupsen/logrus"
    15  )
    16  
    17  type (
    18  	CMessageGroup struct {
    19  		XMLName  xml.Name  `xml:"MessageGroup,omitempty" json:"MessageGroup,omitempty"`
    20  		CDataSet *CDataSet `xml:"http://www.SDMX.org/resources/SDMXML/schemas/v2_0/message DataSet,omitempty" json:"DataSet,omitempty"`
    21  	}
    22  
    23  	CDataSet struct {
    24  		XMLName    xml.Name   `xml:"DataSet,omitempty" json:"DataSet,omitempty"`
    25  		Attraction string     `xml:"action,attr"  json:",omitempty"`
    26  		CSeries    []*CSeries `xml:"http://www.SDMX.org/resources/SDMXML/schemas/v2_0/message Series,omitempty" json:"Series,omitempty"`
    27  	}
    28  
    29  	CSeries struct {
    30  		XMLName xml.Name `xml:"Series,omitempty" json:"Series,omitempty"`
    31  		CObs    []*CObs  `xml:"http://www.SDMX.org/resources/SDMXML/schemas/v2_0/message Obs,omitempty" json:"Obs,omitempty"`
    32  	}
    33  
    34  	CObs struct {
    35  		XMLName         xml.Name `xml:"Obs,omitempty" json:"Obs,omitempty"`
    36  		AttrOBS_VALUE   string   `xml:"OBS_VALUE,attr"  json:",omitempty"`
    37  		AttrTIME_PERIOD string   `xml:"TIME_PERIOD,attr"  json:",omitempty"`
    38  	}
    39  )
    40  
    41  const pathESTER = "../../internal/pkg/ratescraperData/ESTER_Historic_data.xml"
    42  
    43  // GetHistoricESTER downloads historic ESTER data from the ECB Statistical Data
    44  // Warehouse and stores it in an xml file. Here, historic means up until two days before now.
    45  func GetHistoricESTER() error {
    46  	linkESTER := "https://sdw.ecb.europa.eu/export.do?org.apache.struts.taglib.html.TOKEN=" +
    47  		"8e79ff477e3157edfac21bf72dfb31b8&df=true&ec=&dc=&oc=&pb=&rc=&DATASET=0&r" +
    48  		"emoveItem=&removedItemList=&mergeFilter=&activeTab=EST&showHide=&MAX_DOW" +
    49  		"NLOAD_SERIES=500&SERIES_MAX_NUM=50&node=9698150&legendRef=reference&legendNor=&exportType=sdmx&ajaxTab=true"
    50  	pathESTERAbs, _ := filepath.Abs(pathESTER)
    51  	err := os.MkdirAll(filepath.Dir(pathESTERAbs), os.ModePerm)
    52  	if err != nil {
    53  		fmt.Println("here")
    54  		return err
    55  	}
    56  
    57  	err = utils.DownloadResource(pathESTERAbs, linkESTER)
    58  	if err != nil {
    59  		fmt.Println("there")
    60  		return err
    61  	}
    62  	return err
    63  }
    64  
    65  // WriteHistoricESTER makes a GET request to fetch the historic data of the SOFR index
    66  // and writes it into the redis database.
    67  func WriteHistoricESTER(ds models.Datastore) (err error) {
    68  
    69  	log.Printf("Writing historic ESTER data")
    70  
    71  	pathESTERAbs, _ := filepath.Abs(pathESTER)
    72  	xmlFile, err := os.Open(pathESTERAbs) //nolint:gosec
    73  	if err != nil {
    74  		fmt.Println(err)
    75  	}
    76  
    77  	defer func() {
    78  		cerr := xmlFile.Close()
    79  		if err == nil {
    80  			err = cerr
    81  		}
    82  	}()
    83  
    84  	byteValue, _ := ioutil.ReadAll(xmlFile)
    85  	var myVar CMessageGroup
    86  	err = xml.Unmarshal(byteValue, &myVar)
    87  	if err != nil {
    88  		return
    89  	}
    90  
    91  	// A slice containing all historic data. Value data is in the 8th row of series
    92  	histDataSlice := myVar.CDataSet.CSeries[8].CObs
    93  	numData := len(histDataSlice)
    94  
    95  	for i := 0; i < numData; i++ {
    96  
    97  		// Convert interest rate from string to float64
    98  		rate, err := strconv.ParseFloat(histDataSlice[i].AttrOBS_VALUE, 64)
    99  		if err != nil {
   100  			fmt.Println(err)
   101  		}
   102  
   103  		// Convert time string to Time type in UTC and pass date (without daytime)
   104  		// ESTR is published at around 08:00 CET. This timestamp lacks in historic tables.
   105  		myTime := histDataSlice[i].AttrTIME_PERIOD
   106  		layout := "2006-01-02"
   107  		// loc, _ := time.LoadLocation("CET")
   108  		// Parse time to UTC time
   109  		// dateTime, err := time.ParseInLocation(layout, myTime, loc)
   110  		dateTime, err := time.Parse(layout, myTime)
   111  
   112  		if err != nil {
   113  			fmt.Println(err)
   114  		} else {
   115  			dateTime = dateTime.UTC()
   116  		}
   117  
   118  		t := models.InterestRate{
   119  			Symbol:          "ESTER",
   120  			Value:           rate,
   121  			PublicationTime: dateTime,
   122  			EffectiveDate:   dateTime,
   123  			Source:          "ECB",
   124  		}
   125  
   126  		err = ds.SetInterestRate(&t)
   127  		if err != nil {
   128  			log.Error(err)
   129  		}
   130  
   131  	}
   132  
   133  	log.Info("Writing historic ESTER data complete.")
   134  
   135  	return
   136  }