github.com/diadata-org/diadata@v1.4.593/internal/pkg/static-scrapers/historyPreESTER.go (about)

     1  package staticscrapers
     2  
     3  import (
     4  	"encoding/xml"
     5  	"fmt"
     6  	"io/ioutil"
     7  	"os"
     8  	"path/filepath"
     9  	"strconv"
    10  	"time"
    11  
    12  	models "github.com/diadata-org/diadata/pkg/model"
    13  	utils "github.com/diadata-org/diadata/pkg/utils"
    14  	log "github.com/sirupsen/logrus"
    15  )
    16  
    17  type (
    18  	CMessageGroupPre struct {
    19  		XMLName     xml.Name     `xml:"MessageGroup,omitempty" json:"MessageGroup,omitempty"`
    20  		CDataSetPre *CDataSetPre `xml:"http://www.SDMX.org/resources/SDMXML/schemas/v2_0/message DataSet,omitempty" json:"DataSet,omitempty"`
    21  	}
    22  
    23  	CDataSetPre struct {
    24  		XMLName    xml.Name      `xml:"DataSet,omitempty" json:"DataSet,omitempty"`
    25  		Attraction string        `xml:"action,attr"  json:",omitempty"`
    26  		CSeriesPre []*CSeriesPre `xml:"http://www.SDMX.org/resources/SDMXML/schemas/v2_0/message Series,omitempty" json:"Series,omitempty"`
    27  	}
    28  
    29  	CSeriesPre struct {
    30  		XMLName xml.Name   `xml:"Series,omitempty" json:"Series,omitempty"`
    31  		CTitle  string     `xml:"TITLE,attr"  json:",omitempty"`
    32  		CObsPre []*CObsPre `xml:"http://www.SDMX.org/resources/SDMXML/schemas/v2_0/message Obs,omitempty" json:"Obs,omitempty"`
    33  	}
    34  
    35  	CObsPre struct {
    36  		XMLName         xml.Name `xml:"Obs,omitempty" json:"Obs,omitempty"`
    37  		AttrCONF_STATUS string   `xml:"CONF_STATUS,attr"  json:",omitempty"`
    38  		AttrOBS_STATUS  string   `xml:"OBS_STATUS,attr"  json:",omitempty"`
    39  		AttrOBS_VALUE   string   `xml:"OBS_VALUE,attr"  json:",omitempty"`
    40  		AttrTIME_PERIOD string   `xml:"TIME_PERIOD,attr"  json:",omitempty"`
    41  	}
    42  )
    43  
    44  const pathPreESTER = "../../internal/pkg/ratescraperData/PRE-ESTER_Historic_data.xml"
    45  
    46  // GetHistoricPreESTER downloads historic ESTER data from the ECB Statistical Data
    47  // Warehouse and stores it in an xml file. Here, historic means up until two days before now.
    48  func GetHistoricPreESTER() error {
    49  	linkPreESTER := "https://sdw.ecb.europa.eu/export.do?org.apache.struts.taglib.html.TOKEN" +
    50  		"=f27da7a7aef6da13e1141c5c6932a48e&df=true&ec=&dc=&oc=&pb=&rc=&DATASET=0&removeItem" +
    51  		"=&removedItemList=&mergeFilter=&activeTab=MMSR&showHide=&MAX_DOWNLOAD_SERIES=500&" +
    52  		"SERIES_MAX_NUM=50&node=9693657&legendPub=published&legendNor=&exportType=sdmx&ajaxTab=true"
    53  	pathPreESTERAbs, _ := filepath.Abs(pathPreESTER)
    54  	err := os.MkdirAll(filepath.Dir(pathPreESTERAbs), os.ModePerm)
    55  	if err != nil {
    56  		return err
    57  	}
    58  
    59  	err = utils.DownloadResource(pathPreESTERAbs, linkPreESTER)
    60  	if err != nil {
    61  		return err
    62  	}
    63  	return err
    64  }
    65  
    66  // WriteHistoricPreESTER makes a GET request to fetch the historic data of the SOFR index
    67  // and writes it into the redis database.
    68  func WriteHistoricPreESTER(ds models.Datastore) (err error) {
    69  	log.Printf("Writing historic Pre-ESTER data")
    70  
    71  	// The path relative to the calling main / executable
    72  	absPath, _ := filepath.Abs(pathPreESTER)
    73  	xmlFile, err := os.Open(absPath) //nolint:gosec
    74  	if err != nil {
    75  		fmt.Println(err)
    76  	}
    77  
    78  	defer func() {
    79  		cerr := xmlFile.Close()
    80  		if err == nil {
    81  			err = cerr
    82  		}
    83  	}()
    84  
    85  	byteValue, _ := ioutil.ReadAll(xmlFile)
    86  	var myVar CMessageGroupPre
    87  	err = xml.Unmarshal(byteValue, &myVar)
    88  	if err != nil {
    89  		return
    90  	}
    91  
    92  	// A slice containing all historic data. Value data is in the 8th row of series
    93  	histDataSlice := myVar.CDataSetPre.CSeriesPre[7].CObsPre
    94  	numData := len(histDataSlice)
    95  
    96  	for i := 0; i < numData; i++ {
    97  
    98  		// Convert interest rate from string to float64
    99  		rate, err := strconv.ParseFloat(histDataSlice[i].AttrOBS_VALUE, 64)
   100  		if err != nil {
   101  			fmt.Println(err)
   102  		}
   103  
   104  		// Convert time string to Time type in UTC and pass date (without daytime)
   105  		// Pre-ESTER is published at around 08:00 CET. This timestamp lacks in historic tables.
   106  		myTime := histDataSlice[i].AttrTIME_PERIOD + "T08:00:00"
   107  		layout := "2006-01-02T15:04:05"
   108  		loc, _ := time.LoadLocation("CET")
   109  		// Parse time to UTC time
   110  		dateTime, err := time.ParseInLocation(layout, myTime, loc)
   111  
   112  		if err != nil {
   113  			fmt.Println(err)
   114  		} else {
   115  			dateTime = dateTime.UTC()
   116  		}
   117  
   118  		t := models.InterestRate{
   119  			Symbol:          "PRE-ESTER",
   120  			Value:           rate,
   121  			PublicationTime: dateTime,
   122  			Source:          "ECB",
   123  		}
   124  
   125  		err = ds.SetInterestRate(&t)
   126  		if err != nil {
   127  			log.Error(err)
   128  		}
   129  
   130  	}
   131  
   132  	log.Info("Writing historic Pre-ESTER data complete.")
   133  
   134  	return
   135  }