github.com/diadata-org/diadata@v1.4.593/internal/pkg/ratescrapers/updateSONIA.go (about)

     1  package ratescrapers
     2  
     3  import (
     4  	"errors"
     5  	"github.com/anaskhan96/soup"
     6  	models "github.com/diadata-org/diadata/pkg/model"
     7  	"github.com/sirupsen/logrus"
     8  	prefixed "github.com/x-cray/logrus-prefixed-formatter"
     9  	"strconv"
    10  	"strings"
    11  	"time"
    12  )
    13  
    14  var logger = logrus.New()
    15  
    16  type SONIA struct {
    17  	RefDate time.Time
    18  	RefRate float64
    19  }
    20  
    21  func (s *SONIA) reset() {
    22  	s.RefRate = 0
    23  	s.RefDate = time.Time{}
    24  }
    25  
    26  type SONIAHistorical []SONIA
    27  
    28  func (s *RateScraper) UpdateSonia() (err error) {
    29  	err = nil
    30  	logger.Formatter = new(prefixed.TextFormatter)
    31  	logger.Level = logrus.InfoLevel
    32  
    33  	// ! if the data format changes, uncomment this to see where the problem is
    34  	//soup.SetDebug(true)
    35  
    36  	resp, err := soup.Get("https://www.bankofengland.co.uk/boeapps/database/fromshowcolumns.asp?Travel=NIxSUx&FromSeries=1&ToSeries=50&DAT=ALL&FNY=&CSVF=TT&html.x=132&html.y=32&C=5JK&Filter=N")
    37  	if err != nil {
    38  		return
    39  	}
    40  	doc := soup.HTMLParse(resp)
    41  	tds := doc.Find("table", "id", "stats-table").Find("tbody").FindAll("td")
    42  
    43  	nilDt := time.Time{}
    44  	var historicalRates SONIAHistorical
    45  	var sonia = SONIA{}
    46  	var refDt time.Time
    47  	var refRt float64
    48  
    49  	for ix, refDtOrRt := range tds {
    50  		switch ix % 2 {
    51  		case 0:
    52  			refDt, err = time.Parse("2 Jan 06", strings.TrimSpace(refDtOrRt.Text()))
    53  			if err != nil {
    54  				logger.WithFields(logrus.Fields{"prefix": "SONIA"}).Error(err)
    55  				return
    56  			}
    57  			if sonia.RefDate != nilDt || sonia.RefRate != 0 {
    58  				err = errors.New("sonia should be empty but is not. looks like the data format changed")
    59  				logger.WithFields(logrus.Fields{"prefix": "SONIA"}).Error(err)
    60  				return
    61  			}
    62  			sonia.RefDate = refDt
    63  		case 1:
    64  			refRt, err = strconv.ParseFloat(strings.TrimSpace(refDtOrRt.Text()), 64)
    65  			if err != nil {
    66  				logger.WithFields(logrus.Fields{"prefix": "SONIA"}).Error(err)
    67  				return
    68  			}
    69  			if sonia.RefDate == nilDt || sonia.RefRate != 0 {
    70  				err = errors.New("sonia should should have empty date and zero rate. looks like the data format changed")
    71  				logger.WithFields(logrus.Fields{"prefix": "SONIA"}).Error(err)
    72  				return
    73  			}
    74  			sonia.RefRate = refRt
    75  			historicalRates = append(historicalRates, sonia)
    76  			sonia.reset()
    77  		}
    78  	}
    79  
    80  	logger.WithFields(logrus.Fields{"prefix": "SONIA"}).Debug(historicalRates)
    81  
    82  	for _, sonia := range historicalRates {
    83  		t := &models.InterestRate{
    84  			Symbol:          "SONIA",
    85  			Value:           sonia.RefRate,
    86  			PublicationTime: time.Now(),
    87  			EffectiveDate:   sonia.RefDate,
    88  			Source:          "BOE",
    89  		}
    90  
    91  		// Send new data through channel chanInterestRate
    92  		logger.WithFields(logrus.Fields{"prefix": "SONIA"}).Debugf("writing interestRate %#v in %v\n", t, s.chanInterestRate)
    93  		s.chanInterestRate <- t
    94  	}
    95  
    96  	logger.WithFields(logrus.Fields{"prefix": "SONIA"}).Info("update complete")
    97  
    98  	return
    99  }