bosun.org@v0.0.0-20210513094433-e25bc3e69a1f/cmd/backfill/main.go (about)

     1  // Backfill denormalizes historic OpenTSDB data.
     2  //
     3  // For ongoing denormalization use the functionality in tsdbrelay.
     4  package main
     5  
     6  import (
     7  	"flag"
     8  	"fmt"
     9  	"io"
    10  	"io/ioutil"
    11  	"log"
    12  	"net/url"
    13  	"strconv"
    14  	"time"
    15  
    16  	"bosun.org/cmd/tsdbrelay/denormalize"
    17  	"bosun.org/collect"
    18  	"bosun.org/opentsdb"
    19  )
    20  
    21  var (
    22  	start     = flag.String("start", "2013/01/01", "Start date to backfill.")
    23  	end       = flag.String("end", "", "End date to backfill. Will go to now if not specified.")
    24  	ruleFlag  = flag.String("rule", "", "A denormalization rule. ex `os.cpu__host`")
    25  	tsdbHost  = flag.String("host", "", "OpenTSDB host")
    26  	batchSize = flag.Int("batch", 500, "batch size to send points to OpenTSDB")
    27  )
    28  
    29  func main() {
    30  	flag.Parse()
    31  	if *tsdbHost == "" {
    32  		flag.PrintDefaults()
    33  		log.Fatal("host must be supplied")
    34  	}
    35  	putUrl := (&url.URL{Scheme: "http", Host: *tsdbHost, Path: "api/put"}).String()
    36  
    37  	if *ruleFlag == "" {
    38  		flag.PrintDefaults()
    39  		log.Fatal("rule must be supplied")
    40  	}
    41  	rules, err := denormalize.ParseDenormalizationRules(*ruleFlag)
    42  	if err != nil {
    43  		log.Fatal(err)
    44  	}
    45  	if len(rules) > 1 {
    46  		log.Fatal("Please specify only one rule")
    47  	}
    48  	var rule *denormalize.DenormalizationRule
    49  	var metric string
    50  	for k, v := range rules {
    51  		metric = k
    52  		rule = v
    53  	}
    54  
    55  	query := &opentsdb.Query{Metric: metric, Aggregator: "avg"}
    56  	query.Tags, err = queryForAggregateTags(query)
    57  	if err != nil {
    58  		log.Fatal(err)
    59  	}
    60  
    61  	startDate, err := opentsdb.ParseTime(*start)
    62  	if err != nil {
    63  		log.Fatal(err)
    64  	}
    65  	endDate := time.Now().UTC()
    66  	if *end != "" {
    67  		endDate, err = opentsdb.ParseTime(*end)
    68  		if err != nil {
    69  			log.Fatal(err)
    70  		}
    71  	}
    72  
    73  	backfill := func(batchStart, batchEnd time.Time) (err error) {
    74  		startTimeString := batchStart.Format(opentsdb.TSDBTimeFormat)
    75  		endTimeString := batchEnd.Format(opentsdb.TSDBTimeFormat)
    76  		defer func() {
    77  			if err != nil {
    78  				log.Fatalf("Error on batch %s - %s. %v \n", startTimeString, endTimeString, err)
    79  			}
    80  		}()
    81  		req := opentsdb.Request{Start: startTimeString, End: endTimeString, Queries: []*opentsdb.Query{query}}
    82  		resp, err := req.Query(*tsdbHost)
    83  		if err != nil {
    84  			return err
    85  		}
    86  		dps := []*opentsdb.DataPoint{}
    87  		for _, r := range resp {
    88  			for t, p := range r.DPS {
    89  
    90  				timeStamp, err := strconv.ParseInt(t, 10, 64)
    91  				if err != nil {
    92  					return err
    93  				}
    94  				dp := &opentsdb.DataPoint{
    95  					Timestamp: timeStamp,
    96  					Metric:    r.Metric,
    97  					Tags:      r.Tags,
    98  					Value:     p,
    99  				}
   100  				err = rule.Translate(dp)
   101  				if err != nil {
   102  					return err
   103  				}
   104  				dps = append(dps, dp)
   105  			}
   106  		}
   107  		fmt.Printf("%s - %s: %d dps\n", startTimeString, endTimeString, len(dps))
   108  		total := 0
   109  		for len(dps) > 0 {
   110  			count := len(dps)
   111  			if len(dps) > *batchSize {
   112  				count = *batchSize
   113  			}
   114  			putResp, err := collect.SendDataPoints(dps[:count], putUrl)
   115  			if err != nil {
   116  				return err
   117  			}
   118  			defer putResp.Body.Close()
   119  			// Drain up to 512 bytes and close the body to let the Transport reuse the connection
   120  			io.CopyN(ioutil.Discard, putResp.Body, 512)
   121  
   122  			if putResp.StatusCode != 204 {
   123  				return fmt.Errorf("Non 204 status code from opentsdb: %d", putResp.StatusCode)
   124  			}
   125  			dps = dps[count:]
   126  			total += count
   127  		}
   128  		fmt.Printf("Relayed %d data points.\n", total)
   129  		return nil
   130  	}
   131  
   132  	// walk backwards a day at a time
   133  	curEnd := endDate
   134  	for curEnd.After(startDate) {
   135  		curStart := curEnd.Add(-24 * time.Hour)
   136  		if curStart.Before(startDate) {
   137  			curStart = startDate
   138  		}
   139  		backfill(curStart, curEnd)
   140  		curEnd = curEnd.Add(-24 * time.Hour)
   141  	}
   142  }
   143  
   144  func queryForAggregateTags(query *opentsdb.Query) (opentsdb.TagSet, error) {
   145  	req := opentsdb.Request{}
   146  	req.Queries = []*opentsdb.Query{query}
   147  	req.Start = "1h-ago"
   148  	resp, err := req.Query(*tsdbHost)
   149  	if err != nil {
   150  		return nil, err
   151  	}
   152  	if len(resp) < 1 {
   153  		return nil, fmt.Errorf("No points in last hour to learn aggregate tags")
   154  	}
   155  	tagset := make(opentsdb.TagSet)
   156  	for _, t := range resp[0].AggregateTags {
   157  		tagset[t] = "*"
   158  	}
   159  	return tagset, nil
   160  }