github.com/pf-qiu/concourse/v6@v6.7.3-0.20201207032516-1f455d73275f/atc/metric/emitter/influxdb.go (about)

     1  package emitter
     2  
     3  import (
     4  	"time"
     5  
     6  	"code.cloudfoundry.org/lager"
     7  	"github.com/pf-qiu/concourse/v6/atc/metric"
     8  	"github.com/pkg/errors"
     9  
    10  	influxclient "github.com/influxdata/influxdb1-client/v2"
    11  )
    12  
    13  type InfluxDBEmitter struct {
    14  	Client        influxclient.Client
    15  	Database      string
    16  	BatchSize     int
    17  	BatchDuration time.Duration
    18  }
    19  
    20  type InfluxDBConfig struct {
    21  	URL string `long:"influxdb-url" description:"InfluxDB server address to emit points to."`
    22  
    23  	Database string `long:"influxdb-database" description:"InfluxDB database to write points to."`
    24  
    25  	Username string `long:"influxdb-username" description:"InfluxDB server username."`
    26  	Password string `long:"influxdb-password" description:"InfluxDB server password."`
    27  
    28  	InsecureSkipVerify bool `long:"influxdb-insecure-skip-verify" description:"Skip SSL verification when emitting to InfluxDB."`
    29  
    30  	BatchSize     uint32        `long:"influxdb-batch-size" default:"5000" description:"Number of points to batch together when emitting to InfluxDB."`
    31  	BatchDuration time.Duration `long:"influxdb-batch-duration" default:"300s" description:"The duration to wait before emitting a batch of points to InfluxDB, disregarding influxdb-batch-size."`
    32  }
    33  
    34  var (
    35  	batch         []metric.Event
    36  	lastBatchTime time.Time
    37  )
    38  
    39  func init() {
    40  	batch = make([]metric.Event, 0)
    41  	lastBatchTime = time.Now()
    42  	metric.Metrics.RegisterEmitter(&InfluxDBConfig{})
    43  }
    44  
    45  func (config *InfluxDBConfig) Description() string { return "InfluxDB" }
    46  func (config *InfluxDBConfig) IsConfigured() bool  { return config.URL != "" }
    47  
    48  func (config *InfluxDBConfig) NewEmitter() (metric.Emitter, error) {
    49  	client, err := influxclient.NewHTTPClient(influxclient.HTTPConfig{
    50  		Addr:               config.URL,
    51  		Username:           config.Username,
    52  		Password:           config.Password,
    53  		InsecureSkipVerify: config.InsecureSkipVerify,
    54  		Timeout:            time.Minute,
    55  	})
    56  	if err != nil {
    57  		return &InfluxDBEmitter{}, err
    58  	}
    59  
    60  	return &InfluxDBEmitter{
    61  		Client:        client,
    62  		Database:      config.Database,
    63  		BatchSize:     int(config.BatchSize),
    64  		BatchDuration: config.BatchDuration,
    65  	}, nil
    66  }
    67  
    68  func emitBatch(emitter *InfluxDBEmitter, logger lager.Logger, events []metric.Event) {
    69  
    70  	logger.Debug("influxdb-emit-batch", lager.Data{
    71  		"size": len(events),
    72  	})
    73  	bp, err := influxclient.NewBatchPoints(influxclient.BatchPointsConfig{
    74  		Database: emitter.Database,
    75  	})
    76  	if err != nil {
    77  		logger.Error("failed-to-construct-batch-points", err)
    78  		return
    79  	}
    80  
    81  	for _, event := range events {
    82  		tags := map[string]string{
    83  			"host": event.Host,
    84  		}
    85  
    86  		for k, v := range event.Attributes {
    87  			tags[k] = v
    88  		}
    89  
    90  		point, err := influxclient.NewPoint(
    91  			event.Name,
    92  			tags,
    93  			map[string]interface{}{
    94  				"value": event.Value,
    95  			},
    96  			event.Time,
    97  		)
    98  		if err != nil {
    99  			logger.Error("failed-to-construct-point", err)
   100  			continue
   101  		}
   102  
   103  		bp.AddPoint(point)
   104  	}
   105  
   106  	err = emitter.Client.Write(bp)
   107  	if err != nil {
   108  		logger.Error("failed-to-send-points",
   109  			errors.Wrap(metric.ErrFailedToEmit, err.Error()))
   110  		return
   111  	}
   112  }
   113  
   114  func (emitter *InfluxDBEmitter) Emit(logger lager.Logger, event metric.Event) {
   115  	batch = append(batch, event)
   116  	duration := time.Since(lastBatchTime)
   117  	if len(batch) >= emitter.BatchSize || duration >= emitter.BatchDuration {
   118  		logger.Debug("influxdb-pre-emit-batch", lager.Data{
   119  			"influxdb-batch-size":     emitter.BatchSize,
   120  			"current-batch-size":      len(batch),
   121  			"influxdb-batch-duration": emitter.BatchDuration,
   122  			"current-duration":        duration,
   123  		})
   124  		emitter.SubmitBatch(logger)
   125  	}
   126  }
   127  
   128  func (emitter *InfluxDBEmitter) SubmitBatch(logger lager.Logger) {
   129  	batchToSubmit := make([]metric.Event, len(batch))
   130  	copy(batchToSubmit, batch)
   131  	batch = make([]metric.Event, 0)
   132  	lastBatchTime = time.Now()
   133  	go emitBatch(emitter, logger, batchToSubmit)
   134  }