github.com/jonaz/heapster@v1.3.0-beta.0.0.20170208112634-cd3c15ca3d29/metrics/sinks/influxdb/influxdb.go (about)

     1  // Copyright 2015 Google Inc. All Rights Reserved.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package influxdb
    16  
    17  import (
    18  	"fmt"
    19  	"net/url"
    20  	"strings"
    21  	"sync"
    22  	"time"
    23  
    24  	influxdb_common "k8s.io/heapster/common/influxdb"
    25  	"k8s.io/heapster/metrics/core"
    26  
    27  	"github.com/golang/glog"
    28  	influxdb "github.com/influxdata/influxdb/client"
    29  )
    30  
    31  type influxdbSink struct {
    32  	client influxdb_common.InfluxdbClient
    33  	sync.RWMutex
    34  	c        influxdb_common.InfluxdbConfig
    35  	dbExists bool
    36  }
    37  
    38  const (
    39  	// Value Field name
    40  	valueField = "value"
    41  	// Event special tags
    42  	dbNotFoundError = "database not found"
    43  
    44  	// Maximum number of influxdb Points to be sent in one batch.
    45  	maxSendBatchSize = 10000
    46  )
    47  
    48  func (sink *influxdbSink) resetConnection() {
    49  	glog.Infof("Influxdb connection reset")
    50  	sink.dbExists = false
    51  	sink.client = nil
    52  }
    53  
    54  func (sink *influxdbSink) ExportData(dataBatch *core.DataBatch) {
    55  	sink.Lock()
    56  	defer sink.Unlock()
    57  
    58  	dataPoints := make([]influxdb.Point, 0, 0)
    59  	for _, metricSet := range dataBatch.MetricSets {
    60  		for metricName, metricValue := range metricSet.MetricValues {
    61  
    62  			var value interface{}
    63  			if core.ValueInt64 == metricValue.ValueType {
    64  				value = metricValue.IntValue
    65  			} else if core.ValueFloat == metricValue.ValueType {
    66  				value = float64(metricValue.FloatValue)
    67  			} else {
    68  				continue
    69  			}
    70  
    71  			// Prepare measurement without fields
    72  			fieldName := "value"
    73  			measurementName := metricName
    74  			if sink.c.WithFields {
    75  				// Prepare measurement and field names
    76  				serieName := strings.SplitN(metricName, "/", 2)
    77  				measurementName = serieName[0]
    78  				if len(serieName) > 1 {
    79  					fieldName = serieName[1]
    80  				}
    81  			}
    82  
    83  			point := influxdb.Point{
    84  				Measurement: measurementName,
    85  				Tags:        metricSet.Labels,
    86  				Fields: map[string]interface{}{
    87  					fieldName: value,
    88  				},
    89  				Time: dataBatch.Timestamp.UTC(),
    90  			}
    91  			dataPoints = append(dataPoints, point)
    92  			if len(dataPoints) >= maxSendBatchSize {
    93  				sink.sendData(dataPoints)
    94  				dataPoints = make([]influxdb.Point, 0, 0)
    95  			}
    96  		}
    97  
    98  		for _, labeledMetric := range metricSet.LabeledMetrics {
    99  
   100  			var value interface{}
   101  			if core.ValueInt64 == labeledMetric.ValueType {
   102  				value = labeledMetric.IntValue
   103  			} else if core.ValueFloat == labeledMetric.ValueType {
   104  				value = float64(labeledMetric.FloatValue)
   105  			} else {
   106  				continue
   107  			}
   108  
   109  			// Prepare measurement without fields
   110  			fieldName := "value"
   111  			measurementName := labeledMetric.Name
   112  			if sink.c.WithFields {
   113  				// Prepare measurement and field names
   114  				serieName := strings.SplitN(labeledMetric.Name, "/", 2)
   115  				measurementName = serieName[0]
   116  				if len(serieName) > 1 {
   117  					fieldName = serieName[1]
   118  				}
   119  			}
   120  
   121  			point := influxdb.Point{
   122  				Measurement: measurementName,
   123  				Tags:        make(map[string]string),
   124  				Fields: map[string]interface{}{
   125  					fieldName: value,
   126  				},
   127  				Time: dataBatch.Timestamp.UTC(),
   128  			}
   129  			for key, value := range metricSet.Labels {
   130  				point.Tags[key] = value
   131  			}
   132  			for key, value := range labeledMetric.Labels {
   133  				point.Tags[key] = value
   134  			}
   135  
   136  			dataPoints = append(dataPoints, point)
   137  			if len(dataPoints) >= maxSendBatchSize {
   138  				sink.sendData(dataPoints)
   139  				dataPoints = make([]influxdb.Point, 0, 0)
   140  			}
   141  		}
   142  	}
   143  	if len(dataPoints) >= 0 {
   144  		sink.sendData(dataPoints)
   145  	}
   146  }
   147  
   148  func (sink *influxdbSink) sendData(dataPoints []influxdb.Point) {
   149  	if err := sink.createDatabase(); err != nil {
   150  		glog.Errorf("Failed to create infuxdb: %v", err)
   151  		return
   152  	}
   153  	bp := influxdb.BatchPoints{
   154  		Points:          dataPoints,
   155  		Database:        sink.c.DbName,
   156  		RetentionPolicy: "default",
   157  	}
   158  
   159  	start := time.Now()
   160  	if _, err := sink.client.Write(bp); err != nil {
   161  		if strings.Contains(err.Error(), dbNotFoundError) {
   162  			sink.resetConnection()
   163  		} else if _, _, err := sink.client.Ping(); err != nil {
   164  			glog.Errorf("InfluxDB ping failed: %v", err)
   165  			sink.resetConnection()
   166  		}
   167  	}
   168  	end := time.Now()
   169  	glog.V(4).Infof("Exported %d data to influxDB in %s", len(dataPoints), end.Sub(start))
   170  }
   171  
   172  func (sink *influxdbSink) Name() string {
   173  	return "InfluxDB Sink"
   174  }
   175  
   176  func (sink *influxdbSink) Stop() {
   177  	// nothing needs to be done.
   178  }
   179  
   180  func (sink *influxdbSink) ensureClient() error {
   181  	if sink.client == nil {
   182  		client, err := influxdb_common.NewClient(sink.c)
   183  		if err != nil {
   184  			return err
   185  		}
   186  		sink.client = client
   187  	}
   188  
   189  	return nil
   190  }
   191  
   192  func (sink *influxdbSink) createDatabase() error {
   193  	if err := sink.ensureClient(); err != nil {
   194  		return err
   195  	}
   196  
   197  	if sink.dbExists {
   198  		return nil
   199  	}
   200  	q := influxdb.Query{
   201  		Command: fmt.Sprintf(`CREATE DATABASE %s WITH NAME "default"`, sink.c.DbName),
   202  	}
   203  
   204  	if resp, err := sink.client.Query(q); err != nil {
   205  		if !(resp != nil && resp.Err != nil && strings.Contains(resp.Err.Error(), "already exists")) {
   206  			err := sink.createRetentionPolicy()
   207  			if err != nil {
   208  				return err
   209  			}
   210  		}
   211  	}
   212  
   213  	sink.dbExists = true
   214  	glog.Infof("Created database %q on influxDB server at %q", sink.c.DbName, sink.c.Host)
   215  	return nil
   216  }
   217  
   218  func (sink *influxdbSink) createRetentionPolicy() error {
   219  	q := influxdb.Query{
   220  		Command: fmt.Sprintf(`CREATE RETENTION POLICY "default" ON %s DURATION %s REPLICATION 1 DEFAULT`, sink.c.DbName, sink.c.RetentionPolicy),
   221  	}
   222  
   223  	if resp, err := sink.client.Query(q); err != nil {
   224  		if !(resp != nil && resp.Err != nil) {
   225  			return fmt.Errorf("Retention Policy creation failed: %v", err)
   226  		}
   227  	}
   228  
   229  	glog.Infof("Created retention policy 'default' in database %q on influxDB server at %q", sink.c.DbName, sink.c.Host)
   230  	return nil
   231  }
   232  
   233  // Returns a thread-compatible implementation of influxdb interactions.
   234  func new(c influxdb_common.InfluxdbConfig) core.DataSink {
   235  	client, err := influxdb_common.NewClient(c)
   236  	if err != nil {
   237  		glog.Errorf("issues while creating an InfluxDB sink: %v, will retry on use", err)
   238  	}
   239  	return &influxdbSink{
   240  		client: client, // can be nil
   241  		c:      c,
   242  	}
   243  }
   244  
   245  func CreateInfluxdbSink(uri *url.URL) (core.DataSink, error) {
   246  	config, err := influxdb_common.BuildConfig(uri)
   247  	if err != nil {
   248  		return nil, err
   249  	}
   250  	sink := new(*config)
   251  	glog.Infof("created influxdb sink with options: host:%s user:%s db:%s", config.Host, config.User, config.DbName)
   252  	return sink, nil
   253  }