github.com/jonaz/heapster@v1.3.0-beta.0.0.20170208112634-cd3c15ca3d29/metrics/sinks/influxdb/influxdb.go (about) 1 // Copyright 2015 Google Inc. All Rights Reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package influxdb 16 17 import ( 18 "fmt" 19 "net/url" 20 "strings" 21 "sync" 22 "time" 23 24 influxdb_common "k8s.io/heapster/common/influxdb" 25 "k8s.io/heapster/metrics/core" 26 27 "github.com/golang/glog" 28 influxdb "github.com/influxdata/influxdb/client" 29 ) 30 31 type influxdbSink struct { 32 client influxdb_common.InfluxdbClient 33 sync.RWMutex 34 c influxdb_common.InfluxdbConfig 35 dbExists bool 36 } 37 38 const ( 39 // Value Field name 40 valueField = "value" 41 // Event special tags 42 dbNotFoundError = "database not found" 43 44 // Maximum number of influxdb Points to be sent in one batch. 45 maxSendBatchSize = 10000 46 ) 47 48 func (sink *influxdbSink) resetConnection() { 49 glog.Infof("Influxdb connection reset") 50 sink.dbExists = false 51 sink.client = nil 52 } 53 54 func (sink *influxdbSink) ExportData(dataBatch *core.DataBatch) { 55 sink.Lock() 56 defer sink.Unlock() 57 58 dataPoints := make([]influxdb.Point, 0, 0) 59 for _, metricSet := range dataBatch.MetricSets { 60 for metricName, metricValue := range metricSet.MetricValues { 61 62 var value interface{} 63 if core.ValueInt64 == metricValue.ValueType { 64 value = metricValue.IntValue 65 } else if core.ValueFloat == metricValue.ValueType { 66 value = float64(metricValue.FloatValue) 67 } else { 68 continue 69 } 70 71 // Prepare measurement without fields 72 fieldName := "value" 73 measurementName := metricName 74 if sink.c.WithFields { 75 // Prepare measurement and field names 76 serieName := strings.SplitN(metricName, "/", 2) 77 measurementName = serieName[0] 78 if len(serieName) > 1 { 79 fieldName = serieName[1] 80 } 81 } 82 83 point := influxdb.Point{ 84 Measurement: measurementName, 85 Tags: metricSet.Labels, 86 Fields: map[string]interface{}{ 87 fieldName: value, 88 }, 89 Time: dataBatch.Timestamp.UTC(), 90 } 91 dataPoints = append(dataPoints, point) 92 if len(dataPoints) >= maxSendBatchSize { 93 sink.sendData(dataPoints) 94 dataPoints = make([]influxdb.Point, 0, 0) 95 } 96 } 97 98 for _, labeledMetric := range metricSet.LabeledMetrics { 99 100 var value interface{} 101 if core.ValueInt64 == labeledMetric.ValueType { 102 value = labeledMetric.IntValue 103 } else if core.ValueFloat == labeledMetric.ValueType { 104 value = float64(labeledMetric.FloatValue) 105 } else { 106 continue 107 } 108 109 // Prepare measurement without fields 110 fieldName := "value" 111 measurementName := labeledMetric.Name 112 if sink.c.WithFields { 113 // Prepare measurement and field names 114 serieName := strings.SplitN(labeledMetric.Name, "/", 2) 115 measurementName = serieName[0] 116 if len(serieName) > 1 { 117 fieldName = serieName[1] 118 } 119 } 120 121 point := influxdb.Point{ 122 Measurement: measurementName, 123 Tags: make(map[string]string), 124 Fields: map[string]interface{}{ 125 fieldName: value, 126 }, 127 Time: dataBatch.Timestamp.UTC(), 128 } 129 for key, value := range metricSet.Labels { 130 point.Tags[key] = value 131 } 132 for key, value := range labeledMetric.Labels { 133 point.Tags[key] = value 134 } 135 136 dataPoints = append(dataPoints, point) 137 if len(dataPoints) >= maxSendBatchSize { 138 sink.sendData(dataPoints) 139 dataPoints = make([]influxdb.Point, 0, 0) 140 } 141 } 142 } 143 if len(dataPoints) >= 0 { 144 sink.sendData(dataPoints) 145 } 146 } 147 148 func (sink *influxdbSink) sendData(dataPoints []influxdb.Point) { 149 if err := sink.createDatabase(); err != nil { 150 glog.Errorf("Failed to create infuxdb: %v", err) 151 return 152 } 153 bp := influxdb.BatchPoints{ 154 Points: dataPoints, 155 Database: sink.c.DbName, 156 RetentionPolicy: "default", 157 } 158 159 start := time.Now() 160 if _, err := sink.client.Write(bp); err != nil { 161 if strings.Contains(err.Error(), dbNotFoundError) { 162 sink.resetConnection() 163 } else if _, _, err := sink.client.Ping(); err != nil { 164 glog.Errorf("InfluxDB ping failed: %v", err) 165 sink.resetConnection() 166 } 167 } 168 end := time.Now() 169 glog.V(4).Infof("Exported %d data to influxDB in %s", len(dataPoints), end.Sub(start)) 170 } 171 172 func (sink *influxdbSink) Name() string { 173 return "InfluxDB Sink" 174 } 175 176 func (sink *influxdbSink) Stop() { 177 // nothing needs to be done. 178 } 179 180 func (sink *influxdbSink) ensureClient() error { 181 if sink.client == nil { 182 client, err := influxdb_common.NewClient(sink.c) 183 if err != nil { 184 return err 185 } 186 sink.client = client 187 } 188 189 return nil 190 } 191 192 func (sink *influxdbSink) createDatabase() error { 193 if err := sink.ensureClient(); err != nil { 194 return err 195 } 196 197 if sink.dbExists { 198 return nil 199 } 200 q := influxdb.Query{ 201 Command: fmt.Sprintf(`CREATE DATABASE %s WITH NAME "default"`, sink.c.DbName), 202 } 203 204 if resp, err := sink.client.Query(q); err != nil { 205 if !(resp != nil && resp.Err != nil && strings.Contains(resp.Err.Error(), "already exists")) { 206 err := sink.createRetentionPolicy() 207 if err != nil { 208 return err 209 } 210 } 211 } 212 213 sink.dbExists = true 214 glog.Infof("Created database %q on influxDB server at %q", sink.c.DbName, sink.c.Host) 215 return nil 216 } 217 218 func (sink *influxdbSink) createRetentionPolicy() error { 219 q := influxdb.Query{ 220 Command: fmt.Sprintf(`CREATE RETENTION POLICY "default" ON %s DURATION %s REPLICATION 1 DEFAULT`, sink.c.DbName, sink.c.RetentionPolicy), 221 } 222 223 if resp, err := sink.client.Query(q); err != nil { 224 if !(resp != nil && resp.Err != nil) { 225 return fmt.Errorf("Retention Policy creation failed: %v", err) 226 } 227 } 228 229 glog.Infof("Created retention policy 'default' in database %q on influxDB server at %q", sink.c.DbName, sink.c.Host) 230 return nil 231 } 232 233 // Returns a thread-compatible implementation of influxdb interactions. 234 func new(c influxdb_common.InfluxdbConfig) core.DataSink { 235 client, err := influxdb_common.NewClient(c) 236 if err != nil { 237 glog.Errorf("issues while creating an InfluxDB sink: %v, will retry on use", err) 238 } 239 return &influxdbSink{ 240 client: client, // can be nil 241 c: c, 242 } 243 } 244 245 func CreateInfluxdbSink(uri *url.URL) (core.DataSink, error) { 246 config, err := influxdb_common.BuildConfig(uri) 247 if err != nil { 248 return nil, err 249 } 250 sink := new(*config) 251 glog.Infof("created influxdb sink with options: host:%s user:%s db:%s", config.Host, config.User, config.DbName) 252 return sink, nil 253 }