github.com/galamsiva2020/kubernetes-heapster-monitoring@v0.0.0-20210823134957-3c1baa7c1e70/metrics/sinks/elasticsearch/driver.go (about) 1 // Copyright 2015 Google Inc. All Rights Reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package elasticsearch 16 17 import ( 18 "net/url" 19 "sync" 20 "time" 21 22 "github.com/golang/glog" 23 esCommon "k8s.io/heapster/common/elasticsearch" 24 "k8s.io/heapster/metrics/core" 25 "reflect" 26 ) 27 28 // SaveDataFunc is a pluggable function to enforce limits on the object 29 type SaveDataFunc func(date time.Time, typeName string, sinkData []interface{}) error 30 31 type elasticSearchSink struct { 32 esSvc esCommon.ElasticSearchService 33 saveData SaveDataFunc 34 flushData func() error 35 sync.RWMutex 36 } 37 38 type EsFamilyPoints map[core.MetricFamily][]interface{} 39 type esPointTags map[string]string 40 type customTimestamp map[string]time.Time 41 42 type EsSinkPointGeneral struct { 43 GeneralMetricsTimestamp time.Time 44 MetricsTags esPointTags 45 MetricsName string 46 MetricsValue interface{} 47 } 48 type EsSinkPointFamily map[string]interface{} 49 50 func (sink *elasticSearchSink) ExportData(dataBatch *core.DataBatch) { 51 sink.Lock() 52 defer sink.Unlock() 53 54 for _, metricSet := range dataBatch.MetricSets { 55 familyPoints := EsFamilyPoints{} 56 57 for metricName, metricValue := range metricSet.MetricValues { 58 familyPoints = addMetric(familyPoints, metricName, dataBatch.Timestamp, metricSet.Labels, metricValue.GetValue(), sink.esSvc.ClusterName) 59 } 60 for _, metric := range metricSet.LabeledMetrics { 61 labels := make(map[string]string) 62 for k, v := range metricSet.Labels { 63 labels[k] = v 64 } 65 for k, v := range metric.Labels { 66 labels[k] = v 67 } 68 69 familyPoints = addMetric(familyPoints, metric.Name, dataBatch.Timestamp, labels, metric.GetValue(), sink.esSvc.ClusterName) 70 } 71 72 for family, dataPoints := range familyPoints { 73 err := sink.saveData(dataBatch.Timestamp.UTC(), string(family), dataPoints) 74 if err != nil { 75 glog.Warningf("Failed to export data to ElasticSearch sink: %v", err) 76 } 77 } 78 err := sink.flushData() 79 if err != nil { 80 glog.Warningf("Failed to flushing data to ElasticSearch sink: %v", err) 81 } 82 } 83 } 84 85 func addMetric(points EsFamilyPoints, metricName string, date time.Time, tags esPointTags, value interface{}, clusterName string) EsFamilyPoints { 86 family := core.MetricFamilyForName(metricName) 87 88 if points[family] == nil { 89 points[family] = []interface{}{} 90 } 91 92 if family == core.MetricFamilyGeneral { 93 point := EsSinkPointGeneral{} 94 point.MetricsTags = tags 95 point.MetricsTags["cluster_name"] = clusterName 96 point.GeneralMetricsTimestamp = date.UTC() 97 point.MetricsName = metricName 98 point.MetricsValue = EsPointValue(value) 99 100 //add 101 points[family] = append(points[family], point) 102 return points 103 } 104 105 for idx, pt := range points[family] { 106 if point, ok := pt.(EsSinkPointFamily); ok { 107 if point[esCommon.MetricFamilyTimestamp(family)] == date.UTC() && reflect.DeepEqual(point["MetricsTags"], tags) { 108 if metrics, ok := point["Metrics"].(map[string]interface{}); ok { 109 metrics[metricName] = EsPointValue(value) 110 point["Metrics"] = metrics 111 } else { 112 glog.Warningf("Failed to cast metrics to map") 113 } 114 115 if tags, ok := point["MetricsTags"].(esPointTags); ok { 116 tags["cluster_name"] = clusterName 117 point["MetricsTags"] = tags 118 } else { 119 glog.Warningf("Failed to cast metricstags to map") 120 } 121 122 //add 123 points[family][idx] = point 124 return points 125 } 126 } 127 } 128 129 point := EsSinkPointFamily{} 130 point[esCommon.MetricFamilyTimestamp(family)] = date.UTC() 131 tags["cluster_name"] = clusterName 132 point["MetricsTags"] = tags 133 metrics := make(map[string]interface{}) 134 metrics[metricName] = EsPointValue(value) 135 point["Metrics"] = metrics 136 137 //add 138 points[family] = append(points[family], point) 139 return points 140 } 141 142 func EsPointValue(value interface{}) interface{} { 143 return map[string]interface{}{ 144 "value": value, 145 } 146 } 147 148 func (sink *elasticSearchSink) Name() string { 149 return "ElasticSearch Sink" 150 } 151 152 func (sink *elasticSearchSink) Stop() { 153 // nothing needs to be done. 154 } 155 156 func NewElasticSearchSink(uri *url.URL) (core.DataSink, error) { 157 var esSink elasticSearchSink 158 esSvc, err := esCommon.CreateElasticSearchService(uri) 159 if err != nil { 160 glog.Warningf("Failed to config ElasticSearch: %v", err) 161 return nil, err 162 } 163 164 esSink.esSvc = *esSvc 165 esSink.saveData = func(date time.Time, typeName string, sinkData []interface{}) error { 166 return esSvc.SaveData(date, typeName, sinkData) 167 } 168 esSink.flushData = func() error { 169 return esSvc.FlushData() 170 } 171 172 glog.V(2).Info("ElasticSearch sink setup successfully") 173 return &esSink, nil 174 }