github.com/galamsiva2020/kubernetes-heapster-monitoring@v0.0.0-20210823134957-3c1baa7c1e70/common/elasticsearch/elasticsearch.go (about) 1 // Copyright 2015 Google Inc. All Rights Reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 package elasticsearch 15 16 import ( 17 "fmt" 18 "net/url" 19 "strconv" 20 "time" 21 22 "github.com/golang/glog" 23 24 "errors" 25 "os" 26 27 elastic2 "gopkg.in/olivere/elastic.v3" 28 elastic5 "gopkg.in/olivere/elastic.v5" 29 ) 30 31 const ( 32 ESIndex = "heapster" 33 ESClusterName = "default" 34 ) 35 36 type ElasticSearchService struct { 37 EsClient *esClient 38 baseIndex string 39 ClusterName string 40 } 41 42 func (esSvc *ElasticSearchService) Index(date time.Time) string { 43 return date.Format(fmt.Sprintf("%s-2006.01.02", esSvc.baseIndex)) 44 } 45 func (esSvc *ElasticSearchService) IndexAlias(typeName string) string { 46 return fmt.Sprintf("%s-%s", esSvc.baseIndex, typeName) 47 } 48 49 func (esSvc *ElasticSearchService) FlushData() error { 50 return esSvc.EsClient.FlushBulk() 51 } 52 53 // SaveDataIntoES save metrics and events to ES by using ES client 54 func (esSvc *ElasticSearchService) SaveData(date time.Time, typeName string, sinkData []interface{}) error { 55 if typeName == "" || len(sinkData) == 0 { 56 return nil 57 } 58 59 indexName := esSvc.Index(date) 60 61 // Use the IndexExists service to check if a specified index exists. 62 exists, err := esSvc.EsClient.IndexExists(indexName) 63 if err != nil { 64 return err 65 } 66 67 if !exists { 68 // Create a new index. 69 createIndex, err := esSvc.EsClient.CreateIndex(indexName, mapping) 70 if err != nil { 71 return err 72 } 73 74 ack := false 75 switch i := createIndex.(type) { 76 case *elastic2.IndicesCreateResult: 77 ack = i.Acknowledged 78 case *elastic5.IndicesCreateResult: 79 ack = i.Acknowledged 80 } 81 if !ack { 82 return errors.New("Failed to acknoledge index creation") 83 } 84 } 85 86 aliases, err := esSvc.EsClient.GetAliases(indexName) 87 if err != nil { 88 return err 89 } 90 aliasName := esSvc.IndexAlias(typeName) 91 92 hasAlias := false 93 switch a := aliases.(type) { 94 case *elastic2.AliasesResult: 95 hasAlias = a.Indices[indexName].HasAlias(aliasName) 96 case *elastic5.AliasesResult: 97 hasAlias = a.Indices[indexName].HasAlias(aliasName) 98 } 99 if !hasAlias { 100 createAlias, err := esSvc.EsClient.AddAlias(indexName, esSvc.IndexAlias(typeName)) 101 if err != nil { 102 return err 103 } 104 105 ack := false 106 switch i := createAlias.(type) { 107 case *elastic2.AliasResult: 108 ack = i.Acknowledged 109 case *elastic5.AliasResult: 110 ack = i.Acknowledged 111 } 112 if !ack { 113 return errors.New("Failed to acknoledge index alias creation") 114 } 115 } 116 117 for _, data := range sinkData { 118 esSvc.EsClient.AddBulkReq(indexName, typeName, data) 119 } 120 121 return nil 122 } 123 124 // CreateElasticSearchConfig creates an ElasticSearch configuration struct 125 // which contains an ElasticSearch client for later use 126 func CreateElasticSearchService(uri *url.URL) (*ElasticSearchService, error) { 127 128 var esSvc ElasticSearchService 129 opts, err := url.ParseQuery(uri.RawQuery) 130 if err != nil { 131 return nil, fmt.Errorf("Failed to parse url's query string: %s", err) 132 } 133 134 version := 5 135 if len(opts["ver"]) > 0 { 136 version, err = strconv.Atoi(opts["ver"][0]) 137 if err != nil { 138 return nil, fmt.Errorf("Failed to parse URL's version value into an int: %v", err) 139 } 140 } 141 142 esSvc.ClusterName = ESClusterName 143 if len(opts["cluster_name"]) > 0 { 144 esSvc.ClusterName = opts["cluster_name"][0] 145 } 146 147 // set the index for es,the default value is "heapster" 148 esSvc.baseIndex = ESIndex 149 if len(opts["index"]) > 0 { 150 esSvc.baseIndex = opts["index"][0] 151 } 152 153 var startupFnsV5 []elastic5.ClientOptionFunc 154 var startupFnsV2 []elastic2.ClientOptionFunc 155 156 // Set the URL endpoints of the ES's nodes. Notice that when sniffing is 157 // enabled, these URLs are used to initially sniff the cluster on startup. 158 if len(opts["nodes"]) > 0 { 159 startupFnsV2 = append(startupFnsV2, elastic2.SetURL(opts["nodes"]...)) 160 startupFnsV5 = append(startupFnsV5, elastic5.SetURL(opts["nodes"]...)) 161 } else if uri.Scheme != "" && uri.Host != "" { 162 startupFnsV2 = append(startupFnsV2, elastic2.SetURL(uri.Scheme+"://"+uri.Host)) 163 startupFnsV5 = append(startupFnsV5, elastic5.SetURL(uri.Scheme+"://"+uri.Host)) 164 } else { 165 return nil, errors.New("There is no node assigned for connecting ES cluster") 166 } 167 168 // If the ES cluster needs authentication, the username and secret 169 // should be set in sink config.Else, set the Authenticate flag to false 170 if len(opts["esUserName"]) > 0 && len(opts["esUserSecret"]) > 0 { 171 startupFnsV2 = append(startupFnsV2, elastic2.SetBasicAuth(opts["esUserName"][0], opts["esUserSecret"][0])) 172 startupFnsV5 = append(startupFnsV5, elastic5.SetBasicAuth(opts["esUserName"][0], opts["esUserSecret"][0])) 173 } 174 175 if len(opts["maxRetries"]) > 0 { 176 maxRetries, err := strconv.Atoi(opts["maxRetries"][0]) 177 if err != nil { 178 return nil, errors.New("Failed to parse URL's maxRetries value into an int") 179 } 180 startupFnsV2 = append(startupFnsV2, elastic2.SetMaxRetries(maxRetries)) 181 startupFnsV5 = append(startupFnsV5, elastic5.SetMaxRetries(maxRetries)) 182 } 183 184 if len(opts["healthCheck"]) > 0 { 185 healthCheck, err := strconv.ParseBool(opts["healthCheck"][0]) 186 if err != nil { 187 return nil, errors.New("Failed to parse URL's healthCheck value into a bool") 188 } 189 startupFnsV2 = append(startupFnsV2, elastic2.SetHealthcheck(healthCheck)) 190 startupFnsV5 = append(startupFnsV5, elastic5.SetHealthcheck(healthCheck)) 191 } 192 193 if len(opts["startupHealthcheckTimeout"]) > 0 { 194 timeout, err := time.ParseDuration(opts["startupHealthcheckTimeout"][0] + "s") 195 if err != nil { 196 return nil, fmt.Errorf("Failed to parse URL's startupHealthcheckTimeout: %s", err.Error()) 197 } 198 startupFnsV2 = append(startupFnsV2, elastic2.SetHealthcheckTimeoutStartup(timeout)) 199 startupFnsV5 = append(startupFnsV5, elastic5.SetHealthcheckTimeoutStartup(timeout)) 200 } 201 202 if os.Getenv("AWS_ACCESS_KEY_ID") != "" || os.Getenv("AWS_ACCESS_KEY") != "" || 203 os.Getenv("AWS_SECRET_ACCESS_KEY") != "" || os.Getenv("AWS_SECRET_KEY") != "" { 204 glog.Info("Configuring with AWS credentials..") 205 206 awsClient, err := createAWSClient() 207 if err != nil { 208 return nil, err 209 } 210 211 startupFnsV2 = append(startupFnsV2, elastic2.SetHttpClient(awsClient), elastic2.SetSniff(false)) 212 startupFnsV5 = append(startupFnsV5, elastic5.SetHttpClient(awsClient), elastic5.SetSniff(false)) 213 } else { 214 if len(opts["sniff"]) > 0 { 215 sniff, err := strconv.ParseBool(opts["sniff"][0]) 216 if err != nil { 217 return nil, errors.New("Failed to parse URL's sniff value into a bool") 218 } 219 startupFnsV2 = append(startupFnsV2, elastic2.SetSniff(sniff)) 220 startupFnsV5 = append(startupFnsV5, elastic5.SetSniff(sniff)) 221 } 222 } 223 224 bulkWorkers := 5 225 if len(opts["bulkWorkers"]) > 0 { 226 bulkWorkers, err = strconv.Atoi(opts["bulkWorkers"][0]) 227 if err != nil { 228 return nil, errors.New("Failed to parse URL's bulkWorkers value into an int") 229 } 230 } 231 232 pipeline := "" 233 if len(opts["pipeline"]) > 0 { 234 pipeline = opts["pipeline"][0] 235 } 236 237 switch version { 238 case 2: 239 esSvc.EsClient, err = newEsClientV2(startupFnsV2, bulkWorkers) 240 case 5: 241 esSvc.EsClient, err = newEsClientV5(startupFnsV5, bulkWorkers, pipeline) 242 default: 243 return nil, UnsupportedVersion{} 244 } 245 if err != nil { 246 return nil, fmt.Errorf("Failed to create ElasticSearch client: %v", err) 247 } 248 249 glog.V(2).Infof("ElasticSearch sink configure successfully") 250 251 return &esSvc, nil 252 }