volcano.sh/volcano@v1.9.0/pkg/scheduler/metrics/source/metrics_client_elasticsearch.go (about)

     1  /*
     2   Copyright 2023 The Volcano Authors.
     3  
     4   Licensed under the Apache License, Version 2.0 (the "License");
     5   you may not use this file except in compliance with the License.
     6   You may obtain a copy of the License at
     7  
     8       http://www.apache.org/licenses/LICENSE-2.0
     9  
    10   Unless required by applicable law or agreed to in writing, software
    11   distributed under the License is distributed on an "AS IS" BASIS,
    12   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   See the License for the specific language governing permissions and
    14   limitations under the License.
    15  */
    16  
    17  package source
    18  
    19  import (
    20  	"bytes"
    21  	"context"
    22  	"crypto/tls"
    23  	"encoding/json"
    24  	"errors"
    25  	"net/http"
    26  	"time"
    27  
    28  	"github.com/elastic/go-elasticsearch/v7"
    29  )
    30  
    31  const (
    32  	// esHostNameField is the field name of host name in the document
    33  	esHostNameField = "host.hostname"
    34  	// esCPUUsageField is the field name of cpu usage in the document
    35  	esCPUUsageField = "host.cpu.usage"
    36  	// esMemUsageField is the field name of mem usage in the document
    37  	esMemUsageField = "system.memory.actual.used.pct"
    38  )
    39  
    40  type ElasticsearchMetricsClient struct {
    41  	address           string
    42  	indexName         string
    43  	es                *elasticsearch.Client
    44  	hostnameFieldName string
    45  }
    46  
    47  func NewElasticsearchMetricsClient(conf map[string]string) (*ElasticsearchMetricsClient, error) {
    48  	address := conf["address"]
    49  	if len(address) == 0 {
    50  		return nil, errors.New("metrics address is empty")
    51  	}
    52  
    53  	e := &ElasticsearchMetricsClient{address: address}
    54  	indexName := conf["elasticsearch.index"]
    55  	if len(indexName) == 0 {
    56  		e.indexName = "metricbeat-*"
    57  	} else {
    58  		e.indexName = indexName
    59  	}
    60  	hostNameFieldName := conf["elasticsearch.hostnameFieldName"]
    61  	if len(hostNameFieldName) == 0 {
    62  		e.hostnameFieldName = esHostNameField
    63  	} else {
    64  		e.hostnameFieldName = hostNameFieldName
    65  	}
    66  	var err error
    67  	insecureSkipVerify := conf["tls.insecureSkipVerify"] == "true"
    68  	e.es, err = elasticsearch.NewClient(elasticsearch.Config{
    69  		Addresses: []string{address},
    70  		Username:  conf["elasticsearch.username"],
    71  		Password:  conf["elasticsearch.password"],
    72  		Transport: &http.Transport{
    73  			TLSClientConfig: &tls.Config{
    74  				InsecureSkipVerify: insecureSkipVerify,
    75  			},
    76  		},
    77  	})
    78  	if err != nil {
    79  		return nil, err
    80  	}
    81  	return e, nil
    82  }
    83  
    84  func (e *ElasticsearchMetricsClient) NodesMetricsAvg(ctx context.Context, nodeMetricsMap map[string]*NodeMetrics) error {
    85  	for nodeName := range nodeMetricsMap {
    86  		nodeMetrics, err := e.NodeMetricsAvg(ctx, nodeName)
    87  		if err != nil {
    88  			return err
    89  		}
    90  		nodeMetricsMap[nodeName] = nodeMetrics
    91  	}
    92  	return nil
    93  }
    94  
    95  func (e *ElasticsearchMetricsClient) NodeMetricsAvg(ctx context.Context, nodeName string) (*NodeMetrics, error) {
    96  	nodeMetrics := &NodeMetrics{}
    97  	var buf bytes.Buffer
    98  	query := map[string]interface{}{
    99  		"size": 0,
   100  		"query": map[string]interface{}{
   101  			"bool": map[string]interface{}{
   102  				"must": []map[string]interface{}{
   103  					{
   104  						"range": map[string]interface{}{
   105  							"@timestamp": map[string]interface{}{
   106  								"gte": "now-" + NODE_METRICS_PERIOD,
   107  								"lt":  "now",
   108  							},
   109  						},
   110  					},
   111  					{
   112  						"term": map[string]interface{}{
   113  							e.hostnameFieldName: nodeName,
   114  						},
   115  					},
   116  				},
   117  			},
   118  		},
   119  		"aggs": map[string]interface{}{
   120  			"cpu": map[string]interface{}{
   121  				"avg": map[string]interface{}{
   122  					"field": esCPUUsageField,
   123  				},
   124  			},
   125  			"mem": map[string]interface{}{
   126  				"avg": map[string]interface{}{
   127  					"field": esMemUsageField,
   128  				},
   129  			},
   130  		},
   131  	}
   132  	if err := json.NewEncoder(&buf).Encode(query); err != nil {
   133  		return nil, err
   134  	}
   135  	res, err := e.es.Search(
   136  		e.es.Search.WithContext(ctx),
   137  		e.es.Search.WithIndex(e.indexName),
   138  		e.es.Search.WithBody(&buf),
   139  	)
   140  	if err != nil {
   141  		return nil, err
   142  	}
   143  	defer res.Body.Close()
   144  	var r struct {
   145  		Aggregations struct {
   146  			CPU struct {
   147  				Value float64 `json:"value"`
   148  			}
   149  			Mem struct {
   150  				Value float64 `json:"value"`
   151  			}
   152  		} `json:"aggregations"`
   153  	}
   154  	if err := json.NewDecoder(res.Body).Decode(&r); err != nil {
   155  		return nil, err
   156  	}
   157  	// The data obtained from Elasticsearch is in decimals and needs to be multiplied by 100.
   158  	nodeMetrics.CPU = r.Aggregations.CPU.Value * 100
   159  	nodeMetrics.Memory = r.Aggregations.Mem.Value * 100
   160  	nodeMetrics.MetricsTime = time.Now()
   161  	return nodeMetrics, nil
   162  }