github.com/oam-dev/kubevela@v1.9.11/pkg/multicluster/cluster_metrics_management.go (about)

     1  /*
     2  Copyright 2022 The KubeVela Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8  	http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package multicluster
    18  
    19  import (
    20  	"context"
    21  	"time"
    22  
    23  	"github.com/oam-dev/kubevela/pkg/monitor/metrics"
    24  
    25  	"k8s.io/klog/v2"
    26  	"sigs.k8s.io/controller-runtime/pkg/client"
    27  )
    28  
    29  // metricsMap records the metrics of clusters
    30  var metricsMap map[string]*ClusterMetrics
    31  
    32  // ClusterMetricsMgr manage metrics of clusters
    33  type ClusterMetricsMgr struct {
    34  	kubeClient    client.Client
    35  	refreshPeriod time.Duration
    36  }
    37  
    38  // ClusterMetricsHelper is the interface that provides operations for cluster metrics
    39  type ClusterMetricsHelper interface {
    40  	Refresh() error
    41  }
    42  
    43  // NewClusterMetricsMgr will create a cluster metrics manager
    44  func NewClusterMetricsMgr(ctx context.Context, kubeClient client.Client, refreshPeriod time.Duration) (*ClusterMetricsMgr, error) {
    45  	mgr := &ClusterMetricsMgr{
    46  		kubeClient:    kubeClient,
    47  		refreshPeriod: refreshPeriod,
    48  	}
    49  	go mgr.Start(ctx)
    50  	return mgr, nil
    51  }
    52  
    53  // Refresh will re-collect cluster metrics and refresh cache
    54  func (cmm *ClusterMetricsMgr) Refresh() ([]VirtualCluster, error) {
    55  	clusters, _ := ListVirtualClusters(context.Background(), cmm.kubeClient)
    56  	m := make(map[string]*ClusterMetrics)
    57  
    58  	// retrieves metrics by cluster-gateway
    59  	for _, cluster := range clusters {
    60  		isConnected := true
    61  		clusterInfo, err := GetClusterInfo(context.Background(), cmm.kubeClient, cluster.Name)
    62  		if err != nil {
    63  			klog.Warningf("failed to get cluster info of cluster-(%s)", cluster.Name)
    64  			isConnected = false
    65  		}
    66  		clusterUsageMetrics, err := GetClusterMetricsFromMetricsAPI(context.Background(), cmm.kubeClient, cluster.Name)
    67  		if err != nil {
    68  			klog.Warningf("failed to request metrics api of cluster-(%s)", cluster.Name)
    69  		}
    70  		cm := &ClusterMetrics{
    71  			IsConnected:         isConnected,
    72  			ClusterInfo:         clusterInfo,
    73  			ClusterUsageMetrics: clusterUsageMetrics,
    74  		}
    75  		m[cluster.Name] = cm
    76  		cluster.Metrics = cm
    77  	}
    78  	metricsMap = m
    79  	return clusters, nil
    80  }
    81  
    82  // Start will start polling cluster api to collect metrics
    83  func (cmm *ClusterMetricsMgr) Start(ctx context.Context) {
    84  	for {
    85  		select {
    86  		case <-ctx.Done():
    87  			klog.Warning("Stop cluster metrics polling loop.")
    88  			return
    89  		default:
    90  			clusters, _ := cmm.Refresh()
    91  			for _, cluster := range clusters {
    92  				exportMetrics(cluster.Metrics, cluster.Name)
    93  			}
    94  			time.Sleep(cmm.refreshPeriod)
    95  		}
    96  	}
    97  }
    98  
    99  // exportMetrics will report ClusterMetrics with a clusterName label
   100  func exportMetrics(m *ClusterMetrics, clusterName string) {
   101  	if m == nil {
   102  		return
   103  	}
   104  	metrics.ClusterIsConnectedGauge.WithLabelValues(clusterName).Set(func() float64 {
   105  		if m.IsConnected {
   106  			return 1
   107  		}
   108  		return 0
   109  	}())
   110  	if m.ClusterInfo != nil {
   111  		metrics.ClusterWorkerNumberGauge.WithLabelValues(clusterName).Set(float64(m.ClusterInfo.WorkerNumber))
   112  		metrics.ClusterMasterNumberGauge.WithLabelValues(clusterName).Set(float64(m.ClusterInfo.MasterNumber))
   113  		metrics.ClusterMemoryCapacityGauge.WithLabelValues(clusterName).Set(m.ClusterInfo.MemoryCapacity.AsApproximateFloat64())
   114  		metrics.ClusterCPUCapacityGauge.WithLabelValues(clusterName).Set(float64(m.ClusterInfo.CPUCapacity.MilliValue()))
   115  		metrics.ClusterPodCapacityGauge.WithLabelValues(clusterName).Set(m.ClusterInfo.PodCapacity.AsApproximateFloat64())
   116  		metrics.ClusterMemoryAllocatableGauge.WithLabelValues(clusterName).Set(m.ClusterInfo.MemoryAllocatable.AsApproximateFloat64())
   117  		metrics.ClusterCPUAllocatableGauge.WithLabelValues(clusterName).Set(float64(m.ClusterInfo.CPUAllocatable.MilliValue()))
   118  		metrics.ClusterPodAllocatableGauge.WithLabelValues(clusterName).Set(m.ClusterInfo.PodAllocatable.AsApproximateFloat64())
   119  	}
   120  	if m.ClusterUsageMetrics != nil {
   121  		metrics.ClusterMemoryUsageGauge.WithLabelValues(clusterName).Set(m.ClusterUsageMetrics.MemoryUsage.AsApproximateFloat64())
   122  		metrics.ClusterCPUUsageGauge.WithLabelValues(clusterName).Set(float64(m.ClusterUsageMetrics.CPUUsage.MilliValue()))
   123  	}
   124  }