github.com/kubewharf/katalyst-core@v0.5.3/pkg/metaserver/agent/metric/provisioner/kubelet/provisioner.go (about)

     1  /*
     2  Copyright 2022 The Katalyst Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package kubelet
    18  
    19  import (
    20  	"context"
    21  	"sync"
    22  
    23  	"k8s.io/klog/v2"
    24  	statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
    25  
    26  	"github.com/kubewharf/katalyst-core/pkg/config/agent/global"
    27  	"github.com/kubewharf/katalyst-core/pkg/config/agent/metaserver"
    28  	"github.com/kubewharf/katalyst-core/pkg/consts"
    29  	"github.com/kubewharf/katalyst-core/pkg/metaserver/agent/metric/provisioner/kubelet/client"
    30  	"github.com/kubewharf/katalyst-core/pkg/metaserver/agent/metric/types"
    31  	"github.com/kubewharf/katalyst-core/pkg/metaserver/agent/pod"
    32  	"github.com/kubewharf/katalyst-core/pkg/metrics"
    33  	utilmetric "github.com/kubewharf/katalyst-core/pkg/util/metric"
    34  )
    35  
    36  const (
    37  	metricsNamKubeletSummaryUnHealthy = "kubelet_summary_unhealthy"
    38  )
    39  
    40  func NewKubeletSummaryProvisioner(baseConf *global.BaseConfiguration, _ *metaserver.MetricConfiguration,
    41  	emitter metrics.MetricEmitter, _ pod.PodFetcher, metricStore *utilmetric.MetricStore,
    42  ) types.MetricsProvisioner {
    43  	return &KubeletSummaryProvisioner{
    44  		metricStore: metricStore,
    45  		emitter:     emitter,
    46  		client:      client.NewKubeletSummaryClient(baseConf),
    47  	}
    48  }
    49  
    50  type KubeletSummaryProvisioner struct {
    51  	metricStore *utilmetric.MetricStore
    52  	emitter     metrics.MetricEmitter
    53  	client      *client.KubeletSummaryClient
    54  
    55  	startOnce sync.Once
    56  	hasSynced bool
    57  }
    58  
    59  func (p *KubeletSummaryProvisioner) Run(ctx context.Context) {
    60  	p.sample(ctx)
    61  }
    62  
    63  func (p *KubeletSummaryProvisioner) sample(ctx context.Context) {
    64  	summary, err := p.client.Summary(ctx)
    65  	if err != nil {
    66  		klog.Errorf("failed to update stats/summary from kubelet: %q", err)
    67  		_ = p.emitter.StoreInt64(metricsNamKubeletSummaryUnHealthy, 1, metrics.MetricTypeNameRaw)
    68  		return
    69  	}
    70  
    71  	p.processNodeFsStats(summary.Node.Fs)
    72  	if summary.Node.Runtime != nil {
    73  		p.processImageFsStats(summary.Node.Runtime.ImageFs)
    74  	}
    75  
    76  	for _, podStats := range summary.Pods {
    77  		for _, volumeStats := range podStats.VolumeStats {
    78  			p.processVolumeStats(podStats.PodRef.UID, &volumeStats)
    79  		}
    80  
    81  		for _, containerStats := range podStats.Containers {
    82  			p.processContainerRootfsStats(podStats.PodRef.UID, &containerStats)
    83  			p.processContainerLogsStats(podStats.PodRef.UID, &containerStats)
    84  		}
    85  	}
    86  }
    87  
    88  func (p *KubeletSummaryProvisioner) processNodeFsStats(nodeFsStats *statsapi.FsStats) {
    89  	if nodeFsStats == nil {
    90  		return
    91  	}
    92  	updateTime := nodeFsStats.Time.Time
    93  	if nodeFsStats.AvailableBytes != nil {
    94  		p.metricStore.SetNodeMetric(consts.MetricsNodeFsAvailable, utilmetric.MetricData{Value: float64(*nodeFsStats.AvailableBytes), Time: &updateTime})
    95  	}
    96  	if nodeFsStats.CapacityBytes != nil {
    97  		p.metricStore.SetNodeMetric(consts.MetricsNodeFsCapacity, utilmetric.MetricData{Value: float64(*nodeFsStats.CapacityBytes), Time: &updateTime})
    98  	}
    99  	if nodeFsStats.UsedBytes != nil {
   100  		p.metricStore.SetNodeMetric(consts.MetricsNodeFsUsed, utilmetric.MetricData{Value: float64(*nodeFsStats.UsedBytes), Time: &updateTime})
   101  	}
   102  	if nodeFsStats.InodesFree != nil {
   103  		p.metricStore.SetNodeMetric(consts.MetricsNodeFsInodesFree, utilmetric.MetricData{Value: float64(*nodeFsStats.InodesFree), Time: &updateTime})
   104  	}
   105  	if nodeFsStats.InodesUsed != nil {
   106  		p.metricStore.SetNodeMetric(consts.MetricsNodeFsInodesUsed, utilmetric.MetricData{Value: float64(*nodeFsStats.InodesUsed), Time: &updateTime})
   107  	}
   108  	if nodeFsStats.Inodes != nil {
   109  		p.metricStore.SetNodeMetric(consts.MetricsNodeFsInodes, utilmetric.MetricData{Value: float64(*nodeFsStats.Inodes), Time: &updateTime})
   110  	}
   111  }
   112  
   113  func (p *KubeletSummaryProvisioner) processImageFsStats(imageFsStats *statsapi.FsStats) {
   114  	if imageFsStats == nil {
   115  		return
   116  	}
   117  	updateTime := imageFsStats.Time.Time
   118  	if imageFsStats.AvailableBytes != nil {
   119  		p.metricStore.SetNodeMetric(consts.MetricsImageFsAvailable, utilmetric.MetricData{Value: float64(*imageFsStats.AvailableBytes), Time: &updateTime})
   120  	}
   121  	if imageFsStats.CapacityBytes != nil {
   122  		p.metricStore.SetNodeMetric(consts.MetricsImageFsCapacity, utilmetric.MetricData{Value: float64(*imageFsStats.CapacityBytes), Time: &updateTime})
   123  	}
   124  	if imageFsStats.UsedBytes != nil {
   125  		p.metricStore.SetNodeMetric(consts.MetricsImageFsUsed, utilmetric.MetricData{Value: float64(*imageFsStats.UsedBytes), Time: &updateTime})
   126  	}
   127  	if imageFsStats.InodesFree != nil {
   128  		p.metricStore.SetNodeMetric(consts.MetricsImageFsInodesFree, utilmetric.MetricData{Value: float64(*imageFsStats.InodesFree), Time: &updateTime})
   129  	}
   130  	if imageFsStats.InodesUsed != nil {
   131  		p.metricStore.SetNodeMetric(consts.MetricsImageFsInodesUsed, utilmetric.MetricData{Value: float64(*imageFsStats.InodesUsed), Time: &updateTime})
   132  	}
   133  	if imageFsStats.Inodes != nil {
   134  		p.metricStore.SetNodeMetric(consts.MetricsImageFsInodes, utilmetric.MetricData{Value: float64(*imageFsStats.Inodes), Time: &updateTime})
   135  	}
   136  }
   137  
   138  func (p *KubeletSummaryProvisioner) processVolumeStats(podUID string, volumeStats *statsapi.VolumeStats) {
   139  	updateTime := volumeStats.Time.Time
   140  	if volumeStats.AvailableBytes != nil {
   141  		p.metricStore.SetPodVolumeMetric(podUID, volumeStats.Name, consts.MetricsPodVolumeAvailable, utilmetric.MetricData{Value: float64(*volumeStats.AvailableBytes), Time: &updateTime})
   142  	}
   143  	if volumeStats.CapacityBytes != nil {
   144  		p.metricStore.SetPodVolumeMetric(podUID, volumeStats.Name, consts.MetricsPodVolumeCapacity, utilmetric.MetricData{Value: float64(*volumeStats.CapacityBytes), Time: &updateTime})
   145  	}
   146  	if volumeStats.UsedBytes != nil {
   147  		p.metricStore.SetPodVolumeMetric(podUID, volumeStats.Name, consts.MetricsPodVolumeUsed, utilmetric.MetricData{Value: float64(*volumeStats.UsedBytes), Time: &updateTime})
   148  	}
   149  	if volumeStats.Inodes != nil {
   150  		p.metricStore.SetPodVolumeMetric(podUID, volumeStats.Name, consts.MetricsPodVolumeInodes, utilmetric.MetricData{Value: float64(*volumeStats.Inodes), Time: &updateTime})
   151  	}
   152  	if volumeStats.InodesFree != nil {
   153  		p.metricStore.SetPodVolumeMetric(podUID, volumeStats.Name, consts.MetricsPodVolumeInodesFree, utilmetric.MetricData{Value: float64(*volumeStats.InodesFree), Time: &updateTime})
   154  	}
   155  	if volumeStats.InodesUsed != nil {
   156  		p.metricStore.SetPodVolumeMetric(podUID, volumeStats.Name, consts.MetricsPodVolumeInodesUsed, utilmetric.MetricData{Value: float64(*volumeStats.InodesUsed), Time: &updateTime})
   157  	}
   158  }
   159  
   160  func (p *KubeletSummaryProvisioner) processContainerRootfsStats(podUID string, containerStats *statsapi.ContainerStats) {
   161  	if containerStats.Rootfs == nil {
   162  		return
   163  	}
   164  	updateTime := containerStats.Rootfs.Time.Time
   165  	if containerStats.Rootfs.AvailableBytes != nil {
   166  		p.metricStore.SetContainerMetric(podUID, containerStats.Name, consts.MetricsContainerRootfsAvailable, utilmetric.MetricData{Value: float64(*containerStats.Rootfs.AvailableBytes), Time: &updateTime})
   167  	}
   168  	if containerStats.Rootfs.CapacityBytes != nil {
   169  		p.metricStore.SetContainerMetric(podUID, containerStats.Name, consts.MetricsContainerRootfsCapacity, utilmetric.MetricData{Value: float64(*containerStats.Rootfs.CapacityBytes), Time: &updateTime})
   170  	}
   171  	if containerStats.Rootfs.UsedBytes != nil {
   172  		p.metricStore.SetContainerMetric(podUID, containerStats.Name, consts.MetricsContainerRootfsUsed, utilmetric.MetricData{Value: float64(*containerStats.Rootfs.UsedBytes), Time: &updateTime})
   173  	}
   174  	if containerStats.Rootfs.Inodes != nil {
   175  		p.metricStore.SetContainerMetric(podUID, containerStats.Name, consts.MetricsContainerRootfsInodes, utilmetric.MetricData{Value: float64(*containerStats.Rootfs.Inodes), Time: &updateTime})
   176  	}
   177  	if containerStats.Rootfs.InodesFree != nil {
   178  		p.metricStore.SetContainerMetric(podUID, containerStats.Name, consts.MetricsContainerRootfsInodesFree, utilmetric.MetricData{Value: float64(*containerStats.Rootfs.InodesFree), Time: &updateTime})
   179  	}
   180  	if containerStats.Rootfs.InodesUsed != nil {
   181  		p.metricStore.SetContainerMetric(podUID, containerStats.Name, consts.MetricsContainerRootfsInodesUsed, utilmetric.MetricData{Value: float64(*containerStats.Rootfs.InodesUsed), Time: &updateTime})
   182  	}
   183  }
   184  
   185  func (p *KubeletSummaryProvisioner) processContainerLogsStats(podUID string, containerStats *statsapi.ContainerStats) {
   186  	if containerStats.Logs == nil {
   187  		return
   188  	}
   189  	updateTime := containerStats.Logs.Time.Time
   190  	if containerStats.Logs.AvailableBytes != nil {
   191  		p.metricStore.SetContainerMetric(podUID, containerStats.Name, consts.MetricsLogsAvailable, utilmetric.MetricData{Value: float64(*containerStats.Logs.AvailableBytes), Time: &updateTime})
   192  	}
   193  	if containerStats.Logs.CapacityBytes != nil {
   194  		p.metricStore.SetContainerMetric(podUID, containerStats.Name, consts.MetricsLogsCapacity, utilmetric.MetricData{Value: float64(*containerStats.Logs.CapacityBytes), Time: &updateTime})
   195  	}
   196  	if containerStats.Logs.Inodes != nil {
   197  		p.metricStore.SetContainerMetric(podUID, containerStats.Name, consts.MetricsLogsInodes, utilmetric.MetricData{Value: float64(*containerStats.Logs.Inodes), Time: &updateTime})
   198  	}
   199  	if containerStats.Logs.InodesFree != nil {
   200  		p.metricStore.SetContainerMetric(podUID, containerStats.Name, consts.MetricsLogsInodesFree, utilmetric.MetricData{Value: float64(*containerStats.Logs.InodesFree), Time: &updateTime})
   201  	}
   202  	if containerStats.Logs.InodesUsed != nil {
   203  		p.metricStore.SetContainerMetric(podUID, containerStats.Name, consts.MetricsLogsInodesUsed, utilmetric.MetricData{Value: float64(*containerStats.Logs.InodesUsed), Time: &updateTime})
   204  	}
   205  }