k8s.io/kubernetes@v1.31.0-alpha.0.0.20240520171757-56147500dadc/pkg/kubemark/hollow_kubelet.go (about)

     1  /*
     2  Copyright 2015 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package kubemark
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"time"
    23  
    24  	"go.opentelemetry.io/otel/trace"
    25  	v1 "k8s.io/api/core/v1"
    26  	"k8s.io/klog/v2"
    27  	"k8s.io/mount-utils"
    28  
    29  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    30  	clientset "k8s.io/client-go/kubernetes"
    31  	"k8s.io/client-go/tools/record"
    32  	internalapi "k8s.io/cri-api/pkg/apis"
    33  	kubeletapp "k8s.io/kubernetes/cmd/kubelet/app"
    34  	"k8s.io/kubernetes/cmd/kubelet/app/options"
    35  	"k8s.io/kubernetes/pkg/kubelet"
    36  	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
    37  	"k8s.io/kubernetes/pkg/kubelet/cadvisor"
    38  	"k8s.io/kubernetes/pkg/kubelet/cm"
    39  	containertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
    40  	probetest "k8s.io/kubernetes/pkg/kubelet/prober/testing"
    41  	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
    42  	kubeletutil "k8s.io/kubernetes/pkg/kubelet/util"
    43  	"k8s.io/kubernetes/pkg/util/oom"
    44  	"k8s.io/kubernetes/pkg/volume"
    45  	"k8s.io/kubernetes/pkg/volume/configmap"
    46  	"k8s.io/kubernetes/pkg/volume/csi"
    47  	"k8s.io/kubernetes/pkg/volume/downwardapi"
    48  	"k8s.io/kubernetes/pkg/volume/emptydir"
    49  	"k8s.io/kubernetes/pkg/volume/fc"
    50  	"k8s.io/kubernetes/pkg/volume/git_repo"
    51  	"k8s.io/kubernetes/pkg/volume/hostpath"
    52  	"k8s.io/kubernetes/pkg/volume/iscsi"
    53  	"k8s.io/kubernetes/pkg/volume/local"
    54  	"k8s.io/kubernetes/pkg/volume/nfs"
    55  	"k8s.io/kubernetes/pkg/volume/portworx"
    56  	"k8s.io/kubernetes/pkg/volume/projected"
    57  	"k8s.io/kubernetes/pkg/volume/secret"
    58  	"k8s.io/kubernetes/pkg/volume/util/hostutil"
    59  	"k8s.io/kubernetes/pkg/volume/util/subpath"
    60  	"k8s.io/kubernetes/test/utils"
    61  )
    62  
    63  type HollowKubelet struct {
    64  	KubeletFlags         *options.KubeletFlags
    65  	KubeletConfiguration *kubeletconfig.KubeletConfiguration
    66  	KubeletDeps          *kubelet.Dependencies
    67  }
    68  
    69  func volumePlugins() []volume.VolumePlugin {
    70  	allPlugins := []volume.VolumePlugin{}
    71  	allPlugins = append(allPlugins, emptydir.ProbeVolumePlugins()...)
    72  	allPlugins = append(allPlugins, git_repo.ProbeVolumePlugins()...)
    73  	allPlugins = append(allPlugins, hostpath.FakeProbeVolumePlugins(volume.VolumeConfig{})...)
    74  	allPlugins = append(allPlugins, nfs.ProbeVolumePlugins(volume.VolumeConfig{})...)
    75  	allPlugins = append(allPlugins, secret.ProbeVolumePlugins()...)
    76  	allPlugins = append(allPlugins, iscsi.ProbeVolumePlugins()...)
    77  	allPlugins = append(allPlugins, downwardapi.ProbeVolumePlugins()...)
    78  	allPlugins = append(allPlugins, fc.ProbeVolumePlugins()...)
    79  	allPlugins = append(allPlugins, configmap.ProbeVolumePlugins()...)
    80  	allPlugins = append(allPlugins, projected.ProbeVolumePlugins()...)
    81  	allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...)
    82  	allPlugins = append(allPlugins, local.ProbeVolumePlugins()...)
    83  	allPlugins = append(allPlugins, csi.ProbeVolumePlugins()...)
    84  	return allPlugins
    85  }
    86  
    87  func NewHollowKubelet(
    88  	flags *options.KubeletFlags,
    89  	config *kubeletconfig.KubeletConfiguration,
    90  	client *clientset.Clientset,
    91  	heartbeatClient *clientset.Clientset,
    92  	cadvisorInterface cadvisor.Interface,
    93  	imageService internalapi.ImageManagerService,
    94  	runtimeService internalapi.RuntimeService,
    95  	containerManager cm.ContainerManager) *HollowKubelet {
    96  	d := &kubelet.Dependencies{
    97  		KubeClient:                client,
    98  		HeartbeatClient:           heartbeatClient,
    99  		ProbeManager:              probetest.FakeManager{},
   100  		RemoteRuntimeService:      runtimeService,
   101  		RemoteImageService:        imageService,
   102  		CAdvisorInterface:         cadvisorInterface,
   103  		Cloud:                     nil,
   104  		OSInterface:               &containertest.FakeOS{},
   105  		ContainerManager:          containerManager,
   106  		VolumePlugins:             volumePlugins(),
   107  		TLSOptions:                nil,
   108  		OOMAdjuster:               oom.NewFakeOOMAdjuster(),
   109  		Mounter:                   &mount.FakeMounter{},
   110  		Subpather:                 &subpath.FakeSubpath{},
   111  		HostUtil:                  hostutil.NewFakeHostUtil(nil),
   112  		PodStartupLatencyTracker:  kubeletutil.NewPodStartupLatencyTracker(),
   113  		NodeStartupLatencyTracker: kubeletutil.NewNodeStartupLatencyTracker(),
   114  		TracerProvider:            trace.NewNoopTracerProvider(),
   115  		Recorder:                  &record.FakeRecorder{}, // With real recorder we attempt to read /dev/kmsg.
   116  	}
   117  
   118  	return &HollowKubelet{
   119  		KubeletFlags:         flags,
   120  		KubeletConfiguration: config,
   121  		KubeletDeps:          d,
   122  	}
   123  }
   124  
   125  // Starts this HollowKubelet and blocks.
   126  func (hk *HollowKubelet) Run(ctx context.Context) {
   127  	if err := kubeletapp.RunKubelet(ctx, &options.KubeletServer{
   128  		KubeletFlags:         *hk.KubeletFlags,
   129  		KubeletConfiguration: *hk.KubeletConfiguration,
   130  	}, hk.KubeletDeps, false); err != nil {
   131  		klog.Fatalf("Failed to run HollowKubelet: %v. Exiting.", err)
   132  	}
   133  	select {}
   134  }
   135  
   136  // HollowKubeletOptions contains settable parameters for hollow kubelet.
   137  type HollowKubeletOptions struct {
   138  	NodeName            string
   139  	KubeletPort         int
   140  	KubeletReadOnlyPort int
   141  	MaxPods             int
   142  	PodsPerCore         int
   143  	NodeLabels          map[string]string
   144  	RegisterWithTaints  []v1.Taint
   145  }
   146  
   147  // Builds a KubeletConfiguration for the HollowKubelet, ensuring that the
   148  // usual defaults are applied for fields we do not override.
   149  func GetHollowKubeletConfig(opt *HollowKubeletOptions) (*options.KubeletFlags, *kubeletconfig.KubeletConfiguration) {
   150  	testRootDir := utils.MakeTempDirOrDie("hollow-kubelet.", "")
   151  	podFilePath := utils.MakeTempDirOrDie("static-pods", testRootDir)
   152  	podLogsPath := utils.MakeTempDirOrDie("pod-logs", testRootDir)
   153  	klog.Infof("Using %s as root dir for hollow-kubelet", testRootDir)
   154  
   155  	// Flags struct
   156  	f := options.NewKubeletFlags()
   157  	f.RootDirectory = testRootDir
   158  	f.HostnameOverride = opt.NodeName
   159  	f.MinimumGCAge = metav1.Duration{Duration: 1 * time.Minute}
   160  	f.MaxContainerCount = 100
   161  	f.MaxPerPodContainerCount = 2
   162  	f.NodeLabels = opt.NodeLabels
   163  	f.RegisterSchedulable = true
   164  
   165  	// Config struct
   166  	c, err := options.NewKubeletConfiguration()
   167  	if err != nil {
   168  		panic(err)
   169  	}
   170  
   171  	c.ImageServiceEndpoint = "unix:///run/containerd/containerd.sock"
   172  	c.StaticPodURL = ""
   173  	c.EnableServer = true
   174  	c.Address = "0.0.0.0" /* bind address */
   175  	c.Port = int32(opt.KubeletPort)
   176  	c.ReadOnlyPort = int32(opt.KubeletReadOnlyPort)
   177  	c.StaticPodPath = podFilePath
   178  	c.FileCheckFrequency.Duration = 20 * time.Second
   179  	c.HTTPCheckFrequency.Duration = 20 * time.Second
   180  	c.NodeStatusUpdateFrequency.Duration = 10 * time.Second
   181  	c.NodeStatusReportFrequency.Duration = 5 * time.Minute
   182  	c.SyncFrequency.Duration = 10 * time.Second
   183  	c.EvictionPressureTransitionPeriod.Duration = 5 * time.Minute
   184  	c.MaxPods = int32(opt.MaxPods)
   185  	c.PodsPerCore = int32(opt.PodsPerCore)
   186  	c.ClusterDNS = []string{}
   187  	c.ImageGCHighThresholdPercent = 90
   188  	c.ImageGCLowThresholdPercent = 80
   189  	c.ProviderID = fmt.Sprintf("kubemark://%v", opt.NodeName)
   190  	c.VolumeStatsAggPeriod.Duration = time.Minute
   191  	c.CgroupRoot = ""
   192  	c.CPUCFSQuota = true
   193  	c.EnableControllerAttachDetach = false
   194  	c.EnableDebuggingHandlers = true
   195  	c.CgroupsPerQOS = false
   196  	// hairpin-veth is used to allow hairpin packets. Note that this deviates from
   197  	// what the "real" kubelet currently does, because there's no way to
   198  	// set promiscuous mode on docker0.
   199  	c.HairpinMode = kubeletconfig.HairpinVeth
   200  	c.MaxOpenFiles = 1024
   201  	c.RegistryBurst = 10
   202  	c.RegistryPullQPS = 5.0
   203  	c.ResolverConfig = kubetypes.ResolvConfDefault
   204  	c.KubeletCgroups = "/kubelet"
   205  	c.SerializeImagePulls = true
   206  	c.SystemCgroups = ""
   207  	c.ProtectKernelDefaults = false
   208  	c.RegisterWithTaints = opt.RegisterWithTaints
   209  	c.RegisterNode = true
   210  	c.LocalStorageCapacityIsolation = true
   211  	c.PodLogsDir = podLogsPath
   212  
   213  	return f, c
   214  }