k8s.io/kubernetes@v1.29.3/pkg/kubemark/hollow_kubelet.go (about)

     1  /*
     2  Copyright 2015 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package kubemark
    18  
    19  import (
    20  	"fmt"
    21  	"time"
    22  
    23  	"go.opentelemetry.io/otel/trace"
    24  	v1 "k8s.io/api/core/v1"
    25  	"k8s.io/klog/v2"
    26  	"k8s.io/mount-utils"
    27  
    28  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    29  	clientset "k8s.io/client-go/kubernetes"
    30  	"k8s.io/client-go/tools/record"
    31  	internalapi "k8s.io/cri-api/pkg/apis"
    32  	kubeletapp "k8s.io/kubernetes/cmd/kubelet/app"
    33  	"k8s.io/kubernetes/cmd/kubelet/app/options"
    34  	"k8s.io/kubernetes/pkg/kubelet"
    35  	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
    36  	"k8s.io/kubernetes/pkg/kubelet/cadvisor"
    37  	"k8s.io/kubernetes/pkg/kubelet/cm"
    38  	containertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
    39  	probetest "k8s.io/kubernetes/pkg/kubelet/prober/testing"
    40  	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
    41  	kubeletutil "k8s.io/kubernetes/pkg/kubelet/util"
    42  	"k8s.io/kubernetes/pkg/util/oom"
    43  	"k8s.io/kubernetes/pkg/volume"
    44  	"k8s.io/kubernetes/pkg/volume/cephfs"
    45  	"k8s.io/kubernetes/pkg/volume/configmap"
    46  	"k8s.io/kubernetes/pkg/volume/csi"
    47  	"k8s.io/kubernetes/pkg/volume/downwardapi"
    48  	"k8s.io/kubernetes/pkg/volume/emptydir"
    49  	"k8s.io/kubernetes/pkg/volume/fc"
    50  	"k8s.io/kubernetes/pkg/volume/git_repo"
    51  	"k8s.io/kubernetes/pkg/volume/hostpath"
    52  	"k8s.io/kubernetes/pkg/volume/iscsi"
    53  	"k8s.io/kubernetes/pkg/volume/local"
    54  	"k8s.io/kubernetes/pkg/volume/nfs"
    55  	"k8s.io/kubernetes/pkg/volume/portworx"
    56  	"k8s.io/kubernetes/pkg/volume/projected"
    57  	"k8s.io/kubernetes/pkg/volume/rbd"
    58  	"k8s.io/kubernetes/pkg/volume/secret"
    59  	"k8s.io/kubernetes/pkg/volume/util/hostutil"
    60  	"k8s.io/kubernetes/pkg/volume/util/subpath"
    61  	"k8s.io/kubernetes/test/utils"
    62  )
    63  
    64  type HollowKubelet struct {
    65  	KubeletFlags         *options.KubeletFlags
    66  	KubeletConfiguration *kubeletconfig.KubeletConfiguration
    67  	KubeletDeps          *kubelet.Dependencies
    68  }
    69  
    70  func volumePlugins() []volume.VolumePlugin {
    71  	allPlugins := []volume.VolumePlugin{}
    72  	allPlugins = append(allPlugins, emptydir.ProbeVolumePlugins()...)
    73  	allPlugins = append(allPlugins, git_repo.ProbeVolumePlugins()...)
    74  	allPlugins = append(allPlugins, hostpath.FakeProbeVolumePlugins(volume.VolumeConfig{})...)
    75  	allPlugins = append(allPlugins, nfs.ProbeVolumePlugins(volume.VolumeConfig{})...)
    76  	allPlugins = append(allPlugins, secret.ProbeVolumePlugins()...)
    77  	allPlugins = append(allPlugins, iscsi.ProbeVolumePlugins()...)
    78  	allPlugins = append(allPlugins, rbd.ProbeVolumePlugins()...)
    79  	allPlugins = append(allPlugins, cephfs.ProbeVolumePlugins()...)
    80  	allPlugins = append(allPlugins, downwardapi.ProbeVolumePlugins()...)
    81  	allPlugins = append(allPlugins, fc.ProbeVolumePlugins()...)
    82  	allPlugins = append(allPlugins, configmap.ProbeVolumePlugins()...)
    83  	allPlugins = append(allPlugins, projected.ProbeVolumePlugins()...)
    84  	allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...)
    85  	allPlugins = append(allPlugins, local.ProbeVolumePlugins()...)
    86  	allPlugins = append(allPlugins, csi.ProbeVolumePlugins()...)
    87  	return allPlugins
    88  }
    89  
    90  func NewHollowKubelet(
    91  	flags *options.KubeletFlags,
    92  	config *kubeletconfig.KubeletConfiguration,
    93  	client *clientset.Clientset,
    94  	heartbeatClient *clientset.Clientset,
    95  	cadvisorInterface cadvisor.Interface,
    96  	imageService internalapi.ImageManagerService,
    97  	runtimeService internalapi.RuntimeService,
    98  	containerManager cm.ContainerManager) *HollowKubelet {
    99  	d := &kubelet.Dependencies{
   100  		KubeClient:                client,
   101  		HeartbeatClient:           heartbeatClient,
   102  		ProbeManager:              probetest.FakeManager{},
   103  		RemoteRuntimeService:      runtimeService,
   104  		RemoteImageService:        imageService,
   105  		CAdvisorInterface:         cadvisorInterface,
   106  		Cloud:                     nil,
   107  		OSInterface:               &containertest.FakeOS{},
   108  		ContainerManager:          containerManager,
   109  		VolumePlugins:             volumePlugins(),
   110  		TLSOptions:                nil,
   111  		OOMAdjuster:               oom.NewFakeOOMAdjuster(),
   112  		Mounter:                   &mount.FakeMounter{},
   113  		Subpather:                 &subpath.FakeSubpath{},
   114  		HostUtil:                  hostutil.NewFakeHostUtil(nil),
   115  		PodStartupLatencyTracker:  kubeletutil.NewPodStartupLatencyTracker(),
   116  		NodeStartupLatencyTracker: kubeletutil.NewNodeStartupLatencyTracker(),
   117  		TracerProvider:            trace.NewNoopTracerProvider(),
   118  		Recorder:                  &record.FakeRecorder{}, // With real recorder we attempt to read /dev/kmsg.
   119  	}
   120  
   121  	return &HollowKubelet{
   122  		KubeletFlags:         flags,
   123  		KubeletConfiguration: config,
   124  		KubeletDeps:          d,
   125  	}
   126  }
   127  
   128  // Starts this HollowKubelet and blocks.
   129  func (hk *HollowKubelet) Run() {
   130  	if err := kubeletapp.RunKubelet(&options.KubeletServer{
   131  		KubeletFlags:         *hk.KubeletFlags,
   132  		KubeletConfiguration: *hk.KubeletConfiguration,
   133  	}, hk.KubeletDeps, false); err != nil {
   134  		klog.Fatalf("Failed to run HollowKubelet: %v. Exiting.", err)
   135  	}
   136  	select {}
   137  }
   138  
   139  // HollowKubeletOptions contains settable parameters for hollow kubelet.
   140  type HollowKubeletOptions struct {
   141  	NodeName            string
   142  	KubeletPort         int
   143  	KubeletReadOnlyPort int
   144  	MaxPods             int
   145  	PodsPerCore         int
   146  	NodeLabels          map[string]string
   147  	RegisterWithTaints  []v1.Taint
   148  }
   149  
   150  // Builds a KubeletConfiguration for the HollowKubelet, ensuring that the
   151  // usual defaults are applied for fields we do not override.
   152  func GetHollowKubeletConfig(opt *HollowKubeletOptions) (*options.KubeletFlags, *kubeletconfig.KubeletConfiguration) {
   153  	testRootDir := utils.MakeTempDirOrDie("hollow-kubelet.", "")
   154  	podFilePath := utils.MakeTempDirOrDie("static-pods", testRootDir)
   155  	klog.Infof("Using %s as root dir for hollow-kubelet", testRootDir)
   156  
   157  	// Flags struct
   158  	f := options.NewKubeletFlags()
   159  	f.RootDirectory = testRootDir
   160  	f.HostnameOverride = opt.NodeName
   161  	f.MinimumGCAge = metav1.Duration{Duration: 1 * time.Minute}
   162  	f.MaxContainerCount = 100
   163  	f.MaxPerPodContainerCount = 2
   164  	f.NodeLabels = opt.NodeLabels
   165  	f.RegisterSchedulable = true
   166  
   167  	// Config struct
   168  	c, err := options.NewKubeletConfiguration()
   169  	if err != nil {
   170  		panic(err)
   171  	}
   172  
   173  	c.ImageServiceEndpoint = "unix:///run/containerd/containerd.sock"
   174  	c.StaticPodURL = ""
   175  	c.EnableServer = true
   176  	c.Address = "0.0.0.0" /* bind address */
   177  	c.Port = int32(opt.KubeletPort)
   178  	c.ReadOnlyPort = int32(opt.KubeletReadOnlyPort)
   179  	c.StaticPodPath = podFilePath
   180  	c.FileCheckFrequency.Duration = 20 * time.Second
   181  	c.HTTPCheckFrequency.Duration = 20 * time.Second
   182  	c.NodeStatusUpdateFrequency.Duration = 10 * time.Second
   183  	c.NodeStatusReportFrequency.Duration = 5 * time.Minute
   184  	c.SyncFrequency.Duration = 10 * time.Second
   185  	c.EvictionPressureTransitionPeriod.Duration = 5 * time.Minute
   186  	c.MaxPods = int32(opt.MaxPods)
   187  	c.PodsPerCore = int32(opt.PodsPerCore)
   188  	c.ClusterDNS = []string{}
   189  	c.ImageGCHighThresholdPercent = 90
   190  	c.ImageGCLowThresholdPercent = 80
   191  	c.ProviderID = fmt.Sprintf("kubemark://%v", opt.NodeName)
   192  	c.VolumeStatsAggPeriod.Duration = time.Minute
   193  	c.CgroupRoot = ""
   194  	c.CPUCFSQuota = true
   195  	c.EnableControllerAttachDetach = false
   196  	c.EnableDebuggingHandlers = true
   197  	c.CgroupsPerQOS = false
   198  	// hairpin-veth is used to allow hairpin packets. Note that this deviates from
   199  	// what the "real" kubelet currently does, because there's no way to
   200  	// set promiscuous mode on docker0.
   201  	c.HairpinMode = kubeletconfig.HairpinVeth
   202  	c.MaxOpenFiles = 1024
   203  	c.RegistryBurst = 10
   204  	c.RegistryPullQPS = 5.0
   205  	c.ResolverConfig = kubetypes.ResolvConfDefault
   206  	c.KubeletCgroups = "/kubelet"
   207  	c.SerializeImagePulls = true
   208  	c.SystemCgroups = ""
   209  	c.ProtectKernelDefaults = false
   210  	c.RegisterWithTaints = opt.RegisterWithTaints
   211  	c.RegisterNode = true
   212  	c.LocalStorageCapacityIsolation = true
   213  
   214  	return f, c
   215  }