github.com/sealerio/sealer@v0.11.1-0.20240507115618-f4f89c5853ae/pkg/runtime/kubernetes/kubeadm/kubeadm_config.go (about)

     1  // Copyright © 2021 Alibaba Group Holding Ltd.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package kubeadm
    16  
    17  import (
    18  	"fmt"
    19  	"io"
    20  	"net"
    21  	"strings"
    22  
    23  	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
    24  
    25  	versionUtils "github.com/sealerio/sealer/utils/version"
    26  	"github.com/sirupsen/logrus"
    27  
    28  	"github.com/sealerio/sealer/utils"
    29  	strUtils "github.com/sealerio/sealer/utils/strings"
    30  
    31  	"github.com/imdario/mergo"
    32  	"k8s.io/kube-proxy/config/v1alpha1"
    33  	"k8s.io/kubelet/config/v1beta1"
    34  )
    35  
    36  // Read config from https://github.com/sealerio/sealer/blob/main/docs/design/clusterfile-v2.md and overwrite default kubeadm.yaml
    37  // Use github.com/imdario/mergo to merge kubeadm config in Clusterfile and the default kubeadm config
    38  // Using a config filter to handle some edge cases
    39  
    40  // https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/apis/kubeadm/v1beta3/types.go
    41  // Using map to overwrite Kubeadm configs
    42  
    43  // nolint
    44  type KubeadmConfig struct {
    45  	v1beta3.InitConfiguration
    46  	v1beta3.ClusterConfiguration
    47  	v1alpha1.KubeProxyConfiguration
    48  	v1beta1.KubeletConfiguration
    49  	v1beta3.JoinConfiguration
    50  }
    51  
    52  const (
    53  	EtcdServers = "etcd-servers"
    54  )
    55  
    56  const (
    57  	V1991 = "v1.19.1"
    58  	V1992 = "v1.19.2"
    59  	V1150 = "v1.15.0"
    60  	V1200 = "v1.20.0"
    61  	V1230 = "v1.23.0"
    62  
    63  	// kubeadm api version
    64  	KubeadmV1beta1 = "kubeadm.k8s.io/v1beta1"
    65  	KubeadmV1beta2 = "kubeadm.k8s.io/v1beta2"
    66  	KubeadmV1beta3 = "kubeadm.k8s.io/v1beta3"
    67  )
    68  
    69  // LoadFromClusterfile :Load KubeadmConfig from Clusterfile.
    70  // If it has `KubeadmConfig` in Clusterfile, load every field to each configuration.
    71  // If Kubeadm raw config in Clusterfile, just load it.
    72  func (k *KubeadmConfig) LoadFromClusterfile(kubeadmConfig KubeadmConfig) error {
    73  	k.APIServer.CertSANs = strUtils.RemoveDuplicate(append(k.APIServer.CertSANs, kubeadmConfig.APIServer.CertSANs...))
    74  
    75  	return mergo.Merge(k, kubeadmConfig)
    76  }
    77  
    78  // Merge Using github.com/imdario/mergo to merge KubeadmConfig to the ClusterImage default kubeadm config, overwrite some field.
    79  // if defaultKubeadmConfig file not exist, use default raw kubeadm config to merge k.KubeConfigSpec empty value
    80  func (k *KubeadmConfig) Merge(kubeadmYamlPath string, decode func(arg string, kind string) (interface{}, error)) error {
    81  	newConfig, err := LoadKubeadmConfigs(kubeadmYamlPath, decode)
    82  	if err != nil {
    83  		return fmt.Errorf("failed to found kubeadm config from %s: %v", kubeadmYamlPath, err)
    84  	}
    85  	k.APIServer.CertSANs = strUtils.RemoveDuplicate(append(k.APIServer.CertSANs, newConfig.APIServer.CertSANs...))
    86  
    87  	return mergo.Merge(k, newConfig)
    88  }
    89  
    90  func (k *KubeadmConfig) setAPIVersion(apiVersion string) {
    91  	k.InitConfiguration.APIVersion = apiVersion
    92  	k.ClusterConfiguration.APIVersion = apiVersion
    93  	k.JoinConfiguration.APIVersion = apiVersion
    94  }
    95  
    96  func (k *KubeadmConfig) setKubeadmAPIVersion() {
    97  	kv := versionUtils.Version(k.KubernetesVersion)
    98  	greaterThanKV1150, err := kv.GreaterThan(V1150)
    99  	if err != nil {
   100  		logrus.Errorf("compare kubernetes version failed: %s", err)
   101  	}
   102  	greaterThanKV1230, err := kv.GreaterThan(V1230)
   103  	if err != nil {
   104  		logrus.Errorf("compare kubernetes version failed: %s", err)
   105  	}
   106  	switch {
   107  	case greaterThanKV1150 && !greaterThanKV1230:
   108  		k.setAPIVersion(KubeadmV1beta2)
   109  	case greaterThanKV1230:
   110  		k.setAPIVersion(KubeadmV1beta3)
   111  	default:
   112  		// Compatible with versions 1.14 and 1.13. but do not recommend.
   113  		k.setAPIVersion(KubeadmV1beta1)
   114  	}
   115  }
   116  
   117  func (k *KubeadmConfig) GetCertSANS() []string {
   118  	return k.ClusterConfiguration.APIServer.CertSANs
   119  }
   120  
   121  func (k *KubeadmConfig) GetDNSDomain() string {
   122  	return k.ClusterConfiguration.Networking.DNSDomain
   123  }
   124  
   125  func (k *KubeadmConfig) GetSvcCIDR() string {
   126  	return k.ClusterConfiguration.Networking.ServiceSubnet
   127  }
   128  
   129  func LoadKubeadmConfigs(arg string, decode func(arg string, kind string) (interface{}, error)) (KubeadmConfig, error) {
   130  	kubeadmConfig := KubeadmConfig{}
   131  	initConfig, err := decode(arg, InitConfiguration)
   132  	if err != nil && err != io.EOF {
   133  		return kubeadmConfig, err
   134  	} else if initConfig != nil {
   135  		kubeadmConfig.InitConfiguration = *initConfig.(*v1beta3.InitConfiguration)
   136  	}
   137  	clusterConfig, err := decode(arg, ClusterConfiguration)
   138  	if err != nil && err != io.EOF {
   139  		return kubeadmConfig, err
   140  	} else if clusterConfig != nil {
   141  		kubeadmConfig.ClusterConfiguration = *clusterConfig.(*v1beta3.ClusterConfiguration)
   142  	}
   143  	kubeProxyConfig, err := decode(arg, KubeProxyConfiguration)
   144  	if err != nil && err != io.EOF {
   145  		return kubeadmConfig, err
   146  	} else if kubeProxyConfig != nil {
   147  		kubeadmConfig.KubeProxyConfiguration = *kubeProxyConfig.(*v1alpha1.KubeProxyConfiguration)
   148  	}
   149  	kubeletConfig, err := decode(arg, KubeletConfiguration)
   150  	if err != nil && err != io.EOF {
   151  		return kubeadmConfig, err
   152  	} else if kubeletConfig != nil {
   153  		kubeadmConfig.KubeletConfiguration = *kubeletConfig.(*v1beta1.KubeletConfiguration)
   154  	}
   155  	joinConfig, err := decode(arg, JoinConfiguration)
   156  	if err != nil && err != io.EOF {
   157  		return kubeadmConfig, err
   158  	} else if joinConfig != nil {
   159  		kubeadmConfig.JoinConfiguration = *joinConfig.(*v1beta3.JoinConfiguration)
   160  	}
   161  	return kubeadmConfig, nil
   162  }
   163  
   164  func getEtcdEndpointsWithHTTPSPrefix(masters []net.IP) string {
   165  	var tmpSlice []string
   166  	for _, ip := range masters {
   167  		tmpSlice = append(tmpSlice, fmt.Sprintf("https://%s", net.JoinHostPort(ip.String(), "2379")))
   168  	}
   169  
   170  	return strings.Join(tmpSlice, ",")
   171  }
   172  
   173  func NewKubeadmConfig(fromClusterFile KubeadmConfig, fromFile string, masters []net.IP, apiServerDomain,
   174  	cgroupDriver string, imageRepo string, apiServerVIP net.IP, extraSANs []string) (KubeadmConfig, error) {
   175  	conf := KubeadmConfig{}
   176  
   177  	if err := conf.LoadFromClusterfile(fromClusterFile); err != nil {
   178  		return conf, fmt.Errorf("failed to load kubeadm config from clusterfile: %v", err)
   179  	}
   180  	// TODO handle the kubeadm config, like kubeproxy config
   181  	//The configuration set here does not require merge
   182  
   183  	conf.InitConfiguration.LocalAPIEndpoint.AdvertiseAddress = masters[0].String()
   184  	conf.ControlPlaneEndpoint = net.JoinHostPort(apiServerDomain, "6443")
   185  
   186  	if conf.APIServer.ExtraArgs == nil {
   187  		conf.APIServer.ExtraArgs = make(map[string]string)
   188  	}
   189  	conf.APIServer.ExtraArgs[EtcdServers] = getEtcdEndpointsWithHTTPSPrefix(masters)
   190  	conf.IPVS.ExcludeCIDRs = append(conf.KubeProxyConfiguration.IPVS.ExcludeCIDRs, fmt.Sprintf("%s/32", apiServerVIP))
   191  	conf.KubeletConfiguration.CgroupDriver = cgroupDriver
   192  	conf.ClusterConfiguration.APIServer.CertSANs = []string{"127.0.0.1", apiServerDomain, apiServerVIP.String()}
   193  	conf.ClusterConfiguration.APIServer.CertSANs = append(conf.ClusterConfiguration.APIServer.CertSANs, extraSANs...)
   194  	for _, m := range masters {
   195  		conf.ClusterConfiguration.APIServer.CertSANs = append(conf.ClusterConfiguration.APIServer.CertSANs, m.String())
   196  	}
   197  
   198  	if err := conf.Merge(fromFile, utils.DecodeCRDFromFile); err != nil {
   199  		return conf, err
   200  	}
   201  
   202  	if err := conf.Merge(DefaultKubeadmConfig, utils.DecodeCRDFromString); err != nil {
   203  		return conf, err
   204  	}
   205  
   206  	conf.setKubeadmAPIVersion()
   207  
   208  	if conf.ClusterConfiguration.Networking.DNSDomain == "" {
   209  		conf.ClusterConfiguration.Networking.DNSDomain = "cluster.local"
   210  	}
   211  	if conf.JoinConfiguration.Discovery.BootstrapToken == nil {
   212  		conf.JoinConfiguration.Discovery.BootstrapToken = &v1beta3.BootstrapTokenDiscovery{}
   213  	}
   214  
   215  	// set cluster image repo,kubeadm will pull container image from this registry.
   216  	if conf.ClusterConfiguration.ImageRepository == "" {
   217  		conf.ClusterConfiguration.ImageRepository = imageRepo
   218  	}
   219  	if conf.ClusterConfiguration.DNS.ImageMeta.ImageRepository == "" {
   220  		conf.ClusterConfiguration.DNS.ImageMeta.ImageRepository = fmt.Sprintf("%s/%s", imageRepo, "coredns")
   221  	}
   222  
   223  	return conf, nil
   224  }