github.com/sealerio/sealer@v0.11.1-0.20240507115618-f4f89c5853ae/pkg/runtime/kubernetes/join_masters.go (about)

     1  // Copyright © 2022 Alibaba Group Holding Ltd.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package kubernetes
    16  
    17  import (
    18  	"fmt"
    19  	"net"
    20  
    21  	"github.com/sirupsen/logrus"
    22  	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
    23  
    24  	"github.com/sealerio/sealer/pkg/runtime"
    25  	"github.com/sealerio/sealer/pkg/runtime/kubernetes/kubeadm"
    26  	"github.com/sealerio/sealer/utils/shellcommand"
    27  	"github.com/sealerio/sealer/utils/yaml"
    28  )
    29  
    30  func (k *Runtime) joinMasters(newMasters []net.IP, master0 net.IP, kubeadmConfig kubeadm.KubeadmConfig, token v1beta3.BootstrapTokenDiscovery, certKey string) error {
    31  	if len(newMasters) == 0 {
    32  		return nil
    33  	}
    34  
    35  	logrus.Infof("%s will be added as master", newMasters)
    36  
    37  	if err := k.initKube(newMasters); err != nil {
    38  		return err
    39  	}
    40  
    41  	if err := k.copyStaticFiles(newMasters); err != nil {
    42  		return err
    43  	}
    44  
    45  	if err := k.sendKubeConfigFilesToMaster(newMasters, AdminConf, ControllerConf, SchedulerConf); err != nil {
    46  		return err
    47  	}
    48  
    49  	if err := k.sendKubeadmFile(newMasters); err != nil {
    50  		return err
    51  	}
    52  
    53  	// TODO only needs send ca?
    54  	if err := k.sendClusterCert(newMasters); err != nil {
    55  		return err
    56  	}
    57  
    58  	//set master0 as APIServerEndpoint when join master
    59  	vs := net.JoinHostPort(master0.String(), "6443")
    60  	for _, m := range newMasters {
    61  		logrus.Infof("start to join %s as master", m)
    62  
    63  		joinCmd, err := k.Command(JoinMaster, k.getNodeNameOverride(m))
    64  		if err != nil {
    65  			return fmt.Errorf("failed to get join master command: %v", err)
    66  		}
    67  
    68  		hostname, err := k.infra.GetHostName(m)
    69  		if err != nil {
    70  			return err
    71  		}
    72  
    73  		if output, err := k.infra.CmdToString(m, nil, GetCustomizeCRISocket, ""); err == nil && output != "" {
    74  			kubeadmConfig.JoinConfiguration.NodeRegistration.CRISocket = output
    75  		}
    76  
    77  		kubeadmConfig.JoinConfiguration.Discovery.BootstrapToken = &token
    78  		kubeadmConfig.JoinConfiguration.Discovery.BootstrapToken.APIServerEndpoint = vs
    79  		kubeadmConfig.JoinConfiguration.ControlPlane.LocalAPIEndpoint.AdvertiseAddress = m.String()
    80  		kubeadmConfig.JoinConfiguration.ControlPlane.LocalAPIEndpoint.BindPort = int32(6443)
    81  		kubeadmConfig.JoinConfiguration.ControlPlane.CertificateKey = certKey
    82  		str, err := yaml.MarshalWithDelimiter(kubeadmConfig.JoinConfiguration, kubeadmConfig.KubeletConfiguration)
    83  		if err != nil {
    84  			return err
    85  		}
    86  		cmd := fmt.Sprintf("mkdir -p /etc/kubernetes && echo \"%s\" > %s", str, KubeadmFileYml)
    87  		if err = k.infra.CmdAsync(m, nil, cmd); err != nil {
    88  			return fmt.Errorf("failed to set join kubeadm config on host(%s) with cmd(%s): %v", m, cmd, err)
    89  		}
    90  
    91  		if err = k.infra.CmdAsync(m, nil, shellcommand.CommandSetHostAlias(k.getAPIServerDomain(), master0.String())); err != nil {
    92  			return fmt.Errorf("failed to config cluster hosts file cmd: %v", err)
    93  		}
    94  
    95  		certCMD := runtime.RemoteCertCmd(kubeadmConfig.GetCertSANS(), m, hostname, kubeadmConfig.GetSvcCIDR(), "")
    96  		if err = k.infra.CmdAsync(m, nil, certCMD); err != nil {
    97  			return fmt.Errorf("failed to exec command(%s) on master(%s): %v", certCMD, m, err)
    98  		}
    99  
   100  		if err = k.infra.CmdAsync(m, nil, joinCmd); err != nil {
   101  			return fmt.Errorf("failed to exec command(%s) on master(%s): %v", joinCmd, m, err)
   102  		}
   103  
   104  		if err = k.infra.CmdAsync(m, nil, shellcommand.CommandSetHostAlias(k.getAPIServerDomain(), m.String())); err != nil {
   105  			return fmt.Errorf("failed to config cluster hosts file cmd: %v", err)
   106  		}
   107  
   108  		if err = k.infra.CmdAsync(m, nil, "rm -rf .kube/config && mkdir -p /root/.kube && cp /etc/kubernetes/admin.conf /root/.kube/config"); err != nil {
   109  			return err
   110  		}
   111  
   112  		// At beginning, we set APIServerDomain direct to master0 and then kubeadm start scheduler and kcm, then we reset
   113  		// the APIServerDomain to the master itself, but scheduler and kcm already load the domain info and will not reload.
   114  		// So, we need restart them after reset the APIServerDomain.
   115  		if err = k.infra.CmdAsync(m, nil, "mv /etc/kubernetes/manifests/kube-scheduler.yaml /tmp/ && mv /tmp/kube-scheduler.yaml /etc/kubernetes/manifests/"); err != nil {
   116  			return err
   117  		}
   118  		if err = k.infra.CmdAsync(m, nil, "mv /etc/kubernetes/manifests/kube-controller-manager.yaml /tmp/ && mv /tmp/kube-controller-manager.yaml /etc/kubernetes/manifests/"); err != nil {
   119  			return err
   120  		}
   121  
   122  		logrus.Infof("succeeded in joining %s as master", m)
   123  	}
   124  	return nil
   125  }