github.com/caos/orbos@v1.5.14-0.20221103111702-e6cd0cea7ad4/internal/operator/orbiter/kinds/clusters/kubernetes/nodes.go (about)

     1  package kubernetes
     2  
     3  import (
     4  	"fmt"
     5  	"time"
     6  
     7  	v1 "k8s.io/api/core/v1"
     8  
     9  	"github.com/caos/orbos/mntr"
    10  	"github.com/caos/orbos/pkg/kubernetes"
    11  )
    12  
    13  func maintainNodes(allInitializedMachines initializedMachines, monitor mntr.Monitor, k8sClient *kubernetes.Client, pdf func(mntr.Monitor) error) (done bool, err error) {
    14  
    15  	// Delete kubernetes nodes for unexisting machines
    16  	if k8sClient != nil {
    17  		nodes, err := k8sClient.ListNodes()
    18  		if err != nil {
    19  			return false, err
    20  		}
    21  		for nodeIdx := range nodes {
    22  			node := nodes[nodeIdx]
    23  			nodeName := node.GetName()
    24  
    25  			leave := false
    26  			for idx := range allInitializedMachines {
    27  				machine := allInitializedMachines[idx]
    28  				if machine.infra.ID() == nodeName {
    29  					leave = true
    30  					break
    31  				}
    32  			}
    33  			if !leave {
    34  				for idx := range node.Status.Conditions {
    35  					condition := node.Status.Conditions[idx]
    36  					if condition.Type == v1.NodeReady && condition.Status == v1.ConditionTrue {
    37  						return false, fmt.Errorf("there is no infrastructure machine corresponding to Kubernetes node %s, yet the node is still ready", nodeName)
    38  					}
    39  				}
    40  
    41  				if err := k8sClient.DeleteNode(nodeName); err != nil {
    42  					return false, err
    43  				}
    44  				monitor.WithField("node", nodeName).Info("Node deleted")
    45  			}
    46  		}
    47  	}
    48  
    49  	allInitializedMachines.forEach(monitor, func(machine *initializedMachine, machineMonitor mntr.Monitor) bool {
    50  		if err = machine.reconcile(); err != nil {
    51  			return false
    52  		}
    53  		return true
    54  	})
    55  	if err != nil {
    56  		return false, err
    57  	}
    58  
    59  	allInitializedMachines.forEach(monitor, func(machine *initializedMachine, machineMonitor mntr.Monitor) bool {
    60  		req, _, unreq := machine.infra.RebootRequired()
    61  		if !req {
    62  			return true
    63  		}
    64  
    65  		if machine.node != nil {
    66  			if err = k8sClient.Drain(machine.currentMachine, machine.node, kubernetes.Rebooting, false); err != nil {
    67  				return false
    68  			}
    69  		}
    70  		machine.currentMachine.Rebooting = true
    71  		machineMonitor.Info("Requiring reboot")
    72  		unreq()
    73  		machine.desiredNodeagent.RebootRequired = time.Now().Truncate(time.Minute)
    74  		err = pdf(monitor.WithField("reason", "remove machine from reboot list"))
    75  		return false
    76  	})
    77  	if err != nil {
    78  		return false, err
    79  	}
    80  
    81  	done = true
    82  	allInitializedMachines.forEach(monitor, func(machine *initializedMachine, machineMonitor mntr.Monitor) bool {
    83  		if !machine.currentMachine.FirewallIsReady {
    84  			done = false
    85  			machineMonitor.Info("Node agent is not ready yet")
    86  		}
    87  		return true
    88  	})
    89  	return done, nil
    90  }