github.com/caos/orbos@v1.5.14-0.20221103111702-e6cd0cea7ad4/cmd/chore/e2e/run/bootstrap.go (about)

     1  package main
     2  
     3  import (
     4  	"bytes"
     5  	"context"
     6  	"fmt"
     7  	"time"
     8  
     9  	"github.com/caos/orbos/internal/helpers"
    10  	"github.com/caos/orbos/internal/operator/common"
    11  )
    12  
    13  var _ testFunc = bootstrap
    14  
    15  func bootstrap(_ *testSpecs, settings programSettings, conditions *conditions) interactFunc {
    16  
    17  	conditions.testCase = nil
    18  	conditions.orbiter = &condition{
    19  		watcher: watch(15*time.Minute, orbiter),
    20  		checks: func(checkCtx context.Context, newKubectl newKubectlCommandFunc, currentOrbiter currentOrbiter, nodeagents common.NodeAgentsCurrentKind) error {
    21  
    22  			cluster, err := currentOrbiter.cluster(settings)
    23  			if err != nil {
    24  				return err
    25  			}
    26  
    27  			masters := conditions.desiredMasters()
    28  			machines := masters + conditions.desiredWorkers()
    29  
    30  			currentMachinesLen := uint8(len(cluster.Machines.M))
    31  
    32  			if currentMachinesLen != machines {
    33  				err = helpers.Concat(err, fmt.Errorf("current state has %d machines instead of %d", currentMachinesLen, machines))
    34  			}
    35  
    36  			for nodeagentID, nodeagent := range cluster.Machines.M {
    37  				if !nodeagent.Ready ||
    38  					!nodeagent.FirewallIsReady ||
    39  					!nodeagent.Joined {
    40  					err = helpers.Concat(err, fmt.Errorf("nodeagent %s has current states joined=%t, firewallIsReady=%t, ready=%t",
    41  						nodeagentID,
    42  						nodeagent.Ready,
    43  						nodeagent.FirewallIsReady,
    44  						nodeagent.Joined,
    45  					))
    46  				}
    47  			}
    48  
    49  			for nodeagentID, nodeagent := range nodeagents.Current.NA {
    50  				if !nodeagent.NodeIsReady {
    51  					err = helpers.Concat(err, fmt.Errorf("nodeagent %s has not reported readiness yet", nodeagentID))
    52  				}
    53  				if nodeagent.Software.Kubelet.Version != conditions.kubernetes.Versions.Kubernetes ||
    54  					nodeagent.Software.Kubeadm.Version != conditions.kubernetes.Versions.Kubernetes ||
    55  					nodeagent.Software.Kubectl.Version != conditions.kubernetes.Versions.Kubernetes {
    56  					err = helpers.Concat(err, fmt.Errorf("nodeagent %s has current states kubelet=%s, kubeadm=%s, kubectl=%s instead of %s",
    57  						nodeagentID,
    58  						nodeagent.Software.Kubelet.Version,
    59  						nodeagent.Software.Kubeadm.Version,
    60  						nodeagent.Software.Kubectl.Version,
    61  						conditions.kubernetes.Versions.Kubernetes,
    62  					))
    63  				}
    64  			}
    65  
    66  			if cluster.Status != "running" {
    67  				err = helpers.Concat(err, fmt.Errorf("cluster status is %s", cluster.Status))
    68  			}
    69  
    70  			return helpers.Concat(err, helpers.Fanout([]func() error{
    71  				func() error {
    72  					return checkPodsAreReady(
    73  						checkCtx,
    74  						settings,
    75  						newKubectl,
    76  						"kube-system",
    77  						"component in (etcd, kube-apiserver, kube-controller-manager, kube-scheduler)",
    78  						masters*4,
    79  					)
    80  				},
    81  				func() error {
    82  					return checkPodsAreReady(
    83  						checkCtx,
    84  						settings,
    85  						newKubectl,
    86  						"kube-system",
    87  						"k8s-app=kube-proxy",
    88  						machines,
    89  					)
    90  				},
    91  				func() error {
    92  					return checkPodsAreReady(
    93  						checkCtx,
    94  						settings,
    95  						newKubectl,
    96  						"kube-system",
    97  						"k8s-app=kube-dns",
    98  						2,
    99  					)
   100  				},
   101  				func() error {
   102  					return checkPodsAreReady(
   103  						checkCtx,
   104  						settings,
   105  						newKubectl,
   106  						"kube-system",
   107  						"k8s-app=calico-kube-controllers",
   108  						1,
   109  					)
   110  				},
   111  				func() error {
   112  					return checkPodsAreReady(
   113  						checkCtx,
   114  						settings,
   115  						newKubectl,
   116  						"kube-system",
   117  						"k8s-app=calico-node",
   118  						machines,
   119  					)
   120  				},
   121  			})())
   122  		},
   123  	}
   124  
   125  	return func(ctx context.Context, step uint8, newOrbctl newOrbctlCommandFunc) error {
   126  
   127  		takeoffTimeout := 20 * time.Minute
   128  		takeoffCtx, takeoffCancel := context.WithTimeout(ctx, takeoffTimeout)
   129  		defer takeoffCancel()
   130  
   131  		ticker := time.NewTicker(time.Minute)
   132  		defer ticker.Stop()
   133  
   134  		started := time.Now()
   135  		go func() {
   136  			for {
   137  				select {
   138  				case <-ticker.C:
   139  					printProgress(orbctl, settings, fmt.Sprintf("%d (takeoff)", step), started, takeoffTimeout)
   140  				case <-takeoffCtx.Done():
   141  					return
   142  				}
   143  			}
   144  		}()
   145  
   146  		if err := runCommand(settings, orbctl.strPtr(), nil, nil, newOrbctl(takeoffCtx), "--gitops", "takeoff"); err != nil {
   147  			return err
   148  		}
   149  
   150  		buf := new(bytes.Buffer)
   151  		defer buf.Reset()
   152  
   153  		if err := runCommand(settings, nil, buf, nil, newOrbctl(takeoffCtx), "--gitops", "readsecret", fmt.Sprintf("orbiter.%s.kubeconfig.encrypted", settings.orbID)); err != nil {
   154  			return err
   155  		}
   156  
   157  		if err := runCommand(settings, orbctl.strPtr(), nil, nil, newOrbctl(takeoffCtx), "--gitops", "writesecret", fmt.Sprintf("orbiter.%s.kubeconfig.encrypted", settings.orbID), "--value", "dummy"); err != nil {
   158  			return err
   159  		}
   160  
   161  		writeSecretCmd := newOrbctl(takeoffCtx)
   162  		writeSecretCmd.Stdin = buf
   163  
   164  		return runCommand(settings, orbctl.strPtr(), nil, nil, writeSecretCmd, "--gitops", "writesecret", fmt.Sprintf("orbiter.%s.kubeconfig.encrypted", settings.orbID), "--stdin")
   165  	}
   166  }