github.com/SUSE/skuba@v1.4.17/ci/infra/testrunner/skuba/skuba.py (about) 1 import logging 2 import os 3 import toml 4 5 import platforms 6 from checks import Checker 7 from utils.format import Format 8 from utils.utils import (step, Utils) 9 10 logger = logging.getLogger('testrunner') 11 12 13 class Skuba: 14 15 def __init__(self, conf, platform): 16 self.conf = conf 17 self.binpath = self.conf.skuba.binpath 18 self.utils = Utils(self.conf) 19 self.platform = platforms.get_platform(conf, platform) 20 self.workdir = self.conf.skuba.workdir 21 self.cluster = self.conf.skuba.cluster 22 self.cluster_dir =os.path.join(self.workdir, self.cluster) 23 self.utils.setup_ssh() 24 self.checker = Checker(conf, platform) 25 26 def _verify_skuba_bin_dependency(self): 27 if not os.path.isfile(self.binpath): 28 raise FileNotFoundError("skuba not found at {}".format(self.binpath)) 29 30 def _verify_bootstrap_dependency(self): 31 if not os.path.exists(self.cluster_dir): 32 raise ValueError(f'cluster {self.cluster} not found at {self.workdir}.' 33 ' Please run bootstrap and try again') 34 35 @staticmethod 36 @step 37 def cleanup(conf): 38 """Cleanup skuba working environment""" 39 cluster_dir = os.path.join(conf.skuba.workdir, conf.skuba.cluster) 40 Utils.cleanup_file(cluster_dir) 41 42 @step 43 def cluster_deploy(self, kubernetes_version=None, cloud_provider=None, 44 timeout=None, registry_mirror=None): 45 """Deploy a cluster joining all nodes""" 46 self.cluster_bootstrap(kubernetes_version=kubernetes_version, 47 cloud_provider=cloud_provider, 48 timeout=timeout, 49 registry_mirror=registry_mirror) 50 self.join_nodes(timeout=timeout) 51 52 53 @step 54 def cluster_init(self, kubernetes_version=None, cloud_provider=None): 55 logger.debug("Cleaning up any previous cluster dir") 56 self.utils.cleanup_file(self.cluster_dir) 57 58 k8s_version_option, cloud_provider_option = "", "" 59 if kubernetes_version: 60 k8s_version_option = "--kubernetes-version {}".format(kubernetes_version) 61 if cloud_provider: 62 cloud_provider_option = "--cloud-provider {}".format(type(self.platform).__name__.lower()) 63 64 cmd = "cluster init --control-plane {} {} {} test-cluster".format(self.platform.get_lb_ipaddr(), 65 k8s_version_option, cloud_provider_option) 66 # Override work directory, because init must run in the parent directory of the 67 # cluster directory 68 self._run_skuba(cmd, cwd=self.workdir) 69 70 71 @step 72 def cluster_bootstrap(self, kubernetes_version=None, cloud_provider=None, 73 timeout=None, registry_mirror=None): 74 self.cluster_init(kubernetes_version, cloud_provider) 75 self.node_bootstrap(cloud_provider, timeout=timeout, registry_mirror=registry_mirror) 76 77 78 @step 79 def node_bootstrap(self, cloud_provider=None, timeout=None, registry_mirror=None): 80 self._verify_bootstrap_dependency() 81 82 if cloud_provider: 83 self.platform.setup_cloud_provider(self.workdir) 84 85 if registry_mirror: 86 self._setup_container_registries(registry_mirror) 87 88 master0_ip = self.platform.get_nodes_ipaddrs("master")[0] 89 master0_name = self.platform.get_nodes_names("master")[0] 90 cmd = (f'node bootstrap --user {self.utils.ssh_user()} --sudo --target ' 91 f'{master0_ip} {master0_name}') 92 self._run_skuba(cmd) 93 94 self.checker.check_node("master", 0, stage="joined", timeout=timeout) 95 96 def _setup_container_registries(self, registry_mirror): 97 mirrors = {} 98 for l in registry_mirror: 99 if l[0] not in mirrors: 100 mirrors[l[0]] = [] 101 mirrors[l[0]].append(l[1]) 102 103 conf = {'unqualified-search-registries': ['docker.io'], 'registry': []} 104 for location, mirror_list in mirrors.items(): 105 mirror_toml = [] 106 for m in mirror_list: 107 mirror_toml.append({'location': m, 'insecure': True}) 108 conf['registry'].append( 109 {'prefix': location, 'location': location, 110 'mirror': mirror_toml}) 111 conf_string = toml.dumps(conf) 112 c_dir = os.path.join(self.cluster_dir, 'addons/containers') 113 os.mkdir(c_dir) 114 with open(os.path.join(c_dir, 'registries.conf'), 'w') as f: 115 f.write(conf_string) 116 117 @step 118 def node_join(self, role="worker", nr=0, timeout=None): 119 self._verify_bootstrap_dependency() 120 121 ip_addrs = self.platform.get_nodes_ipaddrs(role) 122 node_names = self.platform.get_nodes_names(role) 123 124 if nr < 0: 125 raise ValueError("Node number cannot be negative") 126 127 if nr >= len(ip_addrs): 128 raise Exception(f'Node {role}-{nr} is not deployed in ' 129 'infrastructure') 130 131 cmd = (f'node join --role {role} --user {self.utils.ssh_user()} ' 132 f' --sudo --target {ip_addrs[nr]} {node_names[nr]}') 133 try: 134 self._run_skuba(cmd) 135 except Exception as ex: 136 raise Exception(f'Error joining node {role}-{nr}') from ex 137 138 self.checker.check_node(role, nr, stage="joined", timeout=timeout) 139 140 def join_nodes(self, masters=None, workers=None, timeout=None): 141 if masters is None: 142 masters = self.platform.get_num_nodes("master") 143 if workers is None: 144 workers = self.platform.get_num_nodes("worker") 145 146 for node in range(1, masters): 147 self.node_join("master", node, timeout=timeout) 148 149 for node in range(0, workers): 150 self.node_join("worker", node, timeout=timeout) 151 152 153 @step 154 def node_remove(self, role="worker", nr=0): 155 self._verify_bootstrap_dependency() 156 157 if role not in ("master", "worker"): 158 raise ValueError("Invalid role {}".format(role)) 159 160 n_nodes = self.num_of_nodes(role) 161 162 if nr < 0: 163 raise ValueError("Node number must be non negative") 164 165 if nr >= n_nodes: 166 raise ValueError("Error: there is no {role}-{nr} \ 167 node to remove from cluster".format(role=role, nr=nr)) 168 169 node_names = self.platform.get_nodes_names(role) 170 cmd = f'node remove {node_names[nr]}' 171 172 try: 173 self._run_skuba(cmd) 174 except Exception as ex: 175 raise Exception("Error executing cmd {}".format(cmd)) from ex 176 177 @step 178 def node_upgrade(self, action, role, nr, ignore_errors=False): 179 self._verify_bootstrap_dependency() 180 181 if role not in ("master", "worker"): 182 raise ValueError("Invalid role '{}'".format(role)) 183 184 if nr >= self.num_of_nodes(role): 185 raise ValueError("Error: there is no {}-{} \ 186 node in the cluster".format(role, nr)) 187 188 if action == "plan": 189 node_names = self.platform.get_nodes_names(role) 190 cmd = f'node upgrade plan {node_names[nr]}' 191 elif action == "apply": 192 ip_addrs = self.platform.get_nodes_ipaddrs(role) 193 cmd = (f'node upgrade apply --user {self.utils.ssh_user()} --sudo' 194 f' --target {ip_addrs[nr]}') 195 else: 196 raise ValueError("Invalid action '{}'".format(action)) 197 198 return self._run_skuba(cmd, ignore_errors=ignore_errors) 199 200 @step 201 def cluster_upgrade_plan(self): 202 self._verify_bootstrap_dependency() 203 return self._run_skuba("cluster upgrade plan") 204 205 @step 206 def cluster_status(self): 207 self._verify_bootstrap_dependency() 208 return self._run_skuba("cluster status") 209 210 @step 211 def addon_refresh(self, action): 212 self._verify_bootstrap_dependency() 213 if action not in ['localconfig']: 214 raise ValueError("Invalid action '{}'".format(action)) 215 return self._run_skuba("addon refresh {0}".format(action)) 216 217 @step 218 def addon_upgrade(self, action): 219 self._verify_bootstrap_dependency() 220 if action not in ['plan', 'apply']: 221 raise ValueError("Invalid action '{}'".format(action)) 222 return self._run_skuba("addon upgrade {0}".format(action)) 223 224 @step 225 def num_of_nodes(self, role): 226 227 if role not in ("master", "worker"): 228 raise ValueError("Invalid role '{}'".format(role)) 229 230 cmd = "cluster status" 231 output = self._run_skuba(cmd) 232 return output.count(role) 233 234 def _run_skuba(self, cmd, cwd=None, verbosity=None, ignore_errors=False): 235 """Running skuba command. 236 The cwd defautls to cluster_dir but can be overrided 237 for example, for the init command that must run in {workspace} 238 """ 239 self._verify_skuba_bin_dependency() 240 241 if cwd is None: 242 cwd = self.cluster_dir 243 244 if verbosity is None: 245 verbosity = self.conf.skuba.verbosity 246 try: 247 v = int(verbosity) 248 except ValueError: 249 raise ValueError(f"verbosity '{verbosity}' is not an int") 250 verbosity = v 251 252 return self.utils.runshellcommand( 253 f"{self.binpath} -v {verbosity} {cmd}", 254 cwd=cwd, 255 ignore_errors=ignore_errors, 256 )