github.com/SUSE/skuba@v1.4.17/ci/infra/testrunner/tests/utils.py (about) 1 import signal 2 import time 3 import yaml 4 5 PREVIOUS_VERSION = "1.16.2" 6 CURRENT_VERSION = "1.17.13" 7 8 9 def check_nodes_ready(kubectl): 10 # Retrieve the node name and the Ready condition (True, or False) 11 cmd = ("get nodes -o jsonpath='{ range .items[*]}{@.metadata.name}{\":\"}" 12 "{range @.status.conditions[?(.type==\"Ready\")]}{@.status}{\" \"}'") 13 14 nodes = kubectl.run_kubectl(cmd).strip().split(" ") 15 for node in nodes: 16 node_name, node_status = node.split(":") 17 assert node_status == "True", f'Node {node_name} is not Ready' 18 19 20 def node_is_ready(platform, kubectl, role, nr): 21 node_name = platform.get_nodes_names(role)[nr] 22 cmd = ("get nodes {} -o jsonpath='{{range @.status.conditions[*]}}" 23 "{{@.type}}={{@.status}};{{end}}'").format(node_name) 24 25 return kubectl.run_kubectl(cmd).find("Ready=True") != -1 26 27 28 def node_is_upgraded(kubectl, platform, role, nr): 29 node_name = platform.get_nodes_names(role)[nr] 30 for attempt in range(20): 31 if platform.all_apiservers_responsive(): 32 # kubernetes might be a little bit slow with updating the NodeVersionInfo 33 cmd = ("get nodes {} -o jsonpath=" 34 "'{{.status.nodeInfo.kubeletVersion}}'").format(node_name) 35 version = kubectl.run_kubectl(cmd) 36 if version.find(PREVIOUS_VERSION) == 0: 37 time.sleep(2) 38 else: 39 break 40 else: 41 time.sleep(2) 42 43 # allow system pods to come up again after the upgrade 44 wait(check_pods_ready, 45 kubectl, 46 namespace="kube-system", 47 wait_delay=60, 48 wait_backoff=30, 49 wait_elapsed=60*10, 50 wait_allow=(AssertionError)) 51 52 cmd = "get nodes {} -o jsonpath='{{.status.nodeInfo.kubeletVersion}}'".format(node_name) 53 return kubectl.run_kubectl(cmd).find(CURRENT_VERSION) != -1 54 55 56 def check_pods_ready(kubectl, namespace=None, node=None, pods=[], statuses=['Running', 'Succeeded']): 57 58 ns = f'{"--namespace="+namespace if namespace else ""}' 59 node_selector = f'{"--field-selector spec.nodeName="+node if node else ""}' 60 cmd = (f'get pods {" ".join(pods)} {ns} {node_selector} ' 61 f'-o jsonpath="{{ range .items[*]}}{{@.metadata.name}}:' 62 f'{{@.status.phase}};"') 63 64 result = kubectl.run_kubectl(cmd) 65 pod_list = result.split(";") 66 for name,status in [pod.split(":") for pod in pod_list if pod != ""]: 67 assert status in statuses, (f'Pod {name} status {status}' 68 f'not in expected statuses: {", ".join(statuses)}') 69 70 def wait(func, *args, **kwargs): 71 72 class TimeoutError(Exception): 73 pass 74 75 timeout = kwargs.pop("wait_timeout", 0) 76 delay = kwargs.pop("wait_delay", 0) 77 backoff = kwargs.pop("wait_backoff", 0) 78 retries = kwargs.pop("wait_retries", 0) 79 allow = kwargs.pop("wait_allow", ()) 80 elapsed = kwargs.pop("wait_elapsed", 0) 81 82 if retries > 0 and elapsed > 0: 83 raise ValueError("wait_retries and wait_elapsed cannot both have a non zero value") 84 85 if retries == 0 and elapsed == 0: 86 raise ValueError("either wait_retries or wait_elapsed must have a non zero value") 87 88 def _handle_timeout(signum, frame): 89 raise TimeoutError() 90 91 start = int(time.time()) 92 attempts = 1 93 reason="" 94 95 time.sleep(delay) 96 while True: 97 signal.signal(signal.SIGALRM, _handle_timeout) 98 signal.alarm(timeout) 99 try: 100 return func(*args, **kwargs) 101 except TimeoutError: 102 reason = "timeout of {}s exceded".format(timeout) 103 except allow as ex: 104 reason = "{}: '{}'".format(ex.__class__.__name__, ex) 105 finally: 106 signal.alarm(0) 107 108 if elapsed > 0 and int(time.time())-start >= elapsed: 109 reason = "maximum wait time exceeded: {}s".format(elapsed) 110 break 111 112 if retries > 0 and attempts == retries: 113 break 114 115 time.sleep(backoff) 116 117 attempts = attempts + 1 118 119 raise Exception("Failed waiting for function {} after {} attemps due to {}".format(func.__name__, attempts, reason)) 120 121 122 def create_skuba_config(kubectl, configmap_data, dry_run=False): 123 return kubectl.run_kubectl( 124 'create configmap skuba-config --from-literal {0} -o yaml --namespace kube-system {1}'.format( 125 configmap_data, '--dry-run' if dry_run else '' 126 ) 127 ) 128 129 130 def replace_skuba_config(kubectl, configmap_data): 131 new_configmap = create_skuba_config(kubectl, configmap_data, dry_run=True) 132 return kubectl.run_kubectl("replace -f -", stdin=new_configmap.encode()) 133 134 135 def get_skuba_configuration_dict(kubectl): 136 skubaConf_yml = kubectl.run_kubectl( 137 "get configmap skuba-config --namespace kube-system -o jsonpath='{.data.SkubaConfiguration}'" 138 ) 139 return yaml.load(skubaConf_yml, Loader=yaml.FullLoader)