github.com/SUSE/skuba@v1.4.17/ci/infra/testrunner/tests/utils.py (about)

     1  import signal
     2  import time
     3  import yaml
     4  
     5  PREVIOUS_VERSION = "1.16.2"
     6  CURRENT_VERSION = "1.17.13"
     7  
     8  
     9  def check_nodes_ready(kubectl):
    10      # Retrieve the node name and the Ready condition (True, or False)
    11      cmd = ("get nodes -o jsonpath='{ range .items[*]}{@.metadata.name}{\":\"}"
    12             "{range @.status.conditions[?(.type==\"Ready\")]}{@.status}{\" \"}'")
    13  
    14      nodes = kubectl.run_kubectl(cmd).strip().split(" ")
    15      for node in nodes:
    16          node_name, node_status = node.split(":")
    17          assert node_status == "True", f'Node {node_name} is not Ready'
    18  
    19  
    20  def node_is_ready(platform, kubectl, role, nr):
    21      node_name = platform.get_nodes_names(role)[nr]
    22      cmd = ("get nodes {} -o jsonpath='{{range @.status.conditions[*]}}"
    23             "{{@.type}}={{@.status}};{{end}}'").format(node_name)
    24  
    25      return kubectl.run_kubectl(cmd).find("Ready=True") != -1
    26  
    27  
    28  def node_is_upgraded(kubectl, platform, role, nr):
    29      node_name = platform.get_nodes_names(role)[nr]
    30      for attempt in range(20):
    31          if platform.all_apiservers_responsive():
    32              # kubernetes might be a little bit slow with updating the NodeVersionInfo
    33              cmd = ("get nodes {} -o jsonpath="
    34                     "'{{.status.nodeInfo.kubeletVersion}}'").format(node_name)
    35              version = kubectl.run_kubectl(cmd)
    36              if version.find(PREVIOUS_VERSION) == 0:
    37                  time.sleep(2)
    38              else:
    39                  break
    40          else:
    41              time.sleep(2)
    42  
    43      # allow system pods to come up again after the upgrade
    44      wait(check_pods_ready,
    45           kubectl,
    46           namespace="kube-system",
    47           wait_delay=60,
    48           wait_backoff=30,
    49           wait_elapsed=60*10,
    50           wait_allow=(AssertionError))
    51  
    52      cmd = "get nodes {} -o jsonpath='{{.status.nodeInfo.kubeletVersion}}'".format(node_name)
    53      return kubectl.run_kubectl(cmd).find(CURRENT_VERSION) != -1
    54  
    55  
    56  def check_pods_ready(kubectl, namespace=None, node=None, pods=[], statuses=['Running', 'Succeeded']):
    57  
    58      ns = f'{"--namespace="+namespace if namespace else ""}'
    59      node_selector = f'{"--field-selector spec.nodeName="+node if node else ""}'
    60      cmd = (f'get pods {" ".join(pods)} {ns} {node_selector} '
    61             f'-o jsonpath="{{ range .items[*]}}{{@.metadata.name}}:'
    62             f'{{@.status.phase}};"')
    63  
    64      result = kubectl.run_kubectl(cmd)
    65      pod_list = result.split(";")
    66      for name,status in [pod.split(":") for pod in pod_list if pod != ""]:
    67          assert status in statuses, (f'Pod {name} status {status}'
    68                                      f'not in expected statuses: {", ".join(statuses)}')
    69  
    70  def wait(func, *args, **kwargs):
    71  
    72      class TimeoutError(Exception):
    73          pass
    74  
    75      timeout = kwargs.pop("wait_timeout", 0)
    76      delay   = kwargs.pop("wait_delay", 0)
    77      backoff = kwargs.pop("wait_backoff", 0)
    78      retries = kwargs.pop("wait_retries", 0)
    79      allow   = kwargs.pop("wait_allow", ())
    80      elapsed = kwargs.pop("wait_elapsed", 0)
    81  
    82      if retries > 0 and elapsed > 0:
    83          raise ValueError("wait_retries and wait_elapsed cannot both have a non zero value")
    84  
    85      if retries == 0 and elapsed == 0:
    86          raise ValueError("either wait_retries  or wait_elapsed must have a non zero value")
    87  
    88      def _handle_timeout(signum, frame):
    89          raise TimeoutError()
    90  
    91      start = int(time.time())
    92      attempts = 1
    93      reason=""
    94  
    95      time.sleep(delay)
    96      while True:
    97          signal.signal(signal.SIGALRM, _handle_timeout)
    98          signal.alarm(timeout)
    99          try:
   100              return func(*args, **kwargs)
   101          except TimeoutError:
   102              reason = "timeout of {}s exceded".format(timeout)
   103          except allow as ex:
   104              reason = "{}: '{}'".format(ex.__class__.__name__, ex)
   105          finally:
   106              signal.alarm(0)
   107  
   108          if elapsed > 0 and int(time.time())-start >= elapsed:
   109              reason = "maximum wait time exceeded: {}s".format(elapsed)
   110              break
   111  
   112          if retries > 0 and attempts == retries:
   113              break
   114  
   115          time.sleep(backoff)
   116  
   117          attempts = attempts + 1
   118  
   119      raise Exception("Failed waiting for function {} after {} attemps due to {}".format(func.__name__, attempts, reason))
   120  
   121  
   122  def create_skuba_config(kubectl, configmap_data, dry_run=False):
   123      return kubectl.run_kubectl(
   124          'create configmap skuba-config --from-literal {0} -o yaml --namespace kube-system {1}'.format(
   125              configmap_data, '--dry-run' if dry_run else ''
   126          )
   127      )
   128  
   129  
   130  def replace_skuba_config(kubectl, configmap_data):
   131      new_configmap = create_skuba_config(kubectl, configmap_data, dry_run=True)
   132      return kubectl.run_kubectl("replace -f -", stdin=new_configmap.encode())
   133  
   134  
   135  def get_skuba_configuration_dict(kubectl):
   136      skubaConf_yml = kubectl.run_kubectl(
   137          "get configmap skuba-config --namespace kube-system -o jsonpath='{.data.SkubaConfiguration}'"
   138      )
   139      return yaml.load(skubaConf_yml, Loader=yaml.FullLoader)