github.com/SUSE/skuba@v1.4.17/ci/infra/testrunner/tests/test_node_reboot.py (about)

     1  import json
     2  import logging
     3  import time
     4  
     5  import pytest
     6  
     7  from tests.utils import (check_pods_ready, node_is_ready, wait)
     8  
     9  from timeout_decorator import timeout
    10  
    11  logger = logging.getLogger("testrunner")
    12  
    13  
    14  def check_node_is_ready(platform, kubectl, role, nr):
    15      assert node_is_ready(platform, kubectl, role, nr)
    16  
    17  
    18  @pytest.mark.pr
    19  @pytest.mark.parametrize('role,node', [('master', 1), ('worker', 0)])
    20  def test_hard_reboot(deployment, platform, kubectl, role, node):
    21      """ Reboots master and worker nodes and checks they are back ready.
    22      For masters, reboot master 1, as master 0 is expected to be the
    23      cluster leader and rebooting it may introduce transient etcd errors.
    24      """
    25      assert kubectl.get_num_nodes_by_role(role) > node
    26  
    27      platform.ssh_run(role, node, 'sudo reboot &')
    28  
    29      # Allow time for kubernetes to check node readiness and detect it is not
    30      # ready
    31      time.sleep(60)
    32  
    33      # wait the node to become ready
    34      wait(check_node_is_ready,
    35           platform,
    36           kubectl,
    37           role,
    38           node,
    39           wait_delay=30,
    40           wait_timeout=10,
    41           wait_backoff=30,
    42           wait_elapsed=180,
    43           wait_allow=(AssertionError))
    44  
    45      # wait pods to become ready
    46      wait(check_pods_ready,
    47           kubectl,
    48           wait_timeout=10,
    49           wait_backoff=30,
    50           wait_elapsed=180,
    51           wait_allow=(AssertionError))