github.com/SUSE/skuba@v1.4.17/ci/infra/testrunner/tests/test_node_reboot.py (about) 1 import json 2 import logging 3 import time 4 5 import pytest 6 7 from tests.utils import (check_pods_ready, node_is_ready, wait) 8 9 from timeout_decorator import timeout 10 11 logger = logging.getLogger("testrunner") 12 13 14 def check_node_is_ready(platform, kubectl, role, nr): 15 assert node_is_ready(platform, kubectl, role, nr) 16 17 18 @pytest.mark.pr 19 @pytest.mark.parametrize('role,node', [('master', 1), ('worker', 0)]) 20 def test_hard_reboot(deployment, platform, kubectl, role, node): 21 """ Reboots master and worker nodes and checks they are back ready. 22 For masters, reboot master 1, as master 0 is expected to be the 23 cluster leader and rebooting it may introduce transient etcd errors. 24 """ 25 assert kubectl.get_num_nodes_by_role(role) > node 26 27 platform.ssh_run(role, node, 'sudo reboot &') 28 29 # Allow time for kubernetes to check node readiness and detect it is not 30 # ready 31 time.sleep(60) 32 33 # wait the node to become ready 34 wait(check_node_is_ready, 35 platform, 36 kubectl, 37 role, 38 node, 39 wait_delay=30, 40 wait_timeout=10, 41 wait_backoff=30, 42 wait_elapsed=180, 43 wait_allow=(AssertionError)) 44 45 # wait pods to become ready 46 wait(check_pods_ready, 47 kubectl, 48 wait_timeout=10, 49 wait_backoff=30, 50 wait_elapsed=180, 51 wait_allow=(AssertionError))