github.com/SUSE/skuba@v1.4.17/ci/infra/testrunner/testrunner.py (about) 1 #!/usr/bin/env python3 -Wd -b 2 3 """ 4 Runs end-to-end product tests for v4+. 5 This script can be run from Jenkins or manually, on developer desktops or servers. 6 """ 7 8 import io 9 import logging 10 import sys 11 from argparse import REMAINDER, ArgumentParser 12 13 import platforms 14 from skuba import Skuba 15 from kubectl import Kubectl 16 from tests import TestDriver 17 from utils import BaseConfig, Logger, Utils 18 from checks import Checker 19 20 __version__ = "0.0.3" 21 22 logger = logging.getLogger("testrunner") 23 24 25 def info(options): 26 print(Utils(options.conf).info()) 27 28 29 def config(options): 30 BaseConfig.print(options.conf) 31 32 def cleanup(options): 33 platforms.get_platform(options.conf, options.platform).cleanup() 34 Skuba.cleanup(options.conf) 35 36 37 def provision(options): 38 platforms.get_platform(options.conf, options.platform).provision( 39 num_master=options.master_count, 40 num_worker=options.worker_count) 41 42 43 def deploy(options): 44 skuba = Skuba(options.conf, options.platform) 45 skuba.cluster_deploy( 46 kubernetes_version=options.kubernetes_version, 47 cloud_provider=options.cloud_provider, 48 timeout=options.timeout, 49 registry_mirror=options.registry_mirror) 50 51 52 def bootstrap(options): 53 skuba = Skuba(options.conf, options.platform) 54 skuba.cluster_bootstrap( 55 kubernetes_version=options.kubernetes_version, 56 cloud_provider=options.cloud_provider, 57 timeout=options.timeout, 58 registry_mirror=options.registry_mirror) 59 60 61 def cluster_status(options): 62 print(Skuba(options.conf, options.platform).cluster_status()) 63 64 65 def cluster_upgrade_plan(options): 66 Skuba(options.conf, options.platform).cluster_upgrade_plan() 67 68 69 def get_logs(options): 70 platform_logging_errors = platforms.get_platform( 71 options.conf, options.platform).gather_logs() 72 73 if platform_logging_errors: 74 raise Exception("Failure(s) while collecting logs") 75 76 77 def join_node(options): 78 Skuba(options.conf, options.platform).node_join( 79 role=options.role, nr=options.node, timeout=options.timeout) 80 81 82 def join_nodes(options): 83 skuba = Skuba(options.conf, options.platform) 84 skuba.join_nodes( 85 masters=options.masters, 86 workers=options.workers, 87 timeout=options.timeout 88 ) 89 90 def remove_node(options): 91 Skuba(options.conf, options.platform).node_remove( 92 role=options.role, nr=options.node) 93 94 def node_upgrade(options): 95 Skuba(options.conf, options.platform).node_upgrade( 96 action=options.upgrade_action, role=options.role, nr=options.node) 97 98 def node_check(options): 99 Checker(options.conf, options.platform).check_node( 100 role=options.role, node=options.node, 101 checks=options.checks, stage=options.stage) 102 103 def cluster_check(options): 104 Checker(options.conf, options.platform).check_cluster( 105 checks=options.checks, stage=options.stage) 106 107 def test(options): 108 test_driver = TestDriver(options.conf, options.platform) 109 test_driver.run(module=options.module, test_suite=options.test_suite, test=options.test, 110 verbose=options.verbose, collect=options.collect, skip_setup=options.skip_setup, 111 mark=options.mark, traceback=options.traceback, junit=options.junit) 112 113 114 def ssh(options): 115 platforms.get_platform(options.conf, options.platform).ssh_run( 116 role=options.role, nr=options.node, cmd=" ".join(options.cmd)) 117 118 119 def inhibit_kured(options): 120 Kubectl(options.conf).inhibit_kured() 121 122 123 def main(): 124 help_str = """ 125 This script is meant to be run manually on test servers, developer desktops, or Jenkins. 126 This script supposed to run on python virtualenv from testrunner. Requires root privileges. 127 Warning: it removes docker containers, VMs, images, and network configuration. 128 """ 129 parser = ArgumentParser(help_str) 130 131 # Common parameters 132 parser.add_argument("-v", "--vars", dest="yaml_path", default="vars.yaml", 133 help='path for platform yaml file. Default is vars.yaml. eg: -v myconfig.yaml') 134 parser.add_argument("-p", "--platform", 135 default="openstack", 136 choices=["openstack", "vmware", 137 "bare-metal", "libvirt"], 138 help="The platform you're targeting. Default is openstack") 139 parser.add_argument("-l", "--log-level", dest="log_level", default=None, help="log level", 140 choices=["DEBUG", "INFO", "WARNING", "ERROR"]) 141 parser.add_argument("-c","--print-conf", dest="print_conf", action="store_true", 142 help="prints the configuration") 143 144 # Sub commands 145 commands = parser.add_subparsers(help="command", dest="command") 146 147 cmd_info = commands.add_parser("info", help='ip info') 148 cmd_info.set_defaults(func=info) 149 150 cmd_config = commands.add_parser("config", help='prints configuration to log') 151 cmd_config.set_defaults(func=config) 152 153 cmd_log = commands.add_parser("get_logs", help="gather logs from nodes") 154 cmd_log.set_defaults(func=get_logs) 155 156 cmd_cleanup = commands.add_parser( 157 "cleanup", help="cleanup created skuba environment") 158 cmd_cleanup.set_defaults(func=cleanup) 159 160 cmd_provision = commands.add_parser("provision", help="provision nodes for cluster in " 161 "your configured platform e.g: openstack, vmware.") 162 cmd_provision.set_defaults(func=provision) 163 cmd_provision.add_argument("-m", "--master-count", dest="master_count", type=int, default=-1, 164 help='number of masters nodes to be deployed. eg: -m 2') 165 cmd_provision.add_argument("-w", "--worker-count", dest="worker_count", type=int, default=-1, 166 help='number of workers nodes to be deployed. eg: -w 2') 167 168 # common parameters for cluster bootstrap 169 bootstrap_args = ArgumentParser(add_help=False) 170 bootstrap_args.add_argument("-k", "--kubernetes-version", help="kubernetes version", 171 dest="kubernetes_version", default=None) 172 bootstrap_args.add_argument("-c", "--cloud-provider", action="store_true", 173 help="Use cloud provider integration") 174 bootstrap_args.add_argument("-t", "--timeout", type=int, default=300, 175 help="timeout for waiting a node to become ready (seconds)") 176 bootstrap_args.add_argument("-m", "--registry-mirror", metavar=('R', 'M'), 177 help="Add to the registry R a mirror M." 178 " If an image is available at the mirror" 179 " it will be preferred, otherwise the" 180 " image in the original registry is used." 181 " This argument can be used multiple" 182 " times, then mirrors will be tried in" 183 " that order." 184 " Example: --registry-mirror registry.example.com/path test-registry.example.com/path", 185 nargs=2, action='append') 186 187 cmd_bootstrap = commands.add_parser("bootstrap", parents=[bootstrap_args], 188 help="bootstrap k8s cluster") 189 cmd_bootstrap.set_defaults(func=bootstrap) 190 191 cmd_deploy = commands.add_parser("deploy", parents=[bootstrap_args], 192 help="initializes, bootstrap and join all nodes k8s") 193 cmd_deploy.set_defaults(func=deploy) 194 195 196 cmd_status = commands.add_parser("status", help="check K8s cluster status") 197 cmd_status.set_defaults(func=cluster_status) 198 199 cmd_cluster_upgrade_plan = commands.add_parser( 200 "cluster-upgrade-plan", help="Cluster upgrade plan") 201 cmd_cluster_upgrade_plan.set_defaults(func=cluster_upgrade_plan) 202 203 # common parameters for node commands 204 node_args = ArgumentParser(add_help=False) 205 node_args.add_argument("-r", "--role", dest="role", choices=["master", "worker"], 206 help='role of the node to be added or deleted. eg: --role master') 207 node_args.add_argument("-n", "--node", dest="node", type=int, 208 help='node to be added or deleted. eg: -n 0') 209 # End of common parameters 210 211 cmd_join_node = commands.add_parser("join-node", parents=[node_args], 212 help="add node in k8s cluster with the given role.") 213 cmd_join_node.add_argument("-t", "--timeout", type=int, default=180, 214 help="timeout for waiting the node to become ready (seconds)") 215 cmd_join_node.set_defaults(func=join_node) 216 217 cmd_rem_node = commands.add_parser("remove-node", parents=[node_args], 218 help="remove node from k8s cluster.") 219 cmd_rem_node.set_defaults(func=remove_node) 220 221 cmd_node_upgrade = commands.add_parser("node-upgrade", parents=[node_args], 222 help="upgrade kubernetes version in node") 223 cmd_node_upgrade.add_argument("-a", "--action", dest="upgrade_action", 224 help="action: plan or apply upgrade", choices=["plan", "apply"]) 225 cmd_node_upgrade.set_defaults(func=node_upgrade) 226 227 cmd_check_node = commands.add_parser("check-node", parents=[node_args], 228 help="check node health") 229 cmd_check_node.add_argument("-c", "--check", dest="checks", nargs='+', 230 help="check to be executed (multiple checks can be specified") 231 cmd_check_node.add_argument("-s", "--stage", dest="stage", 232 help="only execute checks that apply to this stage") 233 cmd_check_node.set_defaults(func=node_check) 234 235 # Start Join Nodes 236 cmd_join_nodes = commands.add_parser("join-nodes", 237 help="add multiple provisioned nodes k8s.") 238 cmd_join_nodes.add_argument("-m", "--masters", type=int, 239 help="Specify how many masters to join. Default is all") 240 cmd_join_nodes.add_argument("-w", "--workers", type=int, 241 help="Specify how many workers to join. Default is all") 242 cmd_join_nodes.add_argument("-t", "--timeout", type=int, default=180, 243 help="timeout for waiting the master nodes to become ready (seconds)") 244 cmd_join_nodes.set_defaults(func=join_nodes) 245 # End Join Nodes 246 247 ssh_args = ArgumentParser(add_help=False) 248 ssh_args.add_argument("-c", "--cmd", dest="cmd", nargs=REMAINDER, help="remote command and its arguments. e.g ls -al. Must be last argument for ssh command") 249 cmd_ssh = commands.add_parser("ssh", parents=[node_args, ssh_args], help="Execute command in node via ssh.") 250 cmd_ssh.set_defaults(func=ssh) 251 252 test_args = ArgumentParser(add_help=False) 253 test_args.add_argument("-f", "--filter", dest="mark", help="Filter the tests based on markers") 254 test_args.add_argument("-j", "--junit", help="Name of the xml file to record the results to.") 255 test_args.add_argument("-m", "--module", dest="module", help="folder with the tests") 256 test_args.add_argument("-s", "--suite", dest="test_suite", help="test file name") 257 test_args.add_argument("-t", "--test", dest="test", help="test to execute") 258 test_args.add_argument("-l", "--list", dest="collect", action="store_true", default=False, 259 help="only list tests to be executed") 260 test_args.add_argument("-v", "--verbose", dest="verbose", action="store_true", default=False, 261 help="show all output from testrunner libraries") 262 test_args.add_argument("--skip-setup", 263 choices=['provisioned', 'bootstrapped', 'deployed'], 264 help="Skip the given setup step.\n" 265 "'provisioned' For when you have already provisioned the nodes.\n" 266 "'bootstrapped' For when you have already bootstrapped the cluster.\n" 267 "'deployed' For when you already have a fully deployed cluster.") 268 test_args.add_argument("--traceback", default="short", choices=['long', 'short', 'line', 'no'], 269 help="level of detail in traceback for test failure") 270 cmd_test = commands.add_parser( 271 "test", parents=[test_args], help="execute tests") 272 cmd_test.set_defaults(func=test) 273 274 cmd_inhibit_kured = commands.add_parser("inhibit_kured", 275 help="Prevent kured to reboot nodes") 276 cmd_inhibit_kured.set_defaults(func=inhibit_kured) 277 278 cmd_check_cluster = commands.add_parser("check-cluster", 279 help="check cluster health") 280 cmd_check_cluster.add_argument("-c", "--check", dest="checks", nargs='+', 281 help="check to be executed (multiple checks can be specified") 282 cmd_check_cluster.add_argument("-s", "--stage", dest="stage", 283 help="only execute checks that apply to this stage") 284 cmd_check_cluster.set_defaults(func=cluster_check) 285 286 options = parser.parse_args() 287 try: 288 conf = BaseConfig(options.yaml_path) 289 Logger.config_logger(conf, level=options.log_level) 290 options.conf = conf 291 if options.print_conf: 292 out = io.StringIO() 293 BaseConfig.print(conf, out=out) 294 logger.debug(f'Configuration\n{out.getvalue()}') 295 296 options.func(options) 297 except SystemExit as ex: 298 if ex.code > 0: 299 logger.error(f'Command {options.command} ended with error code {ex.code}') 300 sys.exit(ex.code) 301 except Exception as ex: 302 logger.error(f'Exception {ex} executing command {options.command}', exc_info=True) 303 sys.exit(255) 304 305 sys.exit(0) 306 307 if __name__ == '__main__': 308 main()