github.com/SUSE/skuba@v1.4.17/ci/infra/testrunner/testrunner.py (about)

     1  #!/usr/bin/env python3 -Wd -b
     2  
     3  """
     4      Runs end-to-end product tests for v4+.
     5      This script can be run from Jenkins or manually, on developer desktops or servers.
     6  """
     7  
     8  import io
     9  import logging
    10  import sys
    11  from argparse import REMAINDER, ArgumentParser
    12  
    13  import platforms
    14  from skuba import Skuba
    15  from kubectl import Kubectl
    16  from tests import TestDriver
    17  from utils import BaseConfig, Logger, Utils
    18  from checks import Checker
    19  
    20  __version__ = "0.0.3"
    21  
    22  logger = logging.getLogger("testrunner")
    23  
    24  
    25  def info(options):
    26      print(Utils(options.conf).info())
    27  
    28  
    29  def config(options):
    30      BaseConfig.print(options.conf)
    31  
    32  def cleanup(options):
    33      platforms.get_platform(options.conf, options.platform).cleanup()
    34      Skuba.cleanup(options.conf)
    35  
    36  
    37  def provision(options):
    38      platforms.get_platform(options.conf, options.platform).provision(
    39          num_master=options.master_count,
    40          num_worker=options.worker_count)
    41  
    42  
    43  def deploy(options):
    44      skuba = Skuba(options.conf, options.platform)
    45      skuba.cluster_deploy(
    46          kubernetes_version=options.kubernetes_version,
    47          cloud_provider=options.cloud_provider,
    48          timeout=options.timeout,
    49          registry_mirror=options.registry_mirror)
    50  
    51  
    52  def bootstrap(options):
    53      skuba = Skuba(options.conf, options.platform)
    54      skuba.cluster_bootstrap(
    55          kubernetes_version=options.kubernetes_version,
    56          cloud_provider=options.cloud_provider,
    57          timeout=options.timeout,
    58          registry_mirror=options.registry_mirror)
    59  
    60  
    61  def cluster_status(options):
    62      print(Skuba(options.conf, options.platform).cluster_status())
    63  
    64  
    65  def cluster_upgrade_plan(options):
    66      Skuba(options.conf, options.platform).cluster_upgrade_plan()
    67  
    68  
    69  def get_logs(options):
    70      platform_logging_errors = platforms.get_platform(
    71          options.conf, options.platform).gather_logs()
    72  
    73      if platform_logging_errors:
    74          raise Exception("Failure(s) while collecting logs")
    75  
    76  
    77  def join_node(options):
    78      Skuba(options.conf, options.platform).node_join(
    79          role=options.role, nr=options.node, timeout=options.timeout)
    80  
    81  
    82  def join_nodes(options):
    83      skuba = Skuba(options.conf, options.platform)
    84      skuba.join_nodes(
    85          masters=options.masters,
    86          workers=options.workers,
    87          timeout=options.timeout
    88      )
    89  
    90  def remove_node(options):
    91      Skuba(options.conf, options.platform).node_remove(
    92          role=options.role, nr=options.node)
    93  
    94  def node_upgrade(options):
    95      Skuba(options.conf, options.platform).node_upgrade(
    96          action=options.upgrade_action, role=options.role, nr=options.node)
    97  
    98  def node_check(options):
    99      Checker(options.conf, options.platform).check_node(
   100           role=options.role, node=options.node,
   101           checks=options.checks, stage=options.stage)
   102  
   103  def cluster_check(options):
   104      Checker(options.conf, options.platform).check_cluster(
   105           checks=options.checks, stage=options.stage)
   106  
   107  def test(options):
   108      test_driver = TestDriver(options.conf, options.platform)
   109      test_driver.run(module=options.module, test_suite=options.test_suite, test=options.test,
   110                      verbose=options.verbose, collect=options.collect, skip_setup=options.skip_setup,
   111                      mark=options.mark, traceback=options.traceback, junit=options.junit)
   112  
   113  
   114  def ssh(options):
   115      platforms.get_platform(options.conf, options.platform).ssh_run(
   116          role=options.role, nr=options.node, cmd=" ".join(options.cmd))
   117  
   118  
   119  def inhibit_kured(options):
   120      Kubectl(options.conf).inhibit_kured()
   121  
   122  
   123  def main():
   124      help_str = """
   125      This script is meant to be run manually on test servers, developer desktops, or Jenkins.
   126      This script supposed to run on python virtualenv from testrunner. Requires root privileges.
   127      Warning: it removes docker containers, VMs, images, and network configuration.
   128      """
   129      parser = ArgumentParser(help_str)
   130  
   131      # Common parameters
   132      parser.add_argument("-v", "--vars", dest="yaml_path", default="vars.yaml",
   133                          help='path for platform yaml file. Default is vars.yaml. eg: -v myconfig.yaml')
   134      parser.add_argument("-p", "--platform",
   135                          default="openstack",
   136                          choices=["openstack", "vmware",
   137                                   "bare-metal", "libvirt"],
   138                          help="The platform you're targeting. Default is openstack")
   139      parser.add_argument("-l", "--log-level", dest="log_level", default=None, help="log level",
   140                          choices=["DEBUG", "INFO", "WARNING", "ERROR"])
   141      parser.add_argument("-c","--print-conf", dest="print_conf", action="store_true",
   142                          help="prints the configuration")
   143  
   144      # Sub commands
   145      commands = parser.add_subparsers(help="command", dest="command")
   146  
   147      cmd_info = commands.add_parser("info", help='ip info')
   148      cmd_info.set_defaults(func=info)
   149  
   150      cmd_config = commands.add_parser("config", help='prints configuration to log')
   151      cmd_config.set_defaults(func=config)
   152  
   153      cmd_log = commands.add_parser("get_logs", help="gather logs from nodes")
   154      cmd_log.set_defaults(func=get_logs)
   155  
   156      cmd_cleanup = commands.add_parser(
   157          "cleanup", help="cleanup created skuba environment")
   158      cmd_cleanup.set_defaults(func=cleanup)
   159  
   160      cmd_provision = commands.add_parser("provision", help="provision nodes for cluster in "
   161                                                            "your configured platform e.g: openstack, vmware.")
   162      cmd_provision.set_defaults(func=provision)
   163      cmd_provision.add_argument("-m", "--master-count", dest="master_count", type=int, default=-1,
   164                                 help='number of masters nodes to be deployed. eg: -m 2')
   165      cmd_provision.add_argument("-w", "--worker-count", dest="worker_count", type=int, default=-1,
   166                                 help='number of workers nodes to be deployed. eg: -w 2')
   167  
   168      # common parameters for cluster bootstrap
   169      bootstrap_args = ArgumentParser(add_help=False)
   170      bootstrap_args.add_argument("-k", "--kubernetes-version", help="kubernetes version",
   171                                 dest="kubernetes_version", default=None)
   172      bootstrap_args.add_argument("-c", "--cloud-provider", action="store_true",
   173                                 help="Use cloud provider integration")
   174      bootstrap_args.add_argument("-t", "--timeout", type=int, default=300,
   175                                  help="timeout for waiting a node to become ready (seconds)")
   176      bootstrap_args.add_argument("-m", "--registry-mirror", metavar=('R', 'M'),
   177                                  help="Add to the registry R a mirror M."
   178                                       " If an image is available at the mirror"
   179                                       " it will be preferred, otherwise the"
   180                                       " image in the original registry is used."
   181                                       " This argument can be used multiple"
   182                                       " times, then mirrors will be tried in"
   183                                       " that order."
   184                                       " Example: --registry-mirror registry.example.com/path test-registry.example.com/path",
   185                                  nargs=2, action='append')
   186  
   187      cmd_bootstrap = commands.add_parser("bootstrap", parents=[bootstrap_args],
   188                          help="bootstrap k8s cluster")
   189      cmd_bootstrap.set_defaults(func=bootstrap)
   190  
   191      cmd_deploy = commands.add_parser("deploy", parents=[bootstrap_args],
   192                          help="initializes, bootstrap and join all nodes k8s")
   193      cmd_deploy.set_defaults(func=deploy)
   194  
   195  
   196      cmd_status = commands.add_parser("status", help="check K8s cluster status")
   197      cmd_status.set_defaults(func=cluster_status)
   198  
   199      cmd_cluster_upgrade_plan = commands.add_parser(
   200          "cluster-upgrade-plan", help="Cluster upgrade plan")
   201      cmd_cluster_upgrade_plan.set_defaults(func=cluster_upgrade_plan)
   202  
   203      # common parameters for node commands
   204      node_args = ArgumentParser(add_help=False)
   205      node_args.add_argument("-r", "--role", dest="role", choices=["master", "worker"],
   206                             help='role of the node to be added or deleted. eg: --role master')
   207      node_args.add_argument("-n", "--node", dest="node", type=int,
   208                             help='node to be added or deleted.  eg: -n 0')
   209     # End of common parameters
   210  
   211      cmd_join_node = commands.add_parser("join-node", parents=[node_args],
   212                                          help="add node in k8s cluster with the given role.")
   213      cmd_join_node.add_argument("-t", "--timeout", type=int, default=180,
   214                                  help="timeout for waiting the node to become ready (seconds)")
   215      cmd_join_node.set_defaults(func=join_node)
   216  
   217      cmd_rem_node = commands.add_parser("remove-node", parents=[node_args],
   218                                         help="remove node from k8s cluster.")
   219      cmd_rem_node.set_defaults(func=remove_node)
   220  
   221      cmd_node_upgrade = commands.add_parser("node-upgrade", parents=[node_args],
   222                                             help="upgrade kubernetes version in node")
   223      cmd_node_upgrade.add_argument("-a", "--action", dest="upgrade_action",
   224                                    help="action: plan or apply upgrade", choices=["plan", "apply"])
   225      cmd_node_upgrade.set_defaults(func=node_upgrade)
   226  
   227      cmd_check_node = commands.add_parser("check-node", parents=[node_args],
   228                                            help="check node health")
   229      cmd_check_node.add_argument("-c", "--check", dest="checks", nargs='+',
   230                                  help="check to be executed (multiple checks can be specified")
   231      cmd_check_node.add_argument("-s", "--stage", dest="stage",
   232                                  help="only execute checks that apply to this stage")
   233      cmd_check_node.set_defaults(func=node_check)
   234  
   235      # Start Join Nodes
   236      cmd_join_nodes = commands.add_parser("join-nodes",
   237                                           help="add multiple provisioned nodes k8s.")
   238      cmd_join_nodes.add_argument("-m", "--masters", type=int,
   239                                  help="Specify how many masters to join. Default is all")
   240      cmd_join_nodes.add_argument("-w", "--workers", type=int,
   241                                  help="Specify how many workers to join. Default is all")
   242      cmd_join_nodes.add_argument("-t", "--timeout", type=int, default=180,
   243                                  help="timeout for waiting the master nodes to become ready (seconds)")
   244      cmd_join_nodes.set_defaults(func=join_nodes)
   245      # End Join Nodes
   246  
   247      ssh_args = ArgumentParser(add_help=False)
   248      ssh_args.add_argument("-c", "--cmd", dest="cmd", nargs=REMAINDER, help="remote command and its arguments. e.g ls -al. Must be last argument for ssh command")
   249      cmd_ssh = commands.add_parser("ssh", parents=[node_args, ssh_args], help="Execute command in node via ssh.")
   250      cmd_ssh.set_defaults(func=ssh)
   251  
   252      test_args = ArgumentParser(add_help=False)
   253      test_args.add_argument("-f", "--filter", dest="mark", help="Filter the tests based on markers")
   254      test_args.add_argument("-j", "--junit", help="Name of the xml file to record the results to.")
   255      test_args.add_argument("-m", "--module", dest="module", help="folder with the tests")
   256      test_args.add_argument("-s", "--suite", dest="test_suite", help="test file name")
   257      test_args.add_argument("-t", "--test", dest="test", help="test to execute")
   258      test_args.add_argument("-l", "--list", dest="collect", action="store_true", default=False,
   259                             help="only list tests to be executed")
   260      test_args.add_argument("-v", "--verbose", dest="verbose", action="store_true", default=False,
   261                             help="show all output from testrunner libraries")
   262      test_args.add_argument("--skip-setup",
   263                             choices=['provisioned', 'bootstrapped', 'deployed'],
   264                             help="Skip the given setup step.\n"
   265                                  "'provisioned' For when you have already provisioned the nodes.\n"
   266                                  "'bootstrapped' For when you have already bootstrapped the cluster.\n"
   267                                  "'deployed' For when you already have a fully deployed cluster.")
   268      test_args.add_argument("--traceback", default="short", choices=['long', 'short', 'line', 'no'],
   269                             help="level of detail in traceback for test failure")
   270      cmd_test = commands.add_parser(
   271          "test", parents=[test_args], help="execute tests")
   272      cmd_test.set_defaults(func=test)
   273  
   274      cmd_inhibit_kured = commands.add_parser("inhibit_kured",
   275                             help="Prevent kured to reboot nodes")
   276      cmd_inhibit_kured.set_defaults(func=inhibit_kured)
   277  
   278      cmd_check_cluster = commands.add_parser("check-cluster",
   279                                            help="check cluster health")
   280      cmd_check_cluster.add_argument("-c", "--check", dest="checks", nargs='+',
   281                                  help="check to be executed (multiple checks can be specified")
   282      cmd_check_cluster.add_argument("-s", "--stage", dest="stage",
   283                                  help="only execute checks that apply to this stage")
   284      cmd_check_cluster.set_defaults(func=cluster_check)
   285  
   286      options = parser.parse_args()
   287      try:
   288          conf = BaseConfig(options.yaml_path)
   289          Logger.config_logger(conf, level=options.log_level)
   290          options.conf = conf
   291          if options.print_conf:
   292              out = io.StringIO()
   293              BaseConfig.print(conf, out=out)
   294              logger.debug(f'Configuration\n{out.getvalue()}')
   295  
   296          options.func(options)
   297      except SystemExit as ex:
   298         if ex.code > 0:
   299            logger.error(f'Command {options.command} ended with error code {ex.code}')
   300            sys.exit(ex.code)
   301      except Exception as ex:
   302          logger.error(f'Exception {ex} executing command {options.command}', exc_info=True)
   303          sys.exit(255)
   304  
   305      sys.exit(0)
   306  
   307  if __name__ == '__main__':
   308      main()