github.com/niedbalski/juju@v0.0.0-20190215020005-8ff100488e47/acceptancetests/assess_caas_deploy_charms.py (about)

     1  #!/usr/bin/env python3
     2  """ Test caas k8s cluster bootstrap
     3  
     4      1. spinning up k8s cluster and asserting the cluster is `healthy`;
     5      2. deploy gitlab, mysql charms to caas model;
     6      3. relate gitlab mysql;
     7      4. assert http health check on gitlab
     8  """
     9  
    10  from __future__ import print_function
    11  
    12  import argparse
    13  import logging
    14  import sys
    15  import os
    16  import subprocess
    17  from time import sleep
    18  
    19  import requests
    20  
    21  from deploy_stack import (
    22      BootstrapManager,
    23      deploy_caas_stack,
    24  )
    25  from utility import (
    26      add_basic_testing_arguments,
    27      configure_logging,
    28      JujuAssertionError,
    29  )
    30  
    31  from jujucharm import (
    32      local_charm_path
    33  )
    34  from jujupy.utility import until_timeout
    35  
    36  __metaclass__ = type
    37  
    38  
    39  log = logging.getLogger("assess_caas_charm_deployment")
    40  
    41  JUJU_STORAGECLASS_NAME = "juju-storageclass"
    42  HOST_PATH_PROVISIONER = """
    43  apiVersion: v1
    44  kind: ServiceAccount
    45  metadata:
    46    name: hostpath-provisioner
    47    namespace: kube-system
    48  ---
    49  
    50  apiVersion: rbac.authorization.k8s.io/v1beta1
    51  kind: ClusterRole
    52  metadata:
    53    name: hostpath-provisioner
    54    namespace: kube-system
    55  rules:
    56    - apiGroups: [""]
    57      resources: ["persistentvolumes"]
    58      verbs: ["get", "list", "watch", "create", "delete"]
    59    - apiGroups: [""]
    60      resources: ["persistentvolumeclaims"]
    61      verbs: ["get", "list", "watch", "update"]
    62    - apiGroups: ["storage.k8s.io"]
    63      resources: ["storageclasses"]
    64      verbs: ["get", "list", "watch"]
    65    - apiGroups: [""]
    66      resources: ["events"]
    67      verbs: ["list", "watch", "create", "update", "patch"]
    68  ---
    69  
    70  apiVersion: rbac.authorization.k8s.io/v1beta1
    71  kind: ClusterRoleBinding
    72  metadata:
    73    name: hostpath-provisioner
    74    namespace: kube-system
    75  subjects:
    76    - kind: ServiceAccount
    77      name: hostpath-provisioner
    78      namespace: kube-system
    79  roleRef:
    80    kind: ClusterRole
    81    name: hostpath-provisioner
    82    apiGroup: rbac.authorization.k8s.io
    83  ---
    84  
    85  apiVersion: rbac.authorization.k8s.io/v1beta1
    86  kind: Role
    87  metadata:
    88    name: hostpath-provisioner
    89    namespace: kube-system
    90  rules:
    91    - apiGroups: [""]
    92      resources: ["secrets"]
    93      verbs: ["create", "get", "delete"]
    94  ---
    95  
    96  apiVersion: rbac.authorization.k8s.io/v1beta1
    97  kind: RoleBinding
    98  metadata:
    99    name: hostpath-provisioner
   100    namespace: kube-system
   101  roleRef:
   102    apiGroup: rbac.authorization.k8s.io
   103    kind: Role
   104    name: hostpath-provisioner
   105  subjects:
   106    - kind: ServiceAccount
   107      name: hostpath-provisioner
   108  ---
   109  
   110  # -- Create a daemon set for web requests and send them to the nginx-ingress-controller
   111  apiVersion: extensions/v1beta1
   112  kind: DaemonSet
   113  metadata:
   114    name: hostpath-provisioner
   115    namespace: kube-system
   116  spec:
   117    revisionHistoryLimit: 3
   118    template:
   119      metadata:
   120        labels:
   121          app: hostpath-provisioner
   122      spec:
   123        serviceAccountName: hostpath-provisioner
   124        terminationGracePeriodSeconds: 0
   125        containers:
   126          - name: hostpath-provisioner
   127            image: mazdermind/hostpath-provisioner:latest
   128            imagePullPolicy: "IfNotPresent"
   129            env:
   130              - name: NODE_NAME
   131                valueFrom:
   132                  fieldRef:
   133                    fieldPath: spec.nodeName
   134              - name: PV_DIR
   135                value: /mnt/kubernetes
   136            volumeMounts:
   137              - name: pv-volume
   138                mountPath: /mnt/kubernetes
   139        volumes:
   140          - name: pv-volume
   141            hostPath:
   142              path: /mnt/kubernetes
   143  ---
   144  
   145  # -- Create the standard storage class for running on-node hostpath storage
   146  apiVersion: storage.k8s.io/v1
   147  kind: StorageClass
   148  metadata:
   149    # namespace: kube-system
   150    name: {class_name}
   151    annotations:
   152      storageclass.beta.kubernetes.io/is-default-class: "true"
   153    labels:
   154      kubernetes.io/cluster-service: "true"
   155      addonmanager.kubernetes.io/mode: EnsureExists
   156  provisioner: hostpath
   157  ---
   158  """
   159  
   160  
   161  def check_app_healthy(url, timeout=300, success_hook=lambda: None, fail_hook=lambda: None):
   162      if not callable(success_hook) or not callable(fail_hook):
   163          raise RuntimeError("hooks are not callable")
   164  
   165      status_code = None
   166      for remaining in until_timeout(timeout):
   167          try:
   168              r = requests.get(url)
   169              if r.ok and r.status_code < 400:
   170                  return success_hook()
   171              status_code = r.status_code
   172          except IOError as e:
   173              log.error(e)
   174          finally:
   175              sleep(3)
   176              if remaining % 60 == 0:
   177                  log.info('timeout in %ss', remaining)
   178      log.error('HTTP health check failed -> %s, status_code -> %s !', url, status_code)
   179      fail_hook()
   180      raise JujuAssertionError('gitlab is not healthy')
   181  
   182  
   183  def assess_caas_charm_deployment(client):
   184      # Deploy k8s bundle to spin up k8s cluster
   185      bundle = local_charm_path(
   186          charm='bundles-kubernetes-core-lxd.yaml',
   187          repository=os.environ['JUJU_REPOSITORY'],
   188          juju_ver=client.version
   189      )
   190  
   191      caas_client = deploy_caas_stack(path=bundle, client=client, timeout=4000)
   192      external_hostname = caas_client.get_external_hostname()
   193  
   194      if not caas_client.is_cluster_healthy:
   195          raise JujuAssertionError('k8s cluster is not healthy because kubectl is not accessible')
   196  
   197      # tmp fix kubernetes core ingress issue
   198      ingress_controller_daemonset_name = 'daemonset.apps/nginx-ingress-kubernetes-worker-controller'
   199      o = caas_client.kubectl(
   200          'patch', ingress_controller_daemonset_name, '--patch',
   201          '''
   202          {"spec": {"template": {"spec": {"containers": [{"name": "nginx-ingress-kubernetes-worker","args": ["/nginx-ingress-controller", "--default-backend-service=$(POD_NAMESPACE)/default-http-backend", "--configmap=$(POD_NAMESPACE)/nginx-load-balancer-conf", "--enable-ssl-chain-completion=False", "--publish-status-address=%s"]}]}}}}
   203          ''' % caas_client.get_first_worker_ip()
   204      )
   205      log.info(o)
   206  
   207      o = caas_client.kubectl('get', ingress_controller_daemonset_name, '-o', 'yaml')
   208      log.info(o)
   209  
   210      # add caas model for deploying caas charms on top of it
   211      model_name = 'testcaas'
   212      k8s_model = caas_client.add_model(model_name)
   213  
   214      # ensure tmp dir for storage class.model_name
   215      o = subprocess.check_output(
   216          ('sudo', 'mkdir', '-p', '/mnt/kubernetes/%s' % model_name)  # unfortunately, needs sudo
   217      )
   218      log.debug(o.decode('UTF-8').strip())
   219  
   220      # ensure storage class
   221      caas_client.kubectl_apply(HOST_PATH_PROVISIONER.format(class_name=JUJU_STORAGECLASS_NAME))
   222  
   223      # ensure storage pools for caas operator
   224      k8s_model.juju(
   225          'create-storage-pool',
   226          ('operator-storage', 'kubernetes', 'storage-class=%s' % JUJU_STORAGECLASS_NAME)
   227      )
   228  
   229      # ensure storage pools for mariadb
   230      mariadb_storage_pool_name = 'mariadb-pv'
   231      k8s_model.juju(
   232          'create-storage-pool',
   233          (mariadb_storage_pool_name, 'kubernetes', 'storage-class=%s' % JUJU_STORAGECLASS_NAME)
   234      )
   235  
   236      k8s_model.deploy(
   237          charm="cs:~juju/gitlab-k8s-0",
   238          config='juju-external-hostname={}'.format(external_hostname),
   239      )
   240  
   241      k8s_model.deploy(
   242          charm="cs:~juju/mariadb-k8s-0",
   243          storage='database=100M,{pool_name}'.format(pool_name=mariadb_storage_pool_name),
   244      )
   245  
   246      k8s_model.juju('relate', ('mariadb-k8s', 'gitlab-k8s'))
   247      k8s_model.juju('expose', ('gitlab-k8s',))
   248      k8s_model.wait_for_workloads(timeout=3600)
   249  
   250      def success_hook():
   251          log.info(caas_client.kubectl('get', 'all', '--all-namespaces'))
   252  
   253      def fail_hook():
   254          success_hook()
   255          log.info(caas_client.kubectl('get', ingress_controller_daemonset_name, '-o', 'yaml'))
   256          log.info(caas_client.kubectl('get', 'pv,pvc', '-n', model_name))
   257  
   258      url = '{}://{}/{}'.format('http', external_hostname, 'gitlab-k8s')
   259      check_app_healthy(
   260          url, timeout=1800,
   261          success_hook=success_hook,
   262          fail_hook=fail_hook,
   263      )
   264      k8s_model.juju(k8s_model._show_status, ('--format', 'tabular'))
   265  
   266  
   267  def parse_args(argv):
   268      """Parse all arguments."""
   269      parser = argparse.ArgumentParser(description="Cass charm deployment CI test")
   270      parser.add_argument(
   271          '--caas-image', action='store', default=None,
   272          help='Caas operator docker image name to use with format of <username>/caas-jujud-operator:<tag>.'
   273      )
   274  
   275      add_basic_testing_arguments(parser, existing=False)
   276      return parser.parse_args(argv)
   277  
   278  
   279  def ensure_operator_image_path(client, image_path):
   280      client.controller_juju('controller-config', ('caas-operator-image-path={}'.format(image_path),))
   281  
   282  
   283  def main(argv=None):
   284      args = parse_args(argv)
   285      configure_logging(args.verbose)
   286      bs_manager = BootstrapManager.from_args(args)
   287      with bs_manager.booted_context(args.upload_tools):
   288          client = bs_manager.client
   289          ensure_operator_image_path(client, image_path=args.caas_image)
   290          assess_caas_charm_deployment(client)
   291      return 0
   292  
   293  
   294  if __name__ == '__main__':
   295      sys.exit(main())