github.com/SUSE/skuba@v1.4.17/ci/infra/testrunner/platforms/terraform.py (about)

     1  import json
     2  import logging
     3  import os
     4  from urllib.parse import urlparse
     5  
     6  import hcl
     7  
     8  from platforms.platform import Platform
     9  from utils import (Format, step, Utils)
    10  
    11  logger = logging.getLogger('testrunner')
    12  
    13  
    14  class Terraform(Platform):
    15      def __init__(self, conf, platform):
    16          super().__init__(conf)
    17          if not conf.terraform.stack_name:
    18              raise ValueError("a terraform stack name must be specified")
    19  
    20          self.tfdir = os.path.join(self.conf.terraform.tfdir, platform)
    21          self.tfjson_path = os.path.join(self.conf.terraform.workdir, "tfout.json")
    22          self.tfout_path = os.path.join(self.conf.terraform.workdir, "tfout")
    23          self.utils = Utils(conf)
    24          self.state = None
    25  
    26          self.logs["files"] += ["/var/run/cloud-init/status.json",
    27                                 "/var/log/cloud-init-output.log",
    28                                 "/var/log/cloud-init.log"]
    29  
    30          self.tmp_files = [self.tfout_path,
    31                            self.tfjson_path]
    32  
    33      def destroy(self, variables=[]):
    34          cmd = "destroy -auto-approve"
    35  
    36          for var in variables:
    37              cmd += f" -var {var}"
    38  
    39          self._run_terraform_command(cmd)
    40  
    41      def _provision_platform(self, masters=-1, workers=-1):
    42  
    43          if masters > -1:
    44              self.conf.terraform.master.count = masters
    45          if workers > -1:
    46              self.conf.terraform.worker.count = workers
    47  
    48          exception = None
    49          self._check_tf_deployed()
    50  
    51          init_cmd = "init"
    52          if self.conf.terraform.plugin_dir:
    53              logger.info(f"Installing plugins from {self.conf.terraform.plugin_dir}")
    54              init_cmd += f" -plugin-dir={self.conf.terraform.plugin_dir}"
    55          self._run_terraform_command(init_cmd)
    56  
    57          self._run_terraform_command("version")
    58          self._generate_tfvars_file()
    59          plan_cmd = f"plan -out {self.tfout_path}"
    60          apply_cmd = f"apply -auto-approve {self.tfout_path}"
    61  
    62          self._run_terraform_command(plan_cmd)
    63  
    64          try:
    65              self._run_terraform_command(apply_cmd)
    66          except Exception as ex:
    67              exception = ex
    68          finally:
    69              try:
    70                  self._fetch_terraform_output()
    71              except Exception as inner_ex:
    72                  # don't override original exception if any
    73                  if not exception:
    74                      exception = inner_ex
    75  
    76          if exception:
    77             raise exception
    78  
    79      def _load_tfstate(self):
    80          if self.state is None:
    81              fn = os.path.join(self.tfdir, "terraform.tfstate")
    82              logger.debug("Reading configuration from {}".format(fn))
    83              with open(fn) as f:
    84                  self.state = json.load(f)
    85  
    86      def get_lb_ipaddr(self):
    87          self._load_tfstate()
    88          if self.state["version"] == 3:
    89              return self.state["modules"][0]["outputs"]["ip_load_balancer"]["value"]["{}-lb".format(self.stack_name())]
    90          elif self.state["version"] == 4:
    91              return self.state["outputs"]["ip_load_balancer"]["value"]["{}-lb".format(self.stack_name())]
    92  
    93      def get_num_nodes(self, role):
    94          return len(self.get_nodes_ipaddrs(role))
    95  
    96      def get_nodes_names(self, role):
    97          stack_name = self.stack_name()
    98          return [f'caasp-{role}-{stack_name}-{i}' for i in range(self.get_num_nodes(role))]
    99  
   100      def get_nodes_ipaddrs(self, role):
   101          self._load_tfstate()
   102  
   103          if role not in ("master", "worker"):
   104              raise ValueError("Invalid role: {}".format(role))
   105  
   106          role_key = "ip_" + role + "s"
   107          if self.state["version"] == 3:
   108              return list(self.state["modules"][0]["outputs"][role_key]["value"].values())
   109          elif self.state["version"] == 4:
   110              return list(self.state["outputs"][role_key]["value"].values())
   111  
   112      @step
   113      def _fetch_terraform_output(self):
   114          cmd = f"output -json >{self.tfjson_path}"
   115          self._run_terraform_command(cmd)
   116  
   117      def _generate_tfvars_file(self):
   118          """Generate terraform tfvars file"""
   119          tfvars_template = os.path.join(self.tfdir, self.conf.terraform.tfvars)
   120          tfvars_final = os.path.join(self.tfdir, "terraform.tfvars.json")
   121  
   122          with open(tfvars_template) as f:
   123              if '.json' in os.path.basename(tfvars_template).lower():
   124                  tfvars = json.load(f)
   125              else:
   126                  tfvars = hcl.load(f)
   127  
   128              self._update_tfvars(tfvars)
   129  
   130              with open(tfvars_final, "w") as f:
   131                  json.dump(tfvars, f)
   132  
   133      # take up to 45 characters from stackname to give room to the fixed part
   134      # in the node name: caasp-[master|worker]-<stack name>-xxx (total length
   135      # must be <= 63).
   136      # Also ensure that only valid character are present and that the string
   137      # starts and ends with alphanumeric characters and all lowercase.
   138      def stack_name(self):
   139           stack_name = self.conf.terraform.stack_name[:45]
   140           stack_name = stack_name.replace("_","-").replace("/","-")
   141           stack_name = stack_name.strip("-.")
   142           stack_name = stack_name.lower()
   143  
   144           return stack_name
   145  
   146      def _update_tfvars(self, tfvars):
   147          new_vars = {
   148              "internal_net": self.conf.terraform.internal_net,
   149              "stack_name": self.stack_name(),
   150              "username": self.utils.ssh_user(),
   151              "masters": self.conf.terraform.master.count,
   152              "master_memory": self.conf.terraform.master.memory,
   153              "master_vcpu": self.conf.terraform.master.cpu,
   154              "workers": self.conf.terraform.worker.count,
   155              "worker_memory": self.conf.terraform.worker.memory,
   156              "worker_vcpu": self.conf.terraform.worker.cpu,
   157              "lbs": self.conf.terraform.lb.count,
   158              "lb_memory": self.conf.terraform.lb.memory,
   159              "lb_vcpu": self.conf.terraform.lb.cpu,
   160              "authorized_keys": [self.utils.authorized_keys()]
   161          }
   162  
   163          new_vars.update(self.platform_new_vars)
   164  
   165          for k, v in new_vars.items():
   166              if tfvars.get(k) is not None:
   167                  if isinstance(v, list):
   168                      tfvars[k] = tfvars[k] + v
   169                  elif isinstance(v, dict):
   170                      tfvars[k].update(v)
   171                  else:
   172                      tfvars[k] = v
   173  
   174          # if registry code specified, repositories are not needed
   175          if self.conf.packages.registry_code:
   176              tfvars["caasp_registry_code"] = self.conf.packages.registry_code
   177              tfvars["repositories"] = {}
   178  
   179          repos = tfvars.get("repositories", {})
   180          if self.conf.packages.additional_repos:
   181             for name, url in self.conf.packages.additional_repos.items():
   182                  if not url:
   183                      logger.warning(f'skipping repository {name} with empty url')
   184                      continue
   185                  repos[name] = url
   186  
   187          # Update mirror urls
   188          if self.conf.packages.mirror and repos:
   189              for name, url in repos.items():
   190                  url_parsed = urlparse(url)
   191                  url_updated = url_parsed._replace(netloc=self.conf.packages.mirror)
   192                  tfvars["repositories"][name] = url_updated.geturl()
   193  
   194          if self.conf.packages.additional_pkgs:
   195              tfvars["packages"].extend(self.conf.packages.additional_pkgs)
   196  
   197      def _run_terraform_command(self, cmd, env={}):
   198          """Running terraform command in {terraform.tfdir}/{platform}"""
   199          cmd = f'{self._env_setup_cmd()}; terraform {cmd}'
   200          self.utils.runshellcommand(cmd, cwd=self.tfdir, env=env)
   201  
   202      def _check_tf_deployed(self):
   203          if os.path.exists(self.tfjson_path):
   204              raise Exception(Format.alert(f"tf file found. Please run cleanup and try again {self.tfjson_path}"))
   205  
   206      # TODO: this function is currently not used. Identify points where it should
   207      # be invoked
   208      def _verify_tf_dependency(self):
   209          if not os.path.exists(self.tfjson_path):
   210              raise Exception(Format.alert("tf file not found. Please run terraform and try again{}"))