github.com/deis/deis@v1.13.5-0.20170519182049-1d9e59fbdbfc/contrib/linode/provision-linode-cluster.py (about)

     1  #!/usr/bin/env python
     2  """
     3  Provision a Deis cluster on Linode
     4  
     5  Usage: provision-linode-cluster.py
     6  """
     7  import argparse
     8  import random
     9  import string
    10  import threading
    11  from threading import Thread
    12  import sys
    13  
    14  import paramiko
    15  from linodeapi import LinodeApiCommand
    16  import linodeutils
    17  
    18  class ProvisionCommand(LinodeApiCommand):
    19      _created_linodes = []
    20  
    21      def run(self):
    22          # validate arguments
    23          self._check_num_nodes()
    24          self._check_plan_size()
    25  
    26          # create the linodes
    27          self._create_linodes()
    28  
    29          # print the results
    30          self._report_created()
    31  
    32      def _report_created(self):
    33          # set up the report data
    34          rows = []
    35          ips = []
    36          data_center = self._get_data_center().get('ABBR')
    37          plan = self._get_plan().get('RAM')
    38  
    39          for linode in self._created_linodes:
    40              rows.append((
    41                  linode['hostname'],
    42                  linode['public'],
    43                  linode['private'],
    44                  linode['gateway'],
    45                  data_center,
    46                  plan
    47              ))
    48              ips.append(linode['public'])
    49  
    50          firewall_command = './apply-firewall.py --private-key /path/to/key/deis --hosts ' + string.join(ips, ' ')
    51          header_msg = '  Successfully provisioned ' + str(self.num_nodes) + ' nodes!'
    52          footer_msg = '  Finish up your installation by securing your cluster with the following command:\n  ' + firewall_command + '\n'
    53          linodeutils.log_table(rows, header_msg, footer_msg)
    54  
    55      def _get_plan(self):
    56          if self._plan is None:
    57              plans = self.request('avail.linodeplans', params={'PlanID': self.node_plan})
    58              if len(plans) != 1:
    59                  raise ValueError('The --plan specified is invalid. Use the `list-plans` subcommand to see valid ids.')
    60              self._plan = plans[0]
    61          return self._plan
    62  
    63      def _get_plan_id(self):
    64          return self._get_plan().get('PLANID')
    65  
    66      def _get_data_center(self):
    67          if self._data_center is None:
    68              data_centers = self.request('avail.datacenters')
    69              for data_center in data_centers:
    70                  if data_center.get('DATACENTERID') == self.node_data_center:
    71                      self._data_center = data_center
    72              if self._data_center is None:
    73                  raise ValueError('The --datacenter specified is invalid. Use the `list-data-centers` subcommand to see valid ids.')
    74          return self._data_center
    75  
    76      def _get_data_center_id(self):
    77          return self._get_data_center().get('DATACENTERID')
    78  
    79      def _check_plan_size(self):
    80          ram = self._get_plan().get('RAM')
    81          if ram < 4096:
    82              raise ValueError('Deis cluster members must have at least 4GB of memory. Please choose a plan with more memory.')
    83  
    84      def _check_num_nodes(self):
    85          if self.num_nodes < 1:
    86              raise ValueError('Must provision at least one node.')
    87          elif self.num_nodes < 3:
    88              linodeutils.log_warning('A Deis cluster must have 3 or more nodes, only continue if you adding to a current cluster.')
    89              linodeutils.log_warning('Continue? (y/n)')
    90              accept = None
    91              while True:
    92                  if accept == 'y':
    93                      return
    94                  elif accept == 'n':
    95                      raise StandardError('User canceled provisioning')
    96                  else:
    97                      accept = self._get_user_input('--> ').strip().lower()
    98  
    99      def _get_user_input(self, prompt):
   100          if sys.version_info[0] < 3:
   101              return raw_input(prompt)
   102          else:
   103              return input(prompt)
   104  
   105      def _create_linodes(self):
   106          threads = []
   107          for i in range(0, self.num_nodes):
   108              t = Thread(target=self._create_linode,
   109                         args=(self._get_plan_id(), self._get_data_center_id(), self.node_name_prefix, self.node_display_group))
   110              t.setDaemon(False)
   111              t.start()
   112  
   113              threads.append(t)
   114  
   115          for thread in threads:
   116              thread.join()
   117  
   118      def _create_linode(self, plan_id, data_center_id, name_prefix, display_group):
   119          self.info('Creating the Linode...')
   120  
   121          # create the linode
   122          node_id = self.request('linode.create', params={
   123              'DatacenterID': data_center_id,
   124              'PlanID': plan_id
   125          }).get('LinodeID')
   126  
   127          # update the configuration
   128          self.request('linode.update', params={
   129              'LinodeID': node_id,
   130              'Label': name_prefix + str(node_id),
   131              'lpm_displayGroup': display_group,
   132              'Alert_cpu_enabled': False,
   133              'Alert_diskio_enabled': False,
   134              'Alert_bwin_enabled': False,
   135              'Alert_bwout_enabled': False,
   136              'Alert_bwquota_enabled': False
   137          })
   138  
   139          self.success('Linode ' + str(node_id) + ' created!')
   140          hostname = name_prefix + str(node_id)
   141          threading.current_thread().name = hostname
   142  
   143          # configure the networking
   144          network = self._configure_networking(node_id)
   145          network['hostname'] = hostname
   146  
   147          # generate a password for the provisioning disk
   148          password = ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(24))
   149  
   150          # configure the disks
   151          total_hd = self.request('linode.list', params={'LinodeID': node_id})[0]['TOTALHD']
   152          provision_disk_mb = 600
   153          coreos_disk_mb = total_hd - provision_disk_mb
   154          provision_disk_id = self._create_provisioning_disk(node_id, provision_disk_mb, password)
   155          coreos_disk_id = self._create_coreos_disk(node_id, coreos_disk_mb)
   156  
   157          # create the provision config
   158          provision_config_id = self._create_provision_profile(node_id, provision_disk_id, coreos_disk_id)
   159  
   160          # create the CoreOS config
   161          coreos_config_id = self._create_coreos_profile(node_id, coreos_disk_id)
   162  
   163          # install CoreOS
   164          self._install_coreos(node_id, provision_config_id, network, password)
   165  
   166          # boot in to coreos
   167          self.info('Booting into CoreOS configuration profile...')
   168          self.request('linode.reboot', params={'LinodeID': node_id, 'ConfigID': coreos_config_id})
   169  
   170          # append the linode to the created list
   171          self._created_linodes.append(network)
   172  
   173      def _configure_networking(self, node_id):
   174          self.info('Configuring network...')
   175  
   176          # add the private network
   177          self.request('linode.ip.addprivate', params={'LinodeID': node_id})
   178  
   179          # pull the network config
   180          ip_data = self.request('linode.ip.list', params={'LinodeID': node_id})
   181  
   182          network = {'public': None, 'private': None, 'gateway': None}
   183  
   184          for ip in ip_data:
   185              if ip.get('ISPUBLIC') == 1:
   186                  network['public'] = ip.get('IPADDRESS')
   187                  # the gateway is the public ip with the last octet set to 1
   188                  split_ip = str(network['public']).split('.')
   189                  split_ip[3] = '1'
   190                  network['gateway'] = string.join(split_ip, '.')
   191              else:
   192                  network['private'] = ip.get('IPADDRESS')
   193  
   194          if network.get('public') is None:
   195              raise RuntimeError('Public IP address could not be found.')
   196  
   197          if network.get('private') is None:
   198              raise RuntimeError('Private IP address could not be found.')
   199  
   200          self.success('Network configured!')
   201          self.success('    Public IP:  ' + str(network['public']))
   202          self.success('    Private IP: ' + str(network['private']))
   203          self.success('    Gateway:    ' + str(network['gateway']))
   204  
   205          return network
   206  
   207      def _create_provisioning_disk(self, node_id, size, root_password):
   208          self.info('Creating provisioning disk...')
   209  
   210          disk_id = self.request('linode.disk.createfromdistribution', params={
   211              'LinodeID': node_id,
   212              'Label': 'Provision',
   213              'DistributionID': 130,
   214              'Type': 'ext4',
   215              'Size': size,
   216              'rootPass': root_password
   217          }).get('DiskID')
   218  
   219          self.success('Created provisioning disk!')
   220  
   221          return disk_id
   222  
   223      def _create_coreos_disk(self, node_id, size):
   224          self.info('Creating CoreOS disk...')
   225  
   226          disk_id = self.request('linode.disk.create', params={
   227              'LinodeID': node_id,
   228              'Label': 'CoreOS',
   229              'Type': 'ext4',
   230              'Size': size
   231          }).get('DiskID')
   232  
   233          self.success('Created CoreOS disk!')
   234  
   235          return disk_id
   236  
   237      def _create_provision_profile(self, node_id, provision_disk_id, coreos_disk_id):
   238          self.info('Creating Provision configuration profile...')
   239  
   240          # create a disk the total hd size
   241          config_id = self.request('linode.config.create', params={
   242              'LinodeID': node_id,
   243              'KernelID': 138,
   244              'Label': 'Provision',
   245              'DiskList': str(provision_disk_id) + ',' + str(coreos_disk_id)
   246          }).get('ConfigID')
   247  
   248          self.success('Provision profile created!')
   249  
   250          return config_id
   251  
   252      def _create_coreos_profile(self, node_id, coreos_disk_id):
   253          self.info('Creating CoreOS configuration profile...')
   254  
   255          # create a disk the total hd size
   256          config_id = self.request('linode.config.create', params={
   257              'LinodeID': node_id,
   258              'KernelID': 213,
   259              'Label': 'CoreOS',
   260              'DiskList': str(coreos_disk_id)
   261          }).get('ConfigID')
   262  
   263          self.success('CoreOS profile created!')
   264  
   265          return config_id
   266  
   267      def _get_cloud_config(self):
   268          if self.cloud_config_text is None:
   269              self.cloud_config_text = self.cloud_config.read()
   270          return self.cloud_config_text
   271  
   272      def _install_coreos(self, node_id, provision_config_id, network, password):
   273          self.info('Installing CoreOS...')
   274  
   275          # boot in to the provision configuration
   276          self.info('Booting into Provision configuration profile...')
   277          self.request('linode.boot', params={'LinodeID': node_id, 'ConfigID': provision_config_id})
   278  
   279          # connect to the server via ssh
   280          ssh = paramiko.SSHClient()
   281          ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
   282  
   283          while True:
   284              try:
   285                  ssh.connect(str(network['public']), username='root', password=password, allow_agent=False, look_for_keys=False)
   286                  break
   287              except:
   288                  continue
   289  
   290          # copy the cloud config
   291          self.info('Pushing cloud config...')
   292          cloud_config_template = string.Template(self._get_cloud_config())
   293          cloud_config = cloud_config_template.safe_substitute(public_ipv4=network['public'], private_ipv4=network['private'], gateway=network['gateway'],
   294                                                               hostname=network['hostname'])
   295  
   296          sftp = ssh.open_sftp()
   297          sftp.open('cloud-config.yaml', 'w').write(cloud_config)
   298  
   299          self.info('Installing...')
   300  
   301          commands = [
   302              'wget https://raw.githubusercontent.com/coreos/init/master/bin/coreos-install -O $HOME/coreos-install',
   303              'chmod +x $HOME/coreos-install',
   304              '$HOME/coreos-install -d /dev/sdb -C ' + self.coreos_channel + ' -V ' + self.coreos_version + ' -c $HOME/cloud-config.yaml -t /dev/shm'
   305          ]
   306  
   307          for command in commands:
   308              stdin, stdout, stderr = ssh.exec_command(command)
   309              stdout.channel.recv_exit_status()
   310              print stdout.read()
   311  
   312          ssh.close()
   313  
   314  
   315  class ListDataCentersCommand(LinodeApiCommand):
   316      def run(self):
   317          data = self.request('avail.datacenters')
   318          column_format = "{:<4} {:}"
   319          linodeutils.log_success(column_format.format(*('ID', 'LOCATION')))
   320          for data_center in data:
   321              row = (
   322                  data_center.get('DATACENTERID'),
   323                  data_center.get('LOCATION')
   324              )
   325              linodeutils.log_minor_success(column_format.format(*row))
   326  
   327  
   328  class ListPlansCommand(LinodeApiCommand):
   329      def run(self):
   330          data = self.request('avail.linodeplans')
   331          column_format = "{:<4} {:<16} {:<8} {:<12} {:}"
   332          linodeutils.log_success(column_format.format(*('ID', 'LABEL', 'CORES', 'RAM', 'PRICE')))
   333          for plan in data:
   334              row = (
   335                  plan.get('PLANID'),
   336                  plan.get('LABEL'),
   337                  plan.get('CORES'),
   338                  str(plan.get('RAM')) + 'MB',
   339                  '$' + str(plan.get('PRICE'))
   340              )
   341              linodeutils.log_minor_success(column_format.format(*row))
   342  
   343  
   344  def main():
   345      linodeutils.init()
   346  
   347      parser = argparse.ArgumentParser(description='Provision Linode Deis Cluster')
   348      parser.add_argument('--api-key', required=True, dest='linode_api_key', help='Linode API Key')
   349      subparsers = parser.add_subparsers()
   350  
   351      provision_parser = subparsers.add_parser('provision', help="Provision the Deis cluster")
   352      provision_parser.add_argument('--num', required=False, default=3, type=int, dest='num_nodes', help='Number of nodes to provision')
   353      provision_parser.add_argument('--name-prefix', required=False, default='deis', dest='node_name_prefix', help='Node name prefix')
   354      provision_parser.add_argument('--display-group', required=False, default='deis', dest='node_display_group', help='Node display group')
   355      provision_parser.add_argument('--plan', required=False, default=4, type=int, dest='node_plan', help='Node plan id. Use list-plans to find the id.')
   356      provision_parser.add_argument('--datacenter', required=False, default=2, type=int, dest='node_data_center',
   357                                    help='Node data center id. Use list-data-centers to find the id.')
   358      provision_parser.add_argument('--cloud-config', required=False, default='linode-user-data.yaml', type=file, dest='cloud_config',
   359                                    help='CoreOS cloud config user-data file')
   360      provision_parser.add_argument('--coreos-version', required=False, default='1068.8.0', dest='coreos_version',
   361                                    help='CoreOS version number to install')
   362      provision_parser.add_argument('--coreos-channel', required=False, default='stable', dest='coreos_channel',
   363                                    help='CoreOS channel to install from')
   364      provision_parser.set_defaults(cmd=ProvisionCommand)
   365  
   366      list_data_centers_parser = subparsers.add_parser('list-data-centers', help="Lists the available Linode data centers.")
   367      list_data_centers_parser.set_defaults(cmd=ListDataCentersCommand)
   368  
   369      list_plans_parser = subparsers.add_parser('list-plans', help="Lists the available Linode plans.")
   370      list_plans_parser.set_defaults(cmd=ListPlansCommand)
   371  
   372      args = parser.parse_args()
   373      args.cmd(args).run()
   374  
   375  
   376  if __name__ == "__main__":
   377      try:
   378          main()
   379      except Exception as e:
   380          linodeutils.log_error(e.message)
   381          sys.exit(1)