github.com/rochacon/deis@v1.0.2-0.20150903015341-6839b592a1ff/contrib/linode/provision-linode-cluster.py (about)

     1  #!/usr/bin/env python
     2  """
     3  Provision a Deis cluster on Linode
     4  
     5  Usage: provision-linode-cluster.py
     6  """
     7  import argparse
     8  import random
     9  import string
    10  import threading
    11  from threading import Thread
    12  import sys
    13  
    14  import paramiko
    15  import requests
    16  import colorama
    17  from colorama import Fore, Style
    18  
    19  
    20  class LinodeApiCommand:
    21      def __init__(self, arguments):
    22          self._arguments = vars(arguments)
    23          self._linode_api_key = arguments.linode_api_key
    24  
    25      def __getattr__(self, name):
    26          return self._arguments.get(name)
    27  
    28      def request(self, action, **kwargs):
    29          kwargs['params'] = dict({'api_key': self._linode_api_key, 'api_action': action}.items() + kwargs.get('params', {}).items())
    30          response = requests.request('get', 'https://api.linode.com/api/', **kwargs)
    31  
    32          json = response.json()
    33          errors = json.get('ERRORARRAY', [])
    34          data = json.get('DATA')
    35  
    36          if len(errors) > 0:
    37              raise IOError(str(errors))
    38  
    39          return data
    40  
    41      def run(self):
    42          raise NotImplementedError
    43  
    44      def info(self, message):
    45          print(Fore.MAGENTA + threading.current_thread().name + ': ' + Fore.CYAN + message + Fore.RESET)
    46  
    47      def success(self, message):
    48          print(Fore.MAGENTA + threading.current_thread().name + ': ' + Fore.GREEN + message + Fore.RESET)
    49  
    50  
    51  class ProvisionCommand(LinodeApiCommand):
    52      _created_linodes = []
    53  
    54      def run(self):
    55          # validate arguments
    56          self._check_num_nodes()
    57          self._check_plan_size()
    58  
    59          # create the linodes
    60          self._create_linodes()
    61  
    62          # print the results
    63          self._report_created()
    64  
    65      def _report_created(self):
    66          # set up the report data
    67          rows = []
    68          ips = []
    69          data_center = self._get_data_center().get('ABBR')
    70          plan = self._get_plan().get('RAM')
    71  
    72          for linode in self._created_linodes:
    73              rows.append((
    74                  linode['hostname'],
    75                  linode['public'],
    76                  linode['private'],
    77                  linode['gateway'],
    78                  data_center,
    79                  plan
    80              ))
    81              ips.append(linode['public'])
    82  
    83          firewall_command = './apply-firewall.py --private-key /path/to/key/deis --hosts ' + string.join(ips, ' ')
    84  
    85          # set up the report constants
    86          divider = Style.BRIGHT + Fore.MAGENTA + ('=' * 109) + Fore.RESET + Style.RESET_ALL
    87          column_format = "  {:<20} {:<20} {:<20} {:<20} {:<12} {:>8}"
    88          formatted_header = column_format.format(*('HOSTNAME', 'PUBLIC IP', 'PRIVATE IP', 'GATEWAY', 'DC', 'PLAN'))
    89  
    90          # display the report
    91          print('')
    92          print(divider)
    93          print(divider)
    94          print('')
    95          print(Style.BRIGHT + Fore.LIGHTGREEN_EX + '  Successfully provisioned ' + str(self.num_nodes) + ' nodes!' + Fore.RESET + Style.RESET_ALL)
    96          print('')
    97          print(Style.BRIGHT + Fore.CYAN + formatted_header + Fore.RESET + Style.RESET_ALL)
    98          for row in rows:
    99              print(Fore.CYAN + column_format.format(*row) + Fore.RESET)
   100          print('')
   101          print('')
   102          print(Fore.LIGHTYELLOW_EX + '  Finish up your installation by securing your cluster with the following command:' + Fore.RESET)
   103          print('')
   104          print('  ' + firewall_command)
   105          print('')
   106          print(divider)
   107          print(divider)
   108          print('')
   109  
   110      def _get_plan(self):
   111          if self._plan is None:
   112              plans = self.request('avail.linodeplans', params={'PlanID': self.node_plan})
   113              if len(plans) != 1:
   114                  raise ValueError('The --plan specified is invalid. Use the `list-plans` subcommand to see valid ids.')
   115              self._plan = plans[0]
   116          return self._plan
   117  
   118      def _get_plan_id(self):
   119          return self._get_plan().get('PLANID')
   120  
   121      def _get_data_center(self):
   122          if self._data_center is None:
   123              data_centers = self.request('avail.datacenters')
   124              for data_center in data_centers:
   125                  if data_center.get('DATACENTERID') == self.node_data_center:
   126                      self._data_center = data_center
   127              if self._data_center is None:
   128                  raise ValueError('The --datacenter specified is invalid. Use the `list-data-centers` subcommand to see valid ids.')
   129          return self._data_center
   130  
   131      def _get_data_center_id(self):
   132          return self._get_data_center().get('DATACENTERID')
   133  
   134      def _check_plan_size(self):
   135          ram = self._get_plan().get('RAM')
   136          if ram < 4096:
   137              raise ValueError('Deis cluster members must have at least 4GB of memory. Please choose a plan with more memory.')
   138  
   139      def _check_num_nodes(self):
   140          if self.num_nodes < 1:
   141              raise ValueError('Must provision at least one node.')
   142          elif self.num_nodes < 3:
   143              print(Fore.YELLOW + 'A Deis cluster must have 3 or more nodes, only continue if you adding to a current cluster.' + Fore.RESET)
   144              print(Fore.YELLOW + 'Continue? (y/n)' + Fore.RESET)
   145              accept = None
   146              while True:
   147                  if accept == 'y':
   148                      return
   149                  elif accept == 'n':
   150                      raise StandardError('User canceled provisioning')
   151                  else:
   152                      accept = self._get_user_input('--> ').strip().lower()
   153  
   154      def _get_user_input(self, prompt):
   155          if sys.version_info[0] < 3:
   156              return raw_input(prompt)
   157          else:
   158              return input(prompt)
   159  
   160      def _create_linodes(self):
   161          threads = []
   162          for i in range(0, self.num_nodes):
   163              t = Thread(target=self._create_linode,
   164                         args=(self._get_plan_id(), self._get_data_center_id(), self.node_name_prefix, self.node_display_group))
   165              t.setDaemon(False)
   166              t.start()
   167  
   168              threads.append(t)
   169  
   170          for thread in threads:
   171              thread.join()
   172  
   173      def _create_linode(self, plan_id, data_center_id, name_prefix, display_group):
   174          self.info('Creating the Linode...')
   175  
   176          # create the linode
   177          node_id = self.request('linode.create', params={
   178              'DatacenterID': data_center_id,
   179              'PlanID': plan_id
   180          }).get('LinodeID')
   181  
   182          # update the configuration
   183          self.request('linode.update', params={
   184              'LinodeID': node_id,
   185              'Label': name_prefix + str(node_id),
   186              'lpm_displayGroup': display_group,
   187              'Alert_cpu_enabled': False,
   188              'Alert_diskio_enabled': False,
   189              'Alert_bwin_enabled': False,
   190              'Alert_bwout_enabled': False,
   191              'Alert_bwquota_enabled': False
   192          })
   193  
   194          self.success('Linode ' + str(node_id) + ' created!')
   195          hostname = name_prefix + str(node_id)
   196          threading.current_thread().name = hostname
   197  
   198          # configure the networking
   199          network = self._configure_networking(node_id)
   200          network['hostname'] = hostname
   201  
   202          # generate a password for the provisioning disk
   203          password = ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(24))
   204  
   205          # configure the disks
   206          total_hd = self.request('linode.list', params={'LinodeID': node_id})[0]['TOTALHD']
   207          provision_disk_mb = 600
   208          coreos_disk_mb = total_hd - provision_disk_mb
   209          provision_disk_id = self._create_provisioning_disk(node_id, provision_disk_mb, password)
   210          coreos_disk_id = self._create_coreos_disk(node_id, coreos_disk_mb)
   211  
   212          # create the provision config
   213          provision_config_id = self._create_provision_profile(node_id, provision_disk_id, coreos_disk_id)
   214  
   215          # create the CoreOS config
   216          coreos_config_id = self._create_coreos_profile(node_id, coreos_disk_id)
   217  
   218          # install CoreOS
   219          self._install_coreos(node_id, provision_config_id, network, password)
   220  
   221          # boot in to coreos
   222          self.info('Booting into CoreOS configuration profile...')
   223          self.request('linode.reboot', params={'LinodeID': node_id, 'ConfigID': coreos_config_id})
   224  
   225          # append the linode to the created list
   226          self._created_linodes.append(network)
   227  
   228      def _configure_networking(self, node_id):
   229          self.info('Configuring network...')
   230  
   231          # add the private network
   232          self.request('linode.ip.addprivate', params={'LinodeID': node_id})
   233  
   234          # pull the network config
   235          ip_data = self.request('linode.ip.list', params={'LinodeID': node_id})
   236  
   237          network = {'public': None, 'private': None, 'gateway': None}
   238  
   239          for ip in ip_data:
   240              if ip.get('ISPUBLIC') == 1:
   241                  network['public'] = ip.get('IPADDRESS')
   242                  # the gateway is the public ip with the last octet set to 1
   243                  split_ip = str(network['public']).split('.')
   244                  split_ip[3] = '1'
   245                  network['gateway'] = string.join(split_ip, '.')
   246              else:
   247                  network['private'] = ip.get('IPADDRESS')
   248  
   249          if network.get('public') is None:
   250              raise RuntimeError('Public IP address could not be found.')
   251  
   252          if network.get('private') is None:
   253              raise RuntimeError('Private IP address could not be found.')
   254  
   255          self.success('Network configured!')
   256          self.success('    Public IP:  ' + str(network['public']))
   257          self.success('    Private IP: ' + str(network['private']))
   258          self.success('    Gateway:    ' + str(network['gateway']))
   259  
   260          return network
   261  
   262      def _create_provisioning_disk(self, node_id, size, root_password):
   263          self.info('Creating provisioning disk...')
   264  
   265          disk_id = self.request('linode.disk.createfromdistribution', params={
   266              'LinodeID': node_id,
   267              'Label': 'Provision',
   268              'DistributionID': 130,
   269              'Type': 'ext4',
   270              'Size': size,
   271              'rootPass': root_password
   272          }).get('DiskID')
   273  
   274          self.success('Created provisioning disk!')
   275  
   276          return disk_id
   277  
   278      def _create_coreos_disk(self, node_id, size):
   279          self.info('Creating CoreOS disk...')
   280  
   281          disk_id = self.request('linode.disk.create', params={
   282              'LinodeID': node_id,
   283              'Label': 'CoreOS',
   284              'Type': 'ext4',
   285              'Size': size
   286          }).get('DiskID')
   287  
   288          self.success('Created CoreOS disk!')
   289  
   290          return disk_id
   291  
   292      def _create_provision_profile(self, node_id, provision_disk_id, coreos_disk_id):
   293          self.info('Creating Provision configuration profile...')
   294  
   295          # create a disk the total hd size
   296          config_id = self.request('linode.config.create', params={
   297              'LinodeID': node_id,
   298              'KernelID': 138,
   299              'Label': 'Provision',
   300              'DiskList': str(provision_disk_id) + ',' + str(coreos_disk_id)
   301          }).get('ConfigID')
   302  
   303          self.success('Provision profile created!')
   304  
   305          return config_id
   306  
   307      def _create_coreos_profile(self, node_id, coreos_disk_id):
   308          self.info('Creating CoreOS configuration profile...')
   309  
   310          # create a disk the total hd size
   311          config_id = self.request('linode.config.create', params={
   312              'LinodeID': node_id,
   313              'KernelID': 213,
   314              'Label': 'CoreOS',
   315              'DiskList': str(coreos_disk_id)
   316          }).get('ConfigID')
   317  
   318          self.success('CoreOS profile created!')
   319  
   320          return config_id
   321  
   322      def _get_cloud_config(self):
   323          if self.cloud_config_text is None:
   324              self.cloud_config_text = self.cloud_config.read()
   325          return self.cloud_config_text
   326  
   327      def _install_coreos(self, node_id, provision_config_id, network, password):
   328          self.info('Installing CoreOS...')
   329  
   330          # boot in to the provision configuration
   331          self.info('Booting into Provision configuration profile...')
   332          self.request('linode.boot', params={'LinodeID': node_id, 'ConfigID': provision_config_id})
   333  
   334          # connect to the server via ssh
   335          ssh = paramiko.SSHClient()
   336          ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
   337  
   338          while True:
   339              try:
   340                  ssh.connect(str(network['public']), username='root', password=password, allow_agent=False, look_for_keys=False)
   341                  break
   342              except:
   343                  continue
   344  
   345          # copy the cloud config
   346          self.info('Pushing cloud config...')
   347          cloud_config_template = string.Template(self._get_cloud_config())
   348          cloud_config = cloud_config_template.safe_substitute(public_ipv4=network['public'], private_ipv4=network['private'], gateway=network['gateway'],
   349                                                               hostname=network['hostname'])
   350  
   351          sftp = ssh.open_sftp()
   352          sftp.open('cloud-config.yaml', 'w').write(cloud_config)
   353  
   354          self.info('Installing...')
   355  
   356          commands = [
   357              'wget https://raw.githubusercontent.com/coreos/init/master/bin/coreos-install -O $HOME/coreos-install',
   358              'chmod +x $HOME/coreos-install',
   359              '$HOME/coreos-install -d /dev/sdb -C ' + self.coreos_channel + ' -V ' + self.coreos_version + ' -c $HOME/cloud-config.yaml -t /dev/shm'
   360          ]
   361  
   362          for command in commands:
   363              stdin, stdout, stderr = ssh.exec_command(command)
   364              stdout.channel.recv_exit_status()
   365              print stdout.read()
   366  
   367          ssh.close()
   368  
   369  
   370  class ListDataCentersCommand(LinodeApiCommand):
   371      def run(self):
   372          data = self.request('avail.datacenters')
   373          column_format = "{:<4} {:}"
   374          print(Style.BRIGHT + Fore.GREEN + column_format.format(*('ID', 'LOCATION')) + Fore.RESET + Style.RESET_ALL)
   375          for data_center in data:
   376              row = (
   377                  data_center.get('DATACENTERID'),
   378                  data_center.get('LOCATION')
   379              )
   380              print(Fore.GREEN + column_format.format(*row) + Fore.RESET)
   381  
   382  
   383  class ListPlansCommand(LinodeApiCommand):
   384      def run(self):
   385          data = self.request('avail.linodeplans')
   386          column_format = "{:<4} {:<16} {:<8} {:<12} {:}"
   387          print(Style.BRIGHT + Fore.GREEN + column_format.format(
   388              *('ID', 'LABEL', 'CORES', 'RAM', 'PRICE')) + Fore.RESET + Style.RESET_ALL)
   389          for plan in data:
   390              row = (
   391                  plan.get('PLANID'),
   392                  plan.get('LABEL'),
   393                  plan.get('CORES'),
   394                  str(plan.get('RAM')) + 'MB',
   395                  '$' + str(plan.get('PRICE'))
   396              )
   397              print(Fore.GREEN + column_format.format(*row) + Fore.RESET)
   398  
   399  
   400  if __name__ == '__main__':
   401      colorama.init()
   402  
   403      parser = argparse.ArgumentParser(description='Provision Linode Deis Cluster')
   404      parser.add_argument('--api-key', required=True, dest='linode_api_key', help='Linode API Key')
   405      subparsers = parser.add_subparsers()
   406  
   407      provision_parser = subparsers.add_parser('provision', help="Provision the Deis cluster")
   408      provision_parser.add_argument('--num', required=False, default=3, type=int, dest='num_nodes', help='Number of nodes to provision')
   409      provision_parser.add_argument('--name-prefix', required=False, default='deis', dest='node_name_prefix', help='Node name prefix')
   410      provision_parser.add_argument('--display-group', required=False, default='deis', dest='node_display_group', help='Node display group')
   411      provision_parser.add_argument('--plan', required=False, default=4, type=int, dest='node_plan', help='Node plan id. Use list-plans to find the id.')
   412      provision_parser.add_argument('--datacenter', required=False, default=2, type=int, dest='node_data_center',
   413                                    help='Node data center id. Use list-data-centers to find the id.')
   414      provision_parser.add_argument('--cloud-config', required=False, default='linode-user-data.yaml', type=file, dest='cloud_config',
   415                                    help='CoreOS cloud config user-data file')
   416      provision_parser.add_argument('--coreos-version', required=False, default='647.2.0', dest='coreos_version',
   417                                    help='CoreOS version number to install')
   418      provision_parser.add_argument('--coreos-channel', required=False, default='stable', dest='coreos_channel',
   419                                    help='CoreOS channel to install from')
   420      provision_parser.set_defaults(cmd=ProvisionCommand)
   421  
   422      list_data_centers_parser = subparsers.add_parser('list-data-centers', help="Lists the available Linode data centers.")
   423      list_data_centers_parser.set_defaults(cmd=ListDataCentersCommand)
   424  
   425      list_plans_parser = subparsers.add_parser('list-plans', help="Lists the available Linode plans.")
   426      list_plans_parser.set_defaults(cmd=ListPlansCommand)
   427  
   428      args = parser.parse_args()
   429      cmd = args.cmd(args)
   430  
   431      try:
   432          cmd.run()
   433      except Exception as e:
   434          print(Style.BRIGHT + Fore.RED + e.message + Fore.RESET + Style.RESET_ALL)
   435          sys.exit(1)