github.com/arkadijs/deis@v1.5.1/contrib/azure/azure-coreos-cluster (about)

     1  #!/usr/bin/env python
     2  
     3  from azure import *
     4  from azure.servicemanagement import *
     5  import argparse
     6  import urllib2
     7  import time
     8  import base64
     9  import os
    10  import subprocess
    11  
    12  parser = argparse.ArgumentParser(description='Create a CoreOS cluster on Microsoft Azure.')
    13  parser.add_argument('--version', action='version', version='azure-coreos-cluster 0.1')
    14  parser.add_argument('cloud_service_name',
    15                     help='cloud service name')
    16  parser.add_argument('--ssh-cert',
    17                     help='certificate file with public key for ssh, in .cer format')
    18  parser.add_argument('--ssh-thumb',
    19                     help='thumbprint of ssh cert')
    20  parser.add_argument('--subscription', required=True,
    21                     help='required Azure subscription id')
    22  parser.add_argument('--azure-cert', required=True,
    23                     help='required path to Azure cert pem file')
    24  parser.add_argument('--blob-container-url', required=True,
    25                     help='required url to blob container where vm disk images will be created, including /, ex: https://patcoreos.blob.core.windows.net/vhds/')
    26  parser.add_argument('--vm-size', default='Small',
    27                     help='optional, VM size [Small]')
    28  parser.add_argument('--vm-name-prefix', default='coreos',
    29                     help='optional, VM name prefix [coreos]')
    30  parser.add_argument('--availability-set', default='coreos-as',
    31                     help='optional, name of availability set for cluster [coreos-as]')
    32  parser.add_argument('--location', default='West US',
    33                     help='optional - overriden by affinity-group, [West US]')
    34  parser.add_argument('--affinity-group', default='',
    35                     help='optional, overrides location if specified')
    36  parser.add_argument('--ssh', default=22001, type=int,
    37                     help='optional, starts with 22001 and +1 for each machine in cluster')
    38  parser.add_argument('--coreos-image', default='2b171e93f07c4903bcad35bda10acf22__CoreOS-Stable-607.0.0',
    39                     help='optional, [2b171e93f07c4903bcad35bda10acf22__CoreOS-Stable-607.0.0]')
    40  parser.add_argument('--num-nodes', default=3, type=int,
    41                     help='optional, number of nodes to create (or add), defaults to 3')
    42  parser.add_argument('--virtual-network-name',
    43                     help='optional, name of an existing virtual network to which we will add the VMs')
    44  parser.add_argument('--subnet-names',
    45                     help='optional, subnet name to which the VMs will belong')
    46  parser.add_argument('--custom-data',
    47                     help='optional, path to your own cloud-init file')
    48  parser.add_argument('--discovery-service-url',
    49                     help='optional, url for an existing cluster discovery service. Else we will generate one.')
    50  parser.add_argument('--pip', action='store_true',
    51                     help='optional, assigns public instance ip addresses to each VM')
    52  parser.add_argument('--deis', action='store_true',
    53                     help='optional, automatically opens http and controller endpoints')
    54  parser.add_argument('--data-disk', action='store_true',
    55                     help='optional, attaches a data disk to each VM')
    56  
    57  cloud_init_template = """#cloud-config
    58  
    59  coreos:
    60    etcd:
    61      # generate a new token for each unique cluster from https://discovery.etcd.io/new
    62      discovery: {0}
    63      # deployments across multiple cloud services will need to use $public_ipv4
    64      addr: $private_ipv4:4001
    65      peer-addr: $private_ipv4:7001
    66    units:
    67      - name: etcd.service
    68        command: start
    69      - name: fleet.service
    70        command: start
    71  """
    72  
    73  args = parser.parse_args()
    74  
    75  # Create SSH cert if it's not given
    76  if not args.ssh_cert and not args.ssh_thumb:
    77    print 'SSH arguments not given, generating certificate'
    78    with open(os.devnull, 'w') as shutup:
    79        subprocess.call('openssl req -x509 -nodes -days 365 -newkey rsa:2048 -config cert.conf -keyout ssh-cert.key -out ssh-cert.pem', shell=True, stdout=shutup, stderr=shutup)
    80        subprocess.call('chmod 600 ssh-cert.key', shell=True, stdout=shutup, stderr=shutup)
    81        subprocess.call('openssl  x509 -outform der -in ssh-cert.pem -out ssh-cert.cer', shell=True, stdout=shutup, stderr=shutup)
    82        thumbprint = subprocess.check_output('openssl x509 -in ssh-cert.pem -sha1 -noout -fingerprint | sed s/://g', shell=True)
    83        args.ssh_thumb = thumbprint.split('=')[1].replace('\n', '')
    84        args.ssh_cert = './ssh-cert.cer'
    85    print 'Generated SSH certificate with thumbprint ' + args.ssh_thumb
    86  
    87  # Setup custom data
    88  if args.custom_data:
    89      with open(args.custom_data, 'r') as f:
    90        if not os.path.exists(args.custom_data):
    91          print "Couldn't find the user-data file. Did you remember to run `create-azure-user-data`?"
    92          sys.exit(1)
    93        cloud_init = f.read()
    94      f.closed
    95  else:
    96      if args.discovery_service_url:
    97          cloud_init = cloud_init_template.format(args.discovery_service_url)
    98      else:
    99          response = urllib2.urlopen('https://discovery.etcd.io/new')
   100          discovery_url = response.read()
   101          cloud_init = cloud_init_template.format(discovery_url)
   102  
   103  SERVICE_CERT_FORMAT = 'pfx'
   104  
   105  with open(args.ssh_cert) as f:
   106      service_cert_file_data = base64.b64encode(f.read())
   107  f.closed
   108  
   109  def wait_for_async(request_id, timeout):
   110      count = 0
   111      result = sms.get_operation_status(request_id)
   112      while result.status == 'InProgress':
   113          count = count + 1
   114          if count > timeout:
   115              print('Timed out waiting for async operation to complete.')
   116              return
   117          time.sleep(5)
   118          print('.'),
   119          sys.stdout.flush()
   120          result = sms.get_operation_status(request_id)
   121          if result.error:
   122              print(result.error.code)
   123              print(vars(result.error))
   124      print result.status + ' in ' + str(count*5) + 's'
   125  
   126  def linux_config(hostname, args):
   127      pk = PublicKey(args.ssh_thumb,
   128                     u'/home/core/.ssh/authorized_keys')
   129      system = LinuxConfigurationSet(hostname, 'core', None, True,
   130                custom_data=cloud_init)
   131      system.ssh.public_keys.public_keys.append(pk)
   132      system.disable_ssh_password_authentication = True
   133      return system
   134  
   135  def endpoint_config(name, port, probe=False):
   136      endpoint = ConfigurationSetInputEndpoint(name, 'tcp', port, port, name)
   137      if probe:
   138        endpoint.load_balancer_probe = probe
   139      return endpoint
   140  
   141  def load_balancer_probe(path, port, protocol):
   142      load_balancer_probe = LoadBalancerProbe()
   143      load_balancer_probe.path = path
   144      load_balancer_probe.port = port
   145      load_balancer_probe.protocol = protocol
   146      return load_balancer_probe
   147  
   148  def network_config(subnet_name=None, port='59913', public_ip_name=None):
   149      network = ConfigurationSet()
   150      network.configuration_set_type = 'NetworkConfiguration'
   151      network.input_endpoints.input_endpoints.append(
   152          ConfigurationSetInputEndpoint('ssh', 'tcp', port, '22'))
   153      if subnet_name:
   154          network.subnet_names.append(subnet_name)
   155      if public_ip_name:
   156          ip = PublicIP(name=public_ip_name)
   157          ip.idle_timeout_in_minutes = 20
   158          network.public_ips.public_ips.append(ip)
   159      if args.deis:
   160          # create web endpoint with probe checking /health-check
   161          network.input_endpoints.input_endpoints.append(endpoint_config('web', '80', load_balancer_probe('/health-check', '80', 'http')))
   162          # create builder endpoint TCP probe check
   163          network.input_endpoints.input_endpoints.append(endpoint_config('builder', '2222', load_balancer_probe(None, '2222', 'tcp')))
   164      return network
   165  
   166  def data_hd(target_container_url, target_blob_name, target_lun, target_disk_size_in_gb):
   167      media_link = target_container_url + target_blob_name
   168      data_hd = DataVirtualHardDisk()
   169      data_hd.disk_label = target_blob_name
   170      data_hd.logical_disk_size_in_gb = target_disk_size_in_gb
   171      data_hd.lun = target_lun
   172      data_hd.media_link = media_link
   173      return data_hd
   174  
   175  sms = ServiceManagementService(args.subscription, args.azure_cert)
   176  
   177  #Create the cloud service
   178  try:
   179    print 'Creating the hosted service...',
   180    sys.stdout.flush()
   181    if args.affinity_group:
   182      sms.create_hosted_service(
   183          args.cloud_service_name, label=args.cloud_service_name, affinity_group=args.affinity_group)
   184    else:
   185      sms.create_hosted_service(
   186          args.cloud_service_name, label=args.cloud_service_name, location=args.location)
   187    print('Successfully created hosted service ' + args.cloud_service_name)
   188    sys.stdout.flush()
   189    time.sleep(2)
   190  except WindowsAzureConflictError:
   191    print "Hosted service {} already exists. Delete it or try again with a different name.".format(args.cloud_service_name)
   192    sys.exit(1)
   193  
   194  #upload ssh cert to cloud-service
   195  print 'Uploading SSH certificate...',
   196  sys.stdout.flush()
   197  result = sms.add_service_certificate(args.cloud_service_name,
   198                                       service_cert_file_data, SERVICE_CERT_FORMAT, '')
   199  wait_for_async(result.request_id, 15)
   200  
   201  def get_vm_name(args, i):
   202      return args.cloud_service_name + '-' + args.vm_name_prefix + '-' + str(i)
   203  
   204  vms =[]
   205  
   206  #Create the VMs
   207  for i in range(args.num_nodes):
   208      ssh_port = args.ssh +i
   209      vm_name = get_vm_name(args, i)
   210      if args.pip:
   211          pip_name = vm_name
   212      else:
   213          pip_name = None
   214      media_link = args.blob_container_url + vm_name
   215      os_hd = OSVirtualHardDisk(media_link=media_link,
   216                              source_image_name=args.coreos_image)
   217      system = linux_config(vm_name, args)
   218      network = network_config(subnet_name=args.subnet_names, port=ssh_port, public_ip_name=pip_name)
   219      #specifiy the data disk, important to start at lun = 0
   220      if args.data_disk:
   221          data_disk = data_hd(args.blob_container_url, vm_name + '-data.vhd', 0, 100)
   222          data_disks = DataVirtualHardDisks()
   223          data_disks.data_virtual_hard_disks.append(data_disk)
   224      else:
   225          data_disks = None
   226  
   227      try:
   228        if i == 0:
   229            result = sms.create_virtual_machine_deployment(
   230                        args.cloud_service_name, deployment_name=args.cloud_service_name,
   231                        deployment_slot='production', label=vm_name,
   232                        role_name=vm_name, system_config=system, os_virtual_hard_disk=os_hd, virtual_network_name=args.virtual_network_name,
   233                        role_size=args.vm_size, network_config=network, data_virtual_hard_disks=data_disks)
   234        else:
   235            result = sms.add_role(
   236                        args.cloud_service_name, deployment_name=args.cloud_service_name,
   237                        role_name=vm_name,
   238                        system_config=system, os_virtual_hard_disk=os_hd,
   239                        role_size=args.vm_size, network_config=network, data_virtual_hard_disks=data_disks)
   240      except WindowsAzureError as e:
   241        if "Forbidden" in str(e):
   242          print "Unable to use this CoreOS image. This usually means a newer image has been published."
   243          print "See https://coreos.com/docs/running-coreos/cloud-providers/azure/ for the latest stable image,"
   244          print "and supply it to this script with --coreos-image. If it works, please open a pull request to update this script."
   245          sys.exit(1)
   246        else:
   247          pass
   248  
   249      print 'Creating VM ' + vm_name + '...',
   250      sys.stdout.flush()
   251      wait_for_async(result.request_id, 30)
   252      vms.append({'name':vm_name,
   253                  'host':args.cloud_service_name + '.cloudapp.net',
   254                  'port':ssh_port,
   255                  'user':'core',
   256                  'identity':args.ssh_cert.replace('.cer','.key')})
   257  
   258  #get the ip addresses
   259  def get_ips(service_name, deployment_name):
   260      result = sms.get_deployment_by_name(service_name, deployment_name)
   261      for instance in result.role_instance_list:
   262          ips.append(instance.public_ips[0].address)
   263      return ips
   264  
   265  #print dns config
   266  if args.pip:
   267      ips = []
   268      ips = get_ips(args.cloud_service_name, args.cloud_service_name)
   269      print ''
   270      print '-------'
   271      print "You'll need to configure DNS records for a domain you wish to use with your Deis cluster."
   272      print 'For convenience, the public IP addresses are printed below, along with sane DNS timeouts.'
   273      print ''
   274      for ip in ips:
   275          print '@ 10800 IN A ' + ip
   276      print '* 10800 IN CNAME @'
   277      print '-------'
   278      print 'For more information, see: http://docs.deis.io/en/latest/managing_deis/configure-dns/'
   279      print ''
   280  
   281  #print ~/.ssh/config
   282  print ''
   283  print '-------'
   284  print "Instances on Azure don't use typical SSH ports. It is recommended to configure ~/.ssh/config"
   285  print 'so the instances can easily be referenced when logging in via SSH. For convenience, the config'
   286  print 'directives for your instances are below:'
   287  print ''
   288  for vm in vms:
   289      print 'Host ' + vm['name']
   290      print '    HostName ' + vm['host']
   291      print '    Port ' + str(vm['port'])
   292      print '    User ' + vm['user']
   293      print '    IdentityFile ' + vm['identity']
   294  print '-------'
   295  print ''