github.com/niedbalski/juju@v0.0.0-20190215020005-8ff100488e47/acceptancetests/joyent.py (about)

     1  #!/usr/bin/python
     2  
     3  from __future__ import print_function
     4  
     5  from argparse import ArgumentParser
     6  from datetime import (
     7      datetime,
     8      timedelta,
     9  )
    10  import json
    11  import os
    12  import pprint
    13  import subprocess
    14  import sys
    15  from time import sleep
    16  import urllib2
    17  
    18  from utility import until_timeout
    19  
    20  
    21  VERSION = '0.1.0'
    22  USER_AGENT = "juju-cloud-tool/{} ({}) Python/{}".format(
    23      VERSION, sys.platform, sys.version.split(None, 1)[0])
    24  ISO_8601_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ'
    25  
    26  
    27  SSL_SIGN = """
    28  echo -n "date:" {0} |
    29      openssl dgst -sha256 -sign {1} |
    30      openssl enc -e -a |
    31      tr -d '\n'
    32  """
    33  
    34  OLD_MACHINE_AGE = 12
    35  
    36  
    37  class DeleteRequest(urllib2.Request):
    38  
    39      def get_method(self):
    40          return "DELETE"
    41  
    42  
    43  class HeadRequest(urllib2.Request):
    44  
    45      def get_method(self):
    46          return "HEAD"
    47  
    48  
    49  class PostRequest(urllib2.Request):
    50  
    51      def get_method(self):
    52          return "POST"
    53  
    54  
    55  class PutRequest(urllib2.Request):
    56  
    57      def get_method(self):
    58          return "PUT"
    59  
    60  
    61  def parse_iso_date(string):
    62      return datetime.strptime(string, ISO_8601_FORMAT)
    63  
    64  
    65  class Client:
    66      """A class that mirrors MantaClient without the modern Crypto.
    67  
    68      See https://github.com/joyent/python-manta
    69      """
    70  
    71      def __init__(self, sdc_url, account, key_id, key_path, manta_url,
    72                   user_agent=USER_AGENT, pause=3, dry_run=False, verbose=False):
    73          if sdc_url.endswith('/'):
    74              sdc_url = sdc_url[1:]
    75          self.sdc_url = sdc_url
    76          if manta_url.endswith('/'):
    77              manta_url = manta_url[1:]
    78          self.manta_url = manta_url
    79          self.account = account
    80          self.key_id = key_id
    81          self.key_path = key_path
    82          self.user_agent = user_agent
    83          self.pause = pause
    84          self.dry_run = dry_run
    85          self.verbose = verbose
    86  
    87      def make_request_headers(self, headers=None):
    88          """Return a dict of required headers.
    89  
    90          The Authorization header is always a signing of the "Date" header,
    91          where "date" must be lowercase.
    92          """
    93          timestamp = datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S GMT")
    94          script = SSL_SIGN.format(timestamp, self.key_path)
    95          signature = subprocess.check_output(['bash', '-c', script])
    96          key = "/{}/keys/{}".format(self.account, self.key_id)
    97          auth = (
    98              'Signature keyId="{}",algorithm="rsa-sha256",'.format(key) +
    99              'signature="{}"'.format(signature))
   100          if headers is None:
   101              headers = {}
   102          headers['Date'] = timestamp
   103          headers['Authorization'] = auth
   104          headers["User-Agent"] = USER_AGENT
   105          return headers
   106  
   107      def _request(self, path, method="GET", body=None, headers=None,
   108                   is_manta=False):
   109          headers = self.make_request_headers(headers)
   110          if path.startswith('/'):
   111              path = path[1:]
   112          if is_manta:
   113              base_url = self.manta_url
   114          else:
   115              base_url = self.sdc_url
   116          uri = "{}/{}/{}".format(base_url, self.account, path)
   117          if method == 'DELETE':
   118              request = DeleteRequest(uri, headers=headers)
   119          elif method == 'HEAD':
   120              request = HeadRequest(uri, headers=headers)
   121          elif method == 'POST':
   122              request = PostRequest(uri, data=body, headers=headers)
   123          elif method == 'PUT':
   124              request = PutRequest(uri, data=body, headers=headers)
   125          else:
   126              request = urllib2.Request(uri, headers=headers)
   127          try:
   128              response = urllib2.urlopen(request)
   129          except Exception as err:
   130              print(request.header_items())
   131              print(err.read())
   132              raise
   133          content = response.read()
   134          headers = dict(response.headers.items())
   135          headers['status'] = str(response.getcode())
   136          headers['reason'] = response.msg
   137          return headers, content
   138  
   139      def _list_objects(self, path, deep=False):
   140          headers, content = self._request(path, is_manta=True)
   141          objects = []
   142          for line in content.splitlines():
   143              obj = json.loads(line)
   144              obj['path'] = '%s/%s' % (path, obj['name'])
   145              objects.append(obj)
   146              if obj['type'] == 'directory' and deep:
   147                  objects.extend(self._list_objects(obj['path'], deep=True))
   148          return objects
   149  
   150      def list_objects(self, path, deep=False):
   151          objects = self._list_objects(path, deep=deep)
   152          for obj in objects:
   153              print('{type:9} {mtime} {path}'.format(**obj))
   154  
   155      def delete_old_objects(self, path, old_age):
   156          now = datetime.utcnow()
   157          ago = timedelta(hours=old_age)
   158          objects = self._list_objects(path, deep=True)
   159          # The list is dir, the sub objects. Manta requires the sub objects
   160          # to be deleted first.
   161          objects.reverse()
   162          for obj in objects:
   163              if '.joyent' in obj['path']:
   164                  # The .joyent dir cannot be deleted.
   165                  print('ignoring %s' % obj['path'])
   166                  continue
   167              mtime = parse_iso_date(obj['mtime'])
   168              age = now - mtime
   169              if age < ago:
   170                  print('ignoring young %s' % obj['path'])
   171                  continue
   172              if self.verbose:
   173                  print('Deleting %s' % obj['path'])
   174              if not self.dry_run:
   175                  headers, content = self._request(
   176                      obj['path'], method='DELETE', is_manta=True)
   177  
   178      def _list_machines(self, machine_id=None):
   179          """Return a list of machine dicts."""
   180          if machine_id:
   181              path = '/machines/{}'.format(machine_id)
   182          else:
   183              path = '/machines'
   184          headers, content = self._request(path)
   185          machines = json.loads(content)
   186          if self.verbose:
   187              print(machines)
   188          return machines
   189  
   190      def list_machines(self, machine_id=None):
   191          machines = self._list_machines(machine_id)
   192          pprint.pprint(machines, indent=2)
   193  
   194      def _list_machine_tags(self, machine_id):
   195          path = '/machines/{}/tags'.format(machine_id)
   196          headers, content = self._request(path)
   197          tags = json.loads(content)
   198          if self.verbose:
   199              print(tags)
   200          return tags
   201  
   202      def list_machine_tags(self, machine_id):
   203          tags = self._list_machine_tags(machine_id)
   204          pprint.pprint(tags, indent=2)
   205  
   206      def stop_machine(self, machine_id):
   207          path = '/machines/{}?action=stop'.format(machine_id)
   208          print("Stopping machine {}".format(machine_id))
   209          if not self.dry_run:
   210              headers, content = self._request(path, method='POST')
   211  
   212      def delete_machine(self, machine_id):
   213          path = '/machines/{}'.format(machine_id)
   214          print("Deleting machine {}".format(machine_id))
   215          if not self.dry_run:
   216              headers, content = self._request(path, method='DELETE')
   217  
   218      def attempt_deletion(self, current_stuck):
   219          all_success = True
   220          for machine_id in current_stuck:
   221              if self.verbose:
   222                  print("Attempting to delete {} stuck in provisioning.".format(
   223                        machine_id))
   224              if not self.dry_run:
   225                  try:
   226                      # Officially the we cannot delete non-stopped machines,
   227                      # but using the UI, we can delete machines stuck in
   228                      # provisioning or stopping, so we try.
   229                      self.delete_machine(machine_id)
   230                      if self.verbose:
   231                          print("Deleted {}".format(machine_id))
   232                  except:
   233                      print('Delete stuck machine {} using the UI.'.format(
   234                            machine_id))
   235                      all_success = False
   236          return all_success
   237  
   238      def _delete_running_machine(self, machine_id):
   239          self.stop_machine(machine_id)
   240          for ignored in until_timeout(120):
   241              if self.verbose:
   242                  print(".", end="")
   243                  sys.stdout.flush()
   244              sleep(self.pause)
   245              stopping_machine = self._list_machines(machine_id)
   246              if stopping_machine['state'] == 'stopped':
   247                  break
   248          if self.verbose:
   249              print("stopped")
   250          self.delete_machine(machine_id)
   251  
   252      def delete_old_machines(self, old_age):
   253          machines = self._list_machines()
   254          now = datetime.utcnow()
   255          current_stuck = []
   256          for machine in machines:
   257              created = parse_iso_date(machine['created'])
   258              age = now - created
   259              if age > timedelta(hours=old_age):
   260                  machine_id = machine['id']
   261                  tags = self._list_machine_tags(machine_id)
   262                  if tags.get('permanent', 'false') == 'true':
   263                      continue
   264                  if machine['state'] == 'provisioning':
   265                      current_stuck.append(machine)
   266                      continue
   267                  if self.verbose:
   268                      print("Machine {} is {} old".format(machine_id, age))
   269                  if not self.dry_run:
   270                      self._delete_running_machine(machine_id)
   271          if not self.dry_run and current_stuck:
   272              self.attempt_deletion(current_stuck)
   273  
   274  
   275  def parse_args(argv=None):
   276      """Return the argument parser for this program."""
   277      parser = ArgumentParser('Query and manage joyent.')
   278      parser.add_argument(
   279          '-d', '--dry-run', action='store_true', default=False,
   280          help='Do not make changes.')
   281      parser.add_argument(
   282          '-v', '--verbose', action="store_true", help='Increse verbosity.')
   283      parser.add_argument(
   284          "-u", "--url", dest="sdc_url",
   285          help="SDC URL. Environment: SDC_URL=URL",
   286          default=os.environ.get("SDC_URL"))
   287      parser.add_argument(
   288          "-m", "--manta-url", dest="manta_url",
   289          help="Manta URL. Environment: MANTA_URL=URL",
   290          default=os.environ.get("MANTA_URL"))
   291      parser.add_argument(
   292          "-a", "--account",
   293          help="Manta account. Environment: MANTA_USER=ACCOUNT",
   294          default=os.environ.get("MANTA_USER"))
   295      parser.add_argument(
   296          "-k", "--key-id", dest="key_id",
   297          help="SSH key fingerprint.  Environment: MANTA_KEY_ID=FINGERPRINT",
   298          default=os.environ.get("MANTA_KEY_ID"))
   299      parser.add_argument(
   300          "-p", "--key-path", dest="key_path",
   301          help="Path to the SSH key",
   302          default=os.path.join(os.environ.get('JUJU_HOME', '~/.juju'), 'id_rsa'))
   303      subparsers = parser.add_subparsers(help='sub-command help', dest="command")
   304      subparsers.add_parser('list-machines', help='List running machines')
   305      parser_delete_old_machine = subparsers.add_parser(
   306          'delete-old-machines',
   307          help='Delete machines older than %d hours' % OLD_MACHINE_AGE)
   308      parser_delete_old_machine.add_argument(
   309          '-o', '--old-age', default=OLD_MACHINE_AGE, type=int,
   310          help='Set old machine age to n hours.')
   311      parser_list_tags = subparsers.add_parser(
   312          'list-tags', help='List tags of running machines')
   313      parser_list_tags.add_argument('machine_id', help='The machine id.')
   314      parser_list_objects = subparsers.add_parser(
   315          'list-objects', help='List directories and files in manta')
   316      parser_list_objects.add_argument(
   317          '-r', '--recursive', action='store_true', default=False,
   318          help='Include content in subdirectories.')
   319      parser_list_objects.add_argument('path', help='The path')
   320      parser_delete_old_objects = subparsers.add_parser(
   321          'delete-old-objects',
   322          help='Delete objects older than %d hours' % OLD_MACHINE_AGE)
   323      parser_delete_old_objects.add_argument(
   324          '-o', '--old-age', default=OLD_MACHINE_AGE, type=int,
   325          help='Set old object age to n hours.')
   326      parser_delete_old_objects.add_argument('path', help='The path')
   327  
   328      args = parser.parse_args(argv)
   329      if not args.sdc_url:
   330          print('SDC_URL must be sourced into the environment.')
   331          sys.exit(1)
   332      return args
   333  
   334  
   335  def main(argv):
   336      args = parse_args(argv)
   337      client = Client(
   338          args.sdc_url, args.account, args.key_id, args.key_path, args.manta_url,
   339          dry_run=args.dry_run, verbose=args.verbose)
   340      if args.command == 'list-machines':
   341          client.list_machines()
   342      elif args.command == 'list-tags':
   343          client.list_machine_tags(args.machine_id)
   344      elif args.command == 'list-objects':
   345          client.list_objects(args.path, deep=args.recursive)
   346      elif args.command == 'delete-old-machines':
   347          client.delete_old_machines(args.old_age)
   348      elif args.command == 'delete-old-objects':
   349          client.delete_old_objects(args.path, args.old_age)
   350      else:
   351          print("action not understood.")
   352  
   353  
   354  if __name__ == '__main__':
   355      sys.exit(main(sys.argv[1:]))