github.com/shashidharatd/test-infra@v0.0.0-20171006011030-71304e1ca560/jenkins/check_projects.py (about)

     1  #!/usr/bin/env python
     2  
     3  # Copyright 2016 The Kubernetes Authors.
     4  #
     5  # Licensed under the Apache License, Version 2.0 (the "License");
     6  # you may not use this file except in compliance with the License.
     7  # You may obtain a copy of the License at
     8  #
     9  #     http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14  # See the License for the specific language governing permissions and
    15  # limitations under the License.
    16  
    17  """Check properties of test projects, optionally fixing them.
    18  
    19  Example config:
    20  
    21    {
    22      "IAM": {
    23        "roles/editor": [
    24          "serviceAccount:foo@bar.com",
    25          "user:me@you.com",
    26          "group:whatever@googlegroups.com"],
    27        "roles/viewer": [...],
    28      },
    29      "EnableVmZonalDNS': True,
    30    }
    31  """
    32  
    33  import argparse
    34  import collections
    35  import json
    36  import logging
    37  import os.path
    38  import re
    39  import subprocess
    40  import sys
    41  import threading
    42  
    43  from argparse import RawTextHelpFormatter
    44  
    45  # pylint: disable=invalid-name
    46  _log = logging.getLogger('check_project')
    47  
    48  DEFAULT = {
    49      'IAM': {
    50          'roles/editor': [
    51              'serviceAccount:kubekins@kubernetes-jenkins.iam.gserviceaccount.com',
    52              'serviceAccount:pr-kubekins@kubernetes-jenkins-pull.iam.gserviceaccount.com'],
    53      },
    54      'EnableVMZonalDNS': False,
    55  }
    56  
    57  
    58  class RateLimitedExec(object):
    59      """Runs subprocess commands with a rate limit."""
    60  
    61      def __init__(self):
    62          self.semaphore = threading.Semaphore(10)  # 10 concurrent gcloud calls
    63  
    64      def check_output(self, *args, **kwargs):
    65          """check_output with rate limit"""
    66          with self.semaphore:
    67              return subprocess.check_output(*args, **kwargs)
    68  
    69      def call(self, *args, **kwargs):
    70          """call with rate limit"""
    71          with self.semaphore:
    72              return subprocess.call(*args, **kwargs)
    73  
    74  
    75  class Results(object):
    76      """Results of the check run"""
    77  
    78      class Info(object):
    79          """Per-project information"""
    80          def __init__(self):
    81              self.updated = False
    82              # errors is list of strings describing the error context
    83              self.errors = []
    84              # People with owners rights to update error projects
    85              self.helpers = []
    86  
    87      def __init__(self):
    88          self.lock = threading.Lock()
    89          self.projects = {}  # project -> Info
    90  
    91      @property
    92      def errors(self):
    93          """Returns set of projects that have errors."""
    94          return set(key for key in self.projects if self.projects[key].errors)
    95  
    96      @property
    97      def counts(self):
    98          """Returns count of (total, updated, error'ed) projects."""
    99          with self.lock:
   100              total, updated, errored = (0, 0, 0)
   101              for info in self.projects.values():
   102                  total += 1
   103                  if info.updated:
   104                      updated += 1
   105                  if info.errors:
   106                      errored += 1
   107              return (total, updated, errored)
   108  
   109      def report_project(self, project):
   110          with self.lock:
   111              self.ensure_info(project)
   112  
   113      def report_error(self, project, err):
   114          with self.lock:
   115              info = self.ensure_info(project)
   116              info.errors.append(err)
   117  
   118      def report_updated(self, project):
   119          with self.lock:
   120              info = self.ensure_info(project)
   121              info.updated = True
   122  
   123      def add_helper(self, project, helpers):
   124          with self.lock:
   125              info = self.ensure_info(project)
   126              info.helpers = helpers
   127  
   128      def ensure_info(self, project):
   129          if project not in self.projects:
   130              info = Results.Info()
   131              self.projects[project] = info
   132          return self.projects[project]
   133  
   134  
   135  def run_threads_to_completion(threads):
   136      """Runs the given list of threads to completion."""
   137      for thread in threads:
   138          thread.start()
   139      for thread in threads:
   140          thread.join()
   141  
   142  
   143  def parse_args():
   144      """Returns parsed arguments."""
   145      parser = argparse.ArgumentParser(
   146          description=__doc__, formatter_class=RawTextHelpFormatter)
   147      parser.add_argument(
   148          '--boskos', action='store_true',
   149          help='If need to check boskos projects')
   150      parser.add_argument(
   151          '--filter', default=r'^.+$',
   152          help='Only look for jobs with the specified names')
   153      parser.add_argument(
   154          '--fix', action='store_true', help='Add missing memberships')
   155      parser.add_argument(
   156          '--verbose', action='store_true',
   157          help='Enable verbose output')
   158      parser.add_argument(
   159          'config', type=get_config, default='', nargs='?',
   160          help='Path to json configuration')
   161      return parser.parse_args()
   162  
   163  
   164  def get_config(string):
   165      """Returns configuration for project settings."""
   166      if not string:
   167          return DEFAULT
   168      elif not os.path.isfile(string):
   169          raise argparse.ArgumentTypeError('not a file: %s' % string)
   170      with open(string) as fp:
   171          return json.loads(fp.read())
   172  
   173  
   174  class Checker(object):
   175      """Runs the checks against all projects."""
   176  
   177      def __init__(self, config):
   178          self.config = config
   179          self.rl_exec = RateLimitedExec()
   180          self.results = Results()
   181  
   182      def run(self, filt, fix=False, boskos=False):
   183          """Checks projects for correct settings."""
   184          def check(project, fix):
   185              self.results.report_project(project)
   186              # pylint: disable=no-member
   187              for prop_class in ProjectProperty.__subclasses__():
   188                  prop = prop_class(self.rl_exec, self.results)
   189                  _log.info('Checking project %s for %s', project, prop.name())
   190                  prop.check_and_maybe_update(self.config, project, fix)
   191  
   192          boskos_path = None
   193          if boskos:
   194              boskos_path = '%s/../boskos/resources.json' % os.path.dirname(__file__)
   195          projects = self.load_projects(
   196              '%s/../jobs/config.json' % os.path.dirname(__file__),
   197              boskos_path,
   198              filt)
   199          _log.info('Checking %d projects', len(projects))
   200  
   201          run_threads_to_completion(
   202              [threading.Thread(target=check, args=(project, fix))
   203               for project in sorted(projects)])
   204  
   205          self.log_summary()
   206  
   207      def log_summary(self):
   208          _log.info('====')
   209          _log.info(
   210              'Summary: %d projects, %d have been updated, %d have problems',
   211              *self.results.counts)
   212          _log.info('====')
   213  
   214          for key in self.results.errors:
   215              project = self.results.projects[key]
   216              _log.info(
   217                  'Project %s needs to fix: %s', key, ','.join(project.errors))
   218  
   219          _log.info('Helpers:')
   220          fixers = collections.defaultdict(int)
   221          unk = ['user:unknown']
   222          for project in self.results.errors:
   223              helpers = self.results.projects[project].helpers
   224              if not helpers:
   225                  helpers = unk
   226              for name in helpers:
   227                  fixers[name] += 1
   228              _log.info('  %s: %s', project, ','.join(
   229                  self.sane(s) for s in sorted(helpers)))
   230  
   231          for name, count in sorted(fixers.items(), key=lambda i: i[1]):
   232              _log.info('  %s: %s', count, self.sane(name))
   233  
   234          if self.results.counts[2] != 0:
   235              sys.exit(1)
   236  
   237      @staticmethod
   238      def sane(member):
   239          if ':' not in member:
   240              raise ValueError(member)
   241          email = member.split(':')[1]
   242          return email.split('@')[0]
   243  
   244      @staticmethod
   245      def load_projects(configs, boskos, filt):
   246          """Scans the project directories for GCP projects to check."""
   247          filter_re = re.compile(filt)
   248          match_re = re.compile(r'--gcp-project=(.+)')
   249          projects = set()
   250  
   251          with open(configs) as fp:
   252              config = json.load(fp)
   253  
   254          for job, value in config.iteritems():
   255              if not filter_re.match(job):
   256                  continue
   257              for arg in value.get('args', []):
   258                  mat = match_re.match(arg)
   259                  if not mat:
   260                      continue
   261                  projects.add(mat.group(1))
   262  
   263          if not boskos:
   264              return projects
   265  
   266          with open(boskos) as fp:
   267              for rtype in json.loads(fp.read()):
   268                  if 'project' in rtype['type']:
   269                      for name in rtype['names']:
   270                          projects.add(name)
   271  
   272          return projects
   273  
   274  
   275  class ProjectProperty(object):
   276      """Base class for properties that are checked for each project.
   277  
   278      Subclasses of this class will be checked against every project.
   279      """
   280  
   281      def name(self):
   282          """
   283          Returns:
   284              human readable name of the property.
   285          """
   286          raise NotImplementedError()
   287  
   288      def check_and_maybe_update(self, config, project, fix):
   289          """Check and maybe update the project for the required property.
   290  
   291          Args:
   292              config: project configuration
   293              project: project to check
   294              fix: if True, update the project property.
   295          """
   296          raise NotImplementedError()
   297  
   298  
   299  class IAMProperty(ProjectProperty):
   300      """Project has the correct IAM properties."""
   301  
   302      def __init__(self, rl_exec, results):
   303          self.rl_exec = rl_exec
   304          self.results = results
   305  
   306      def name(self):
   307          return 'IAM'
   308  
   309      def check_and_maybe_update(self, config, project, fix):
   310          if 'IAM' not in config:
   311              return
   312  
   313          try:
   314              out = self.rl_exec.check_output([
   315                  'gcloud',
   316                  'projects',
   317                  'get-iam-policy',
   318                  project,
   319                  '--format=json(bindings)'])
   320          except subprocess.CalledProcessError:
   321              _log.info('Cannot access %s', project)
   322              self.results.report_error(project, 'access')
   323              return
   324  
   325          needed = config['IAM']
   326          bindings = json.loads(out)
   327          fixes = {}
   328          roles = set()
   329          for binding in bindings['bindings']:
   330              role = binding['role']
   331              roles.add(role)
   332              members = binding['members']
   333              if role in needed:
   334                  missing = set(needed[role]) - set(members)
   335                  if missing:
   336                      fixes[role] = missing
   337              if role == 'roles/owner':
   338                  self.results.add_helper(project, members)
   339          missing_roles = set(needed) - roles
   340          for role in missing_roles:
   341              fixes[role] = needed[role]
   342  
   343          if not fixes:
   344              _log.info('Project %s IAM is already configured', project)
   345              return
   346  
   347          if not fix:
   348              _log.info('Will not --fix %s, wanted fixed %s', project, fixes)
   349              self.results.report_error(project, self.name())
   350              return
   351  
   352          updates = []
   353          for role, members in sorted(fixes.items()):
   354              updates.extend(
   355                  threading.Thread(target=self.update, args=(project, role, m))
   356                  for m in members)
   357          run_threads_to_completion(updates)
   358  
   359      def update(self, project, role, member):
   360          cmdline = [
   361              'gcloud', '-q', 'projects', 'add-iam-policy-binding',
   362              '--role=%s' % role,
   363              '--member=%s' % member,
   364              project
   365          ]
   366          err = self.rl_exec.call(cmdline, stdout=open('/dev/null', 'w'))
   367          if not err:
   368              _log.info('Added %s as %s to %s', member, role, project)
   369              self.results.report_updated(project)
   370          else:
   371              _log.info('Could not update IAM for %s', project)
   372              self.results.report_error(
   373                  project, 'update %s (role=%s, member=%s)' %
   374                  (self.name(), role, member))
   375  
   376  
   377  class EnableVmZonalDNS(ProjectProperty):
   378      """Project has Zonal DNS enabled."""
   379  
   380      def __init__(self, rl_exec, results):
   381          self.rl_exec = rl_exec
   382          self.results = results
   383  
   384      def name(self):
   385          return 'EnableVMZonalDNS'
   386  
   387      def check_and_maybe_update(self, config, project, fix):
   388          try:
   389              out = self.rl_exec.check_output([
   390                  'gcloud', 'compute', 'project-info', 'describe',
   391                  '--project=' + project,
   392                  '--format=json(commonInstanceMetadata.items)'])
   393          except subprocess.CalledProcessError:
   394              _log.info('Cannot access %s', project)
   395              return
   396  
   397          enabled = False
   398          metadata = json.loads(out)
   399          if (metadata and metadata['commonInstanceMetadata']
   400                  and metadata['commonInstanceMetadata']['items']):
   401              for item in metadata['commonInstanceMetadata']['items']:
   402                  if item['key'] == 'EnableVmZonalDNS':
   403                      enabled = item['value'].lower() == 'yes'
   404  
   405          desired = config.get('EnableVMZonalDNS', False)
   406          if desired == enabled:
   407              _log.info(
   408                  'Project %s %s is already configured', project, self.name())
   409              return
   410  
   411          if not fix:
   412              _log.info(
   413                  'Will not --fix %s, needs to change EnableVMZonalDNS to %s',
   414                  project, desired)
   415              self.results.report_error(project, self.name())
   416              return
   417  
   418          if desired != enabled:
   419              _log.info('Updating project %s EnableVMZonalDNS from %s to %s',
   420                        project, enabled, desired)
   421              self.update(project, desired)
   422  
   423      def update(self, project, desired):
   424          if desired:
   425              err = self.rl_exec.call(
   426                  ['gcloud', 'compute', 'project-info', 'add-metadata',
   427                   '--metadata=EnableVmZonalDNS=Yes',
   428                   '--project=' + project],
   429                  stdout=open('/dev/null', 'w'))
   430          else:
   431              err = self.rl_exec.call(
   432                  ['gcloud', 'compute', 'project-info', 'remove-metadata',
   433                   '--keys=EnableVmZonalDNS',
   434                   '--project=' + project],
   435                  stdout=open('/dev/null', 'w'))
   436  
   437          if not err:
   438              _log.info('Updated zonal DNS for %s: %s', project, desired)
   439              self.results.report_updated(project)
   440          else:
   441              _log.info('Could not update zonal DNS for %s', project)
   442              self.results.report_error(project, 'update ' + self.name())
   443  
   444  
   445  def main():
   446      args = parse_args()
   447      logging.basicConfig(
   448          format="%(asctime)s %(levelname)s %(name)s] %(message)s",
   449          level=logging.DEBUG if args.verbose else logging.INFO)
   450      Checker(args.config).run(args.filter, args.fix, args.boskos)
   451  
   452  
   453  if __name__ == '__main__':
   454      main()