github.com/shashidharatd/test-infra@v0.0.0-20171006011030-71304e1ca560/jenkins/check_projects.py (about) 1 #!/usr/bin/env python 2 3 # Copyright 2016 The Kubernetes Authors. 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 """Check properties of test projects, optionally fixing them. 18 19 Example config: 20 21 { 22 "IAM": { 23 "roles/editor": [ 24 "serviceAccount:foo@bar.com", 25 "user:me@you.com", 26 "group:whatever@googlegroups.com"], 27 "roles/viewer": [...], 28 }, 29 "EnableVmZonalDNS': True, 30 } 31 """ 32 33 import argparse 34 import collections 35 import json 36 import logging 37 import os.path 38 import re 39 import subprocess 40 import sys 41 import threading 42 43 from argparse import RawTextHelpFormatter 44 45 # pylint: disable=invalid-name 46 _log = logging.getLogger('check_project') 47 48 DEFAULT = { 49 'IAM': { 50 'roles/editor': [ 51 'serviceAccount:kubekins@kubernetes-jenkins.iam.gserviceaccount.com', 52 'serviceAccount:pr-kubekins@kubernetes-jenkins-pull.iam.gserviceaccount.com'], 53 }, 54 'EnableVMZonalDNS': False, 55 } 56 57 58 class RateLimitedExec(object): 59 """Runs subprocess commands with a rate limit.""" 60 61 def __init__(self): 62 self.semaphore = threading.Semaphore(10) # 10 concurrent gcloud calls 63 64 def check_output(self, *args, **kwargs): 65 """check_output with rate limit""" 66 with self.semaphore: 67 return subprocess.check_output(*args, **kwargs) 68 69 def call(self, *args, **kwargs): 70 """call with rate limit""" 71 with self.semaphore: 72 return subprocess.call(*args, **kwargs) 73 74 75 class Results(object): 76 """Results of the check run""" 77 78 class Info(object): 79 """Per-project information""" 80 def __init__(self): 81 self.updated = False 82 # errors is list of strings describing the error context 83 self.errors = [] 84 # People with owners rights to update error projects 85 self.helpers = [] 86 87 def __init__(self): 88 self.lock = threading.Lock() 89 self.projects = {} # project -> Info 90 91 @property 92 def errors(self): 93 """Returns set of projects that have errors.""" 94 return set(key for key in self.projects if self.projects[key].errors) 95 96 @property 97 def counts(self): 98 """Returns count of (total, updated, error'ed) projects.""" 99 with self.lock: 100 total, updated, errored = (0, 0, 0) 101 for info in self.projects.values(): 102 total += 1 103 if info.updated: 104 updated += 1 105 if info.errors: 106 errored += 1 107 return (total, updated, errored) 108 109 def report_project(self, project): 110 with self.lock: 111 self.ensure_info(project) 112 113 def report_error(self, project, err): 114 with self.lock: 115 info = self.ensure_info(project) 116 info.errors.append(err) 117 118 def report_updated(self, project): 119 with self.lock: 120 info = self.ensure_info(project) 121 info.updated = True 122 123 def add_helper(self, project, helpers): 124 with self.lock: 125 info = self.ensure_info(project) 126 info.helpers = helpers 127 128 def ensure_info(self, project): 129 if project not in self.projects: 130 info = Results.Info() 131 self.projects[project] = info 132 return self.projects[project] 133 134 135 def run_threads_to_completion(threads): 136 """Runs the given list of threads to completion.""" 137 for thread in threads: 138 thread.start() 139 for thread in threads: 140 thread.join() 141 142 143 def parse_args(): 144 """Returns parsed arguments.""" 145 parser = argparse.ArgumentParser( 146 description=__doc__, formatter_class=RawTextHelpFormatter) 147 parser.add_argument( 148 '--boskos', action='store_true', 149 help='If need to check boskos projects') 150 parser.add_argument( 151 '--filter', default=r'^.+$', 152 help='Only look for jobs with the specified names') 153 parser.add_argument( 154 '--fix', action='store_true', help='Add missing memberships') 155 parser.add_argument( 156 '--verbose', action='store_true', 157 help='Enable verbose output') 158 parser.add_argument( 159 'config', type=get_config, default='', nargs='?', 160 help='Path to json configuration') 161 return parser.parse_args() 162 163 164 def get_config(string): 165 """Returns configuration for project settings.""" 166 if not string: 167 return DEFAULT 168 elif not os.path.isfile(string): 169 raise argparse.ArgumentTypeError('not a file: %s' % string) 170 with open(string) as fp: 171 return json.loads(fp.read()) 172 173 174 class Checker(object): 175 """Runs the checks against all projects.""" 176 177 def __init__(self, config): 178 self.config = config 179 self.rl_exec = RateLimitedExec() 180 self.results = Results() 181 182 def run(self, filt, fix=False, boskos=False): 183 """Checks projects for correct settings.""" 184 def check(project, fix): 185 self.results.report_project(project) 186 # pylint: disable=no-member 187 for prop_class in ProjectProperty.__subclasses__(): 188 prop = prop_class(self.rl_exec, self.results) 189 _log.info('Checking project %s for %s', project, prop.name()) 190 prop.check_and_maybe_update(self.config, project, fix) 191 192 boskos_path = None 193 if boskos: 194 boskos_path = '%s/../boskos/resources.json' % os.path.dirname(__file__) 195 projects = self.load_projects( 196 '%s/../jobs/config.json' % os.path.dirname(__file__), 197 boskos_path, 198 filt) 199 _log.info('Checking %d projects', len(projects)) 200 201 run_threads_to_completion( 202 [threading.Thread(target=check, args=(project, fix)) 203 for project in sorted(projects)]) 204 205 self.log_summary() 206 207 def log_summary(self): 208 _log.info('====') 209 _log.info( 210 'Summary: %d projects, %d have been updated, %d have problems', 211 *self.results.counts) 212 _log.info('====') 213 214 for key in self.results.errors: 215 project = self.results.projects[key] 216 _log.info( 217 'Project %s needs to fix: %s', key, ','.join(project.errors)) 218 219 _log.info('Helpers:') 220 fixers = collections.defaultdict(int) 221 unk = ['user:unknown'] 222 for project in self.results.errors: 223 helpers = self.results.projects[project].helpers 224 if not helpers: 225 helpers = unk 226 for name in helpers: 227 fixers[name] += 1 228 _log.info(' %s: %s', project, ','.join( 229 self.sane(s) for s in sorted(helpers))) 230 231 for name, count in sorted(fixers.items(), key=lambda i: i[1]): 232 _log.info(' %s: %s', count, self.sane(name)) 233 234 if self.results.counts[2] != 0: 235 sys.exit(1) 236 237 @staticmethod 238 def sane(member): 239 if ':' not in member: 240 raise ValueError(member) 241 email = member.split(':')[1] 242 return email.split('@')[0] 243 244 @staticmethod 245 def load_projects(configs, boskos, filt): 246 """Scans the project directories for GCP projects to check.""" 247 filter_re = re.compile(filt) 248 match_re = re.compile(r'--gcp-project=(.+)') 249 projects = set() 250 251 with open(configs) as fp: 252 config = json.load(fp) 253 254 for job, value in config.iteritems(): 255 if not filter_re.match(job): 256 continue 257 for arg in value.get('args', []): 258 mat = match_re.match(arg) 259 if not mat: 260 continue 261 projects.add(mat.group(1)) 262 263 if not boskos: 264 return projects 265 266 with open(boskos) as fp: 267 for rtype in json.loads(fp.read()): 268 if 'project' in rtype['type']: 269 for name in rtype['names']: 270 projects.add(name) 271 272 return projects 273 274 275 class ProjectProperty(object): 276 """Base class for properties that are checked for each project. 277 278 Subclasses of this class will be checked against every project. 279 """ 280 281 def name(self): 282 """ 283 Returns: 284 human readable name of the property. 285 """ 286 raise NotImplementedError() 287 288 def check_and_maybe_update(self, config, project, fix): 289 """Check and maybe update the project for the required property. 290 291 Args: 292 config: project configuration 293 project: project to check 294 fix: if True, update the project property. 295 """ 296 raise NotImplementedError() 297 298 299 class IAMProperty(ProjectProperty): 300 """Project has the correct IAM properties.""" 301 302 def __init__(self, rl_exec, results): 303 self.rl_exec = rl_exec 304 self.results = results 305 306 def name(self): 307 return 'IAM' 308 309 def check_and_maybe_update(self, config, project, fix): 310 if 'IAM' not in config: 311 return 312 313 try: 314 out = self.rl_exec.check_output([ 315 'gcloud', 316 'projects', 317 'get-iam-policy', 318 project, 319 '--format=json(bindings)']) 320 except subprocess.CalledProcessError: 321 _log.info('Cannot access %s', project) 322 self.results.report_error(project, 'access') 323 return 324 325 needed = config['IAM'] 326 bindings = json.loads(out) 327 fixes = {} 328 roles = set() 329 for binding in bindings['bindings']: 330 role = binding['role'] 331 roles.add(role) 332 members = binding['members'] 333 if role in needed: 334 missing = set(needed[role]) - set(members) 335 if missing: 336 fixes[role] = missing 337 if role == 'roles/owner': 338 self.results.add_helper(project, members) 339 missing_roles = set(needed) - roles 340 for role in missing_roles: 341 fixes[role] = needed[role] 342 343 if not fixes: 344 _log.info('Project %s IAM is already configured', project) 345 return 346 347 if not fix: 348 _log.info('Will not --fix %s, wanted fixed %s', project, fixes) 349 self.results.report_error(project, self.name()) 350 return 351 352 updates = [] 353 for role, members in sorted(fixes.items()): 354 updates.extend( 355 threading.Thread(target=self.update, args=(project, role, m)) 356 for m in members) 357 run_threads_to_completion(updates) 358 359 def update(self, project, role, member): 360 cmdline = [ 361 'gcloud', '-q', 'projects', 'add-iam-policy-binding', 362 '--role=%s' % role, 363 '--member=%s' % member, 364 project 365 ] 366 err = self.rl_exec.call(cmdline, stdout=open('/dev/null', 'w')) 367 if not err: 368 _log.info('Added %s as %s to %s', member, role, project) 369 self.results.report_updated(project) 370 else: 371 _log.info('Could not update IAM for %s', project) 372 self.results.report_error( 373 project, 'update %s (role=%s, member=%s)' % 374 (self.name(), role, member)) 375 376 377 class EnableVmZonalDNS(ProjectProperty): 378 """Project has Zonal DNS enabled.""" 379 380 def __init__(self, rl_exec, results): 381 self.rl_exec = rl_exec 382 self.results = results 383 384 def name(self): 385 return 'EnableVMZonalDNS' 386 387 def check_and_maybe_update(self, config, project, fix): 388 try: 389 out = self.rl_exec.check_output([ 390 'gcloud', 'compute', 'project-info', 'describe', 391 '--project=' + project, 392 '--format=json(commonInstanceMetadata.items)']) 393 except subprocess.CalledProcessError: 394 _log.info('Cannot access %s', project) 395 return 396 397 enabled = False 398 metadata = json.loads(out) 399 if (metadata and metadata['commonInstanceMetadata'] 400 and metadata['commonInstanceMetadata']['items']): 401 for item in metadata['commonInstanceMetadata']['items']: 402 if item['key'] == 'EnableVmZonalDNS': 403 enabled = item['value'].lower() == 'yes' 404 405 desired = config.get('EnableVMZonalDNS', False) 406 if desired == enabled: 407 _log.info( 408 'Project %s %s is already configured', project, self.name()) 409 return 410 411 if not fix: 412 _log.info( 413 'Will not --fix %s, needs to change EnableVMZonalDNS to %s', 414 project, desired) 415 self.results.report_error(project, self.name()) 416 return 417 418 if desired != enabled: 419 _log.info('Updating project %s EnableVMZonalDNS from %s to %s', 420 project, enabled, desired) 421 self.update(project, desired) 422 423 def update(self, project, desired): 424 if desired: 425 err = self.rl_exec.call( 426 ['gcloud', 'compute', 'project-info', 'add-metadata', 427 '--metadata=EnableVmZonalDNS=Yes', 428 '--project=' + project], 429 stdout=open('/dev/null', 'w')) 430 else: 431 err = self.rl_exec.call( 432 ['gcloud', 'compute', 'project-info', 'remove-metadata', 433 '--keys=EnableVmZonalDNS', 434 '--project=' + project], 435 stdout=open('/dev/null', 'w')) 436 437 if not err: 438 _log.info('Updated zonal DNS for %s: %s', project, desired) 439 self.results.report_updated(project) 440 else: 441 _log.info('Could not update zonal DNS for %s', project) 442 self.results.report_error(project, 'update ' + self.name()) 443 444 445 def main(): 446 args = parse_args() 447 logging.basicConfig( 448 format="%(asctime)s %(levelname)s %(name)s] %(message)s", 449 level=logging.DEBUG if args.verbose else logging.INFO) 450 Checker(args.config).run(args.filter, args.fix, args.boskos) 451 452 453 if __name__ == '__main__': 454 main()