github.com/munnerz/test-infra@v0.0.0-20190108210205-ce3d181dc989/gubernator/view_pr.py (about) 1 # Copyright 2016 The Kubernetes Authors. 2 # 3 # Licensed under the Apache License, Version 2.0 (the "License"); 4 # you may not use this file except in compliance with the License. 5 # You may obtain a copy of the License at 6 # 7 # http://www.apache.org/licenses/LICENSE-2.0 8 # 9 # Unless required by applicable law or agreed to in writing, software 10 # distributed under the License is distributed on an "AS IS" BASIS, 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 # See the License for the specific language governing permissions and 13 # limitations under the License. 14 15 import datetime 16 import json 17 import logging 18 import os 19 import time 20 21 import filters 22 import gcs_async 23 import github.models as ghm 24 import pull_request 25 import view_base 26 import view_build 27 28 29 @view_base.memcache_memoize('pr-details://', expires=60 * 3) 30 def pr_builds(path): 31 """Return {job: [(build, {started.json}, {finished.json})]} for each job under gcs path.""" 32 jobs_dirs_fut = gcs_async.listdirs(path) 33 34 def base(path): 35 return os.path.basename(os.path.dirname(path)) 36 37 jobs_futures = [(job, gcs_async.listdirs(job)) for job in jobs_dirs_fut.get_result()] 38 futures = [] 39 40 for job, builds_fut in jobs_futures: 41 for build in builds_fut.get_result(): 42 futures.append([ 43 base(job), 44 base(build), 45 gcs_async.read('/%sstarted.json' % build), 46 gcs_async.read('/%sfinished.json' % build)]) 47 48 futures.sort(key=lambda (job, build, s, f): (job, view_base.pad_numbers(build)), reverse=True) 49 50 jobs = {} 51 for job, build, started_fut, finished_fut in futures: 52 started, finished = view_build.normalize_metadata(started_fut, finished_fut) 53 jobs.setdefault(job, []).append((build, started, finished)) 54 55 return jobs 56 57 58 def pr_path(org, repo, pr, default_org, default_repo, pull_prefix): 59 """Builds the correct gs://prefix/maybe_kubernetes/maybe_repo_org/pr.""" 60 if org == default_org and repo == default_repo: 61 return '%s/%s' % (pull_prefix, pr) 62 if org == default_org: 63 return '%s/%s/%s' % (pull_prefix, repo, pr) 64 return '%s/%s_%s/%s' % (pull_prefix, org, repo, pr) 65 66 67 def org_repo(path, default_org, default_repo): 68 """Converts /maybe_org/maybe_repo into (org, repo).""" 69 parts = path.split('/')[1:] 70 if len(parts) == 2: 71 org, repo = parts 72 elif len(parts) == 1: 73 org = default_org 74 repo = parts[0] 75 else: 76 org = default_org 77 repo = default_repo 78 return org, repo 79 80 81 def get_pull_prefix(config, org): 82 if org in config['external_services']: 83 return config['external_services'][org]['gcs_pull_prefix'] 84 return config['default_external_services']['gcs_pull_prefix'] 85 86 87 class PRHandler(view_base.BaseHandler): 88 """Show a list of test runs for a PR.""" 89 def get(self, path, pr): 90 # pylint: disable=too-many-locals 91 org, repo = org_repo(path=path, 92 default_org=self.app.config['default_org'], 93 default_repo=self.app.config['default_repo'], 94 ) 95 path = pr_path(org=org, repo=repo, pr=pr, 96 pull_prefix=get_pull_prefix(self.app.config, org), 97 default_org=self.app.config['default_org'], 98 default_repo=self.app.config['default_repo'], 99 ) 100 builds = pr_builds(path) 101 # TODO(fejta): assume all builds are monotonically increasing. 102 for bs in builds.itervalues(): 103 if any(len(b) > 8 for b, _, _ in bs): 104 bs.sort(key=lambda (b, s, f): -(s or {}).get('timestamp', 0)) 105 if pr == 'batch': # truncate batch results to last day 106 cutoff = time.time() - 60 * 60 * 24 107 builds = {} 108 for job, job_builds in builds.iteritems(): 109 builds[job] = [ 110 (b, s, f) for b, s, f in job_builds 111 if not s or s.get('timestamp') > cutoff 112 ] 113 114 max_builds, headings, rows = pull_request.builds_to_table(builds) 115 digest = ghm.GHIssueDigest.get('%s/%s' % (org, repo), pr) 116 self.render( 117 'pr.html', 118 dict( 119 pr=pr, 120 digest=digest, 121 max_builds=max_builds, 122 header=headings, 123 org=org, 124 repo=repo, 125 rows=rows, 126 path=path, 127 ) 128 ) 129 130 131 def get_acks(login, prs): 132 acks = {} 133 result = ghm.GHUserState.make_key(login).get() 134 if result: 135 acks = result.acks 136 if prs: 137 # clear acks for PRs that user is no longer involved in. 138 stale = set(acks) - set(pr.key.id() for pr in prs) 139 if stale: 140 for key in stale: 141 result.acks.pop(key) 142 result.put() 143 return acks 144 145 146 class InsensitiveString(str): 147 """A string that uses str.lower() to compare itself to others. 148 149 Does not override __in__ (that uses hash()) or sorting.""" 150 def __eq__(self, other): 151 try: 152 return other.lower() == self.lower() 153 except AttributeError: 154 return str.__eq__(self, other) 155 156 157 class PRDashboard(view_base.BaseHandler): 158 def get(self, user=None): 159 # pylint: disable=singleton-comparison 160 login = self.session.get('user') 161 if not user: 162 user = login 163 if not user: 164 self.redirect('/github_auth/pr') 165 return 166 logging.debug('user=%s', user) 167 elif user == 'all': 168 user = None 169 qs = [ghm.GHIssueDigest.is_pr == True] 170 if not self.request.get('all', False): 171 qs.append(ghm.GHIssueDigest.is_open == True) 172 if user: 173 qs.append(ghm.GHIssueDigest.involved == user.lower()) 174 prs = list(ghm.GHIssueDigest.query(*qs).fetch(batch_size=200)) 175 prs.sort(key=lambda x: x.updated_at, reverse=True) 176 177 acks = None 178 if login and user == login: # user getting their own page 179 acks = get_acks(login, prs) 180 181 fmt = self.request.get('format', 'html') 182 if fmt == 'json': 183 self.response.headers['Content-Type'] = 'application/json' 184 def serial(obj): 185 if isinstance(obj, datetime.datetime): 186 return obj.isoformat() 187 elif isinstance(obj, ghm.GHIssueDigest): 188 # pylint: disable=protected-access 189 keys = ['repo', 'number'] + list(obj._values) 190 return {k: getattr(obj, k) for k in keys} 191 raise TypeError 192 self.response.write(json.dumps(prs, sort_keys=True, default=serial, indent=True)) 193 elif fmt == 'html': 194 if user: 195 user = InsensitiveString(user) 196 def acked(p): 197 if filters.has_lgtm_without_missing_approval(p, user): 198 # LGTM is an implicit Ack (suppress from incoming)... 199 # if it doesn't also need approval 200 return True 201 if acks is None: 202 return False 203 return filters.do_get_latest(p.payload, user) <= acks.get(p.key.id(), 0) 204 def needs_attention(p): 205 labels = p.payload.get('labels', {}) 206 for u, reason in p.payload['attn'].iteritems(): 207 if user == u: # case insensitive compare 208 if acked(p): 209 continue # hide acked PRs 210 if reason == 'needs approval' and 'lgtm' not in labels: 211 continue # hide PRs that need approval but haven't been LGTMed yet 212 return True 213 return False 214 cats = [ 215 ('Needs Attention', needs_attention, ''), 216 ('Approvable', lambda p: user in p.payload.get('approvers', []), 217 'is:open is:pr ("additional approvers: {0}" ' + 218 'OR "additional approver: {0}")'.format(user)), 219 ('Incoming', lambda p: user != p.payload['author'] and 220 user in p.payload['assignees'], 221 'is:open is:pr user:kubernetes assignee:%s' % user), 222 ('Outgoing', lambda p: user == p.payload['author'], 223 'is:open is:pr user:kubernetes author:%s' % user), 224 ] 225 else: 226 cats = [('Open Kubernetes PRs', lambda x: True, 227 'is:open is:pr user:kubernetes')] 228 229 milestone = self.request.get('milestone') 230 milestones = {p.payload.get('milestone') for p in prs} - {None} 231 if milestone: 232 prs = [pr for pr in prs if pr.payload.get('milestone') == milestone] 233 234 self.render('pr_dashboard.html', dict( 235 prs=prs, cats=cats, user=user, login=login, acks=acks, 236 milestone=milestone, milestones=milestones)) 237 else: 238 self.abort(406) 239 240 def post(self): 241 login = self.session.get('user') 242 if not login: 243 self.abort(403) 244 state = ghm.GHUserState.make_key(login).get() 245 if state is None: 246 state = ghm.GHUserState.make(login) 247 body = json.loads(self.request.body) 248 if body['command'] == 'ack': 249 delta = {'%s %s' % (body['repo'], body['number']): body['latest']} 250 state.acks.update(delta) 251 state.put() 252 elif body['command'] == 'ack-clear': 253 state.acks = {} 254 state.put() 255 else: 256 self.abort(400) 257 258 259 class PRBuildLogHandler(view_base.BaseHandler): 260 def get(self, path): 261 org, _ = org_repo(path=path, 262 default_org=self.app.config['default_org'], 263 default_repo=self.app.config['default_repo'], 264 ) 265 self.redirect('https://storage.googleapis.com/%s/%s' % ( 266 get_pull_prefix(self.app.config, org), path 267 ))