github.com/yrj2011/jx-test-infra@v0.0.0-20190529031832-7a2065ee98eb/gubernator/filters.py (about) 1 # Copyright 2016 The Kubernetes Authors. 2 # 3 # Licensed under the Apache License, Version 2.0 (the "License"); 4 # you may not use this file except in compliance with the License. 5 # You may obtain a copy of the License at 6 # 7 # http://www.apache.org/licenses/LICENSE-2.0 8 # 9 # Unless required by applicable law or agreed to in writing, software 10 # distributed under the License is distributed on an "AS IS" BASIS, 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 # See the License for the specific language governing permissions and 13 # limitations under the License. 14 15 import logging 16 import datetime 17 import hashlib 18 import os 19 import re 20 import time 21 import urllib 22 import urlparse 23 24 import jinja2 25 26 27 GITHUB_VIEW_TEMPLATE = 'https://github.com/%s/blob/%s/%s#L%s' 28 GITHUB_COMMIT_TEMPLATE = 'https://github.com/%s/commit/%s' 29 LINKIFY_RE = re.compile( 30 r'(^\s*/\S*/)(kubernetes/(\S+):(\d+)(?: \+0x[0-9a-f]+)?)$', 31 flags=re.MULTILINE) 32 33 34 def do_timestamp(unix_time, css_class='timestamp', tmpl='%F %H:%M'): 35 """Convert an int Unix timestamp into a human-readable datetime.""" 36 t = datetime.datetime.utcfromtimestamp(unix_time) 37 return jinja2.Markup('<span class="%s" data-epoch="%s">%s</span>' % 38 (css_class, unix_time, t.strftime(tmpl))) 39 40 41 def do_dt_to_epoch(dt): 42 return time.mktime(dt.timetuple()) 43 44 45 def do_shorttimestamp(unix_time): 46 t = datetime.datetime.utcfromtimestamp(unix_time) 47 return jinja2.Markup('<span class="shorttimestamp" data-epoch="%s">%s</span>' % 48 (unix_time, t.strftime('%d %H:%M'))) 49 50 51 def do_duration(seconds): 52 """Convert a numeric duration in seconds into a human-readable string.""" 53 hours, seconds = divmod(seconds, 3600) 54 minutes, seconds = divmod(seconds, 60) 55 if hours: 56 return '%dh%dm' % (hours, minutes) 57 if minutes: 58 return '%dm%ds' % (minutes, seconds) 59 else: 60 if seconds < 10: 61 return '%.2fs' % seconds 62 return '%ds' % seconds 63 64 65 def do_slugify(inp): 66 """Convert an arbitrary string into a url-safe slug.""" 67 inp = re.sub(r'[^\w\s-]+', '', inp) 68 return re.sub(r'\s+', '-', inp).lower() 69 70 71 def do_linkify_stacktrace(inp, commit, repo): 72 """Add links to a source code viewer for every mentioned source line.""" 73 inp = unicode(jinja2.escape(inp)) 74 if not commit: 75 return jinja2.Markup(inp) # this was already escaped, mark it safe! 76 def rep(m): 77 prefix, full, path, line = m.groups() 78 return '%s<a href="%s">%s</a>' % ( 79 prefix, 80 GITHUB_VIEW_TEMPLATE % (repo, commit, path, line), 81 full) 82 return jinja2.Markup(LINKIFY_RE.sub(rep, inp)) 83 84 85 def do_github_commit_link(commit, repo): 86 commit_url = jinja2.escape(GITHUB_COMMIT_TEMPLATE % (repo, commit)) 87 return jinja2.Markup('<a href="%s">%s</a>' % (commit_url, commit[:8])) 88 89 90 def do_maybe_linkify(inp): 91 try: 92 if urlparse.urlparse(inp).scheme in ('http', 'https'): 93 inp = unicode(jinja2.escape(inp)) 94 return jinja2.Markup('<a href="%s">%s</a>' % (inp, inp)) 95 except (AttributeError, TypeError): 96 pass 97 return inp 98 99 100 def do_testcmd(name): 101 if name.startswith('k8s.io/'): 102 try: 103 pkg, name = name.split(' ') 104 except ValueError: # don't block the page render 105 logging.error('Unexpected Go unit test name %r', name) 106 return name 107 return 'go test -v %s -run %s$' % (pkg, name) 108 elif name.startswith('istio.io/'): 109 return '' 110 elif name.startswith('//'): 111 return 'bazel test %s' % name 112 elif name.startswith('verify '): 113 return 'make verify WHAT=%s' % name.split(' ')[1] 114 else: 115 name = re.sub(r'^\[k8s\.io\] ', '', name) 116 name_escaped = re.escape(name).replace('\\ ', '\\s') 117 118 test_args = ('--ginkgo.focus=%s$' % name_escaped) 119 return "go run hack/e2e.go -v --test --test_args='%s'" % test_args 120 121 122 def do_parse_pod_name(text): 123 """Find the pod name from the failure and return the pod name.""" 124 p = re.search(r' pod (\S+)', text) 125 if p: 126 return re.sub(r'[\'"\\:]', '', p.group(1)) 127 else: 128 return "" 129 130 131 def do_label_attr(labels, name): 132 """ 133 >> do_label_attr(['needs-rebase', 'size/XS'], 'size') 134 'XS' 135 """ 136 name += '/' 137 for label in labels: 138 if label.startswith(name): 139 return label[len(name):] 140 return '' 141 142 def do_classify_size(payload): 143 """ 144 Determine the size class for a PR, based on either its labels or 145 on the magnitude of its changes. 146 """ 147 size = do_label_attr(payload['labels'], 'size') 148 if not size and 'additions' in payload and 'deletions' in payload: 149 lines = payload['additions'] + payload['deletions'] 150 # based on mungegithub/mungers/size.go 151 for limit, label in [ 152 (10, 'XS'), 153 (30, 'S'), 154 (100, 'M'), 155 (500, 'L'), 156 (1000, 'XL') 157 ]: 158 if lines < limit: 159 return label 160 return 'XXL' 161 return size 162 163 164 def has_lgtm_without_missing_approval(payload, user): 165 labels = payload.get('labels', []) or [] 166 return 'lgtm' in labels and not ( 167 user in payload.get('approvers', []) 168 and 'approved' not in labels) 169 170 171 def do_render_status(payload, user): 172 states = set() 173 174 text = 'Pending' 175 if has_lgtm_without_missing_approval(payload, user): 176 text = 'LGTM' 177 elif user in payload.get('attn', {}): 178 text = payload['attn'][user].title() 179 if '#' in text: # strip start/end attn timestamps 180 text = text[:text.index('#')] 181 182 for ctx, (state, _url, desc) in payload.get('status', {}).items(): 183 if ctx == 'Submit Queue' and state == 'pending': 184 if 'does not have lgtm' in desc.lower(): 185 # Don't show overall status as pending when Submit 186 # won't continue without LGTM. 187 continue 188 if ctx == 'tide' and state == 'pending': 189 # Ignore pending tide statuses for now. 190 continue 191 if ctx == 'code-review/reviewable' and state == 'pending': 192 # Reviewable isn't a CI, so we don't care if it's pending. 193 # Its dashboard might replace all of this eventually. 194 continue 195 states.add(state) 196 197 icon = '' 198 title = '' 199 if 'failure' in states: 200 icon = 'x' 201 state = 'failure' 202 title = 'failing tests' 203 elif 'pending' in states: 204 icon = 'primitive-dot' 205 state = 'pending' 206 title = 'pending tests' 207 elif 'success' in states: 208 icon = 'check' 209 state = 'success' 210 title = 'tests passing' 211 if icon: 212 icon = '<span class="text-%s octicon octicon-%s" title="%s"></span>' % ( 213 state, icon, title) 214 return jinja2.Markup('%s%s' % (icon, text)) 215 216 217 def do_get_latest(payload, user): 218 text = payload.get('attn', {}).get(user) 219 if not text: 220 return None 221 if '#' not in text: 222 return None 223 _text, _start, latest = text.rsplit('#', 2) 224 return float(latest) 225 226 227 def do_ltrim(s, needle): 228 if s.startswith(needle): 229 return s[len(needle):] 230 return s 231 232 233 def do_select(seq, pred): 234 return filter(pred, seq) 235 236 237 def do_tg_url(testgrid_query, test_name=''): 238 if test_name: 239 regex = '^Overall$|' + re.escape(test_name) 240 testgrid_query += '&include-filter-by-regex=%s' % urllib.quote(regex) 241 return 'https://k8s-testgrid.appspot.com/%s' % testgrid_query 242 243 244 def do_gcs_browse_url(gcs_path): 245 if not gcs_path.endswith('/'): 246 gcs_path += '/' 247 return 'http://gcsweb.k8s.io/gcs' + gcs_path 248 249 250 static_hashes = {} 251 252 def do_static(filename): 253 filename = 'static/%s' % filename 254 if filename not in static_hashes: 255 data = open(filename).read() 256 static_hashes[filename] = hashlib.sha1(data).hexdigest()[:10] 257 return '/%s?%s' % (filename, static_hashes[filename]) 258 259 260 do_basename = os.path.basename 261 do_dirname = os.path.dirname 262 do_quote_plus = urllib.quote_plus 263 264 265 def register(filters): 266 """Register do_* functions in this module in a dictionary.""" 267 for name, func in globals().items(): 268 if name.startswith('do_'): 269 filters[name[3:]] = func