github.com/munnerz/test-infra@v0.0.0-20190108210205-ce3d181dc989/gubernator/testgrid.py (about) 1 #!/usr/bin/env python 2 # Copyright 2016 The Kubernetes Authors. 3 # 4 # Licensed under the Apache License, Version 2.0 (the "License"); 5 # you may not use this file except in compliance with the License. 6 # You may obtain a copy of the License at 7 # 8 # http://www.apache.org/licenses/LICENSE-2.0 9 # 10 # Unless required by applicable law or agreed to in writing, software 11 # distributed under the License is distributed on an "AS IS" BASIS, 12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 # See the License for the specific language governing permissions and 14 # limitations under the License. 15 16 import logging 17 import re 18 19 import cloudstorage as gcs 20 21 import pb_glance 22 23 24 CONFIG_PROTO_SCHEMA = { 25 1: { 26 'name': 'test_groups', 27 1: 'name', 28 2: 'query', 29 9: {}, 30 }, 31 2: { 32 'name': 'dashboards', 33 1: { 34 'name': 'dashboard_tab', 35 1: 'name', 36 2: 'test_group_name', 37 6: 'base_options', 38 7: {}, 39 8: {2: {}}, 40 9: {}, 41 11: {}, 42 12: {}, 43 }, 44 2: 'name', 45 } 46 } 47 48 _testgrid_config = None 49 50 51 def get_config(): 52 """ 53 Load the testgrid config loaded from a proto stored on GCS. 54 It will be cached locally in memory for the life of this process. 55 56 Returns: 57 dict: { 58 'test_groups': [{'name': ..., 'query': ...}], 59 'dashboards': [{ 60 'name': ..., 61 'dashboard_tab': [{'name': ..., 'test_group_name': ...}] 62 }] 63 } 64 """ 65 global _testgrid_config # pylint: disable=global-statement 66 if not _testgrid_config: 67 try: 68 data = gcs.open('/k8s-testgrid/config').read() 69 except gcs.NotFoundError: 70 # Fallback to local files for development-- the k8s-testgrid bucket 71 # has restrictive ACLs that dev_appserver.py can't read. 72 data = open('tg-config').read() 73 _testgrid_config = pb_glance.parse_protobuf(data, CONFIG_PROTO_SCHEMA) 74 return _testgrid_config 75 76 77 def path_to_group_name(path): 78 """ 79 Args: 80 path: a job directory like "/kubernetes-jenkins/jobs/e2e-gce" 81 Returns: 82 test_group_name: the group name in the config, or None if not found 83 """ 84 try: 85 config = get_config() 86 except gcs.errors.Error: 87 logging.exception('unable to load testgrid config') 88 return None 89 path = path.strip('/') # the config doesn't have leading/trailing slashes 90 if '/pull/' in path: # translate PR to all-pr result form 91 path = re.sub(r'/pull/([^/]+/)?\d+/', '/directory/', path) 92 for test_group in config.get('test_groups', []): 93 if path in test_group['query']: 94 return test_group['name'][0] 95 96 97 def path_to_query(path): 98 """ 99 Convert a GCS job directory to the testgrid path for its results. 100 101 Args: 102 path: a job directory like "/kubernetes-jenkins/jobs/e2e-gce" 103 Returns: 104 query: the url for the job, like "k8s#gce", or "" if not found. 105 """ 106 group = path_to_group_name(path) 107 if not group: 108 return '' 109 110 # Tabs can appear on multiple dashboards. Favor selecting 'k8s' over others, 111 # otherwise pick a random tab. 112 113 options = {} 114 for dashboard in get_config().get('dashboards', []): 115 dashboard_name = dashboard['name'][0] 116 tabs = dashboard['dashboard_tab'] 117 for (skip_base_options, penalty) in ((True, 0), (False, 1000)): 118 for tab in tabs: 119 if 'base_options' in tab and skip_base_options: 120 continue 121 if group in tab['test_group_name']: 122 query = '%s#%s' % (dashboard_name, tab['name'][0]) 123 options[dashboard_name] = (-len(tabs) + penalty, query) 124 if dashboard_name in options: 125 break 126 if 'k8s' in options: 127 return options['k8s'][1] 128 elif len(options) > 1: 129 logging.info('ambiguous testgrid options: %s', options) 130 elif len(options) == 0: 131 return '' 132 return sorted(options.values())[0][1]