github.com/munnerz/test-infra@v0.0.0-20190108210205-ce3d181dc989/gubernator/testgrid.py (about)

     1  #!/usr/bin/env python
     2  # Copyright 2016 The Kubernetes Authors.
     3  #
     4  # Licensed under the Apache License, Version 2.0 (the "License");
     5  # you may not use this file except in compliance with the License.
     6  # You may obtain a copy of the License at
     7  #
     8  #     http://www.apache.org/licenses/LICENSE-2.0
     9  #
    10  # Unless required by applicable law or agreed to in writing, software
    11  # distributed under the License is distributed on an "AS IS" BASIS,
    12  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  # See the License for the specific language governing permissions and
    14  # limitations under the License.
    15  
    16  import logging
    17  import re
    18  
    19  import cloudstorage as gcs
    20  
    21  import pb_glance
    22  
    23  
    24  CONFIG_PROTO_SCHEMA = {
    25      1: {
    26          'name': 'test_groups',
    27          1: 'name',
    28          2: 'query',
    29          9: {},
    30      },
    31      2: {
    32          'name': 'dashboards',
    33          1: {
    34              'name': 'dashboard_tab',
    35              1: 'name',
    36              2: 'test_group_name',
    37              6: 'base_options',
    38              7: {},
    39              8: {2: {}},
    40              9: {},
    41              11: {},
    42              12: {},
    43          },
    44          2: 'name',
    45      }
    46  }
    47  
    48  _testgrid_config = None
    49  
    50  
    51  def get_config():
    52      """
    53      Load the testgrid config loaded from a proto stored on GCS.
    54      It will be cached locally in memory for the life of this process.
    55  
    56      Returns:
    57          dict: {
    58              'test_groups': [{'name': ..., 'query': ...}],
    59              'dashboards': [{
    60                  'name': ...,
    61                  'dashboard_tab': [{'name': ..., 'test_group_name': ...}]
    62              }]
    63          }
    64      """
    65      global _testgrid_config  # pylint: disable=global-statement
    66      if not _testgrid_config:
    67          try:
    68              data = gcs.open('/k8s-testgrid/config').read()
    69          except gcs.NotFoundError:
    70              # Fallback to local files for development-- the k8s-testgrid bucket
    71              # has restrictive ACLs that dev_appserver.py can't read.
    72              data = open('tg-config').read()
    73          _testgrid_config = pb_glance.parse_protobuf(data, CONFIG_PROTO_SCHEMA)
    74      return _testgrid_config
    75  
    76  
    77  def path_to_group_name(path):
    78      """
    79      Args:
    80          path: a job directory like "/kubernetes-jenkins/jobs/e2e-gce"
    81      Returns:
    82          test_group_name: the group name in the config, or None if not found
    83      """
    84      try:
    85          config = get_config()
    86      except gcs.errors.Error:
    87          logging.exception('unable to load testgrid config')
    88          return None
    89      path = path.strip('/')  # the config doesn't have leading/trailing slashes
    90      if '/pull/' in path:  # translate PR to all-pr result form
    91          path = re.sub(r'/pull/([^/]+/)?\d+/', '/directory/', path)
    92      for test_group in config.get('test_groups', []):
    93          if path in test_group['query']:
    94              return test_group['name'][0]
    95  
    96  
    97  def path_to_query(path):
    98      """
    99      Convert a GCS job directory to the testgrid path for its results.
   100  
   101      Args:
   102          path: a job directory like "/kubernetes-jenkins/jobs/e2e-gce"
   103      Returns:
   104          query: the url for the job, like "k8s#gce", or "" if not found.
   105      """
   106      group = path_to_group_name(path)
   107      if not group:
   108          return ''
   109  
   110      # Tabs can appear on multiple dashboards. Favor selecting 'k8s' over others,
   111      # otherwise pick a random tab.
   112  
   113      options = {}
   114      for dashboard in get_config().get('dashboards', []):
   115          dashboard_name = dashboard['name'][0]
   116          tabs = dashboard['dashboard_tab']
   117          for (skip_base_options, penalty) in ((True, 0), (False, 1000)):
   118              for tab in tabs:
   119                  if 'base_options' in tab and skip_base_options:
   120                      continue
   121                  if group in tab['test_group_name']:
   122                      query = '%s#%s' % (dashboard_name, tab['name'][0])
   123                      options[dashboard_name] = (-len(tabs) + penalty, query)
   124              if dashboard_name in options:
   125                  break
   126      if 'k8s' in options:
   127          return options['k8s'][1]
   128      elif len(options) > 1:
   129          logging.info('ambiguous testgrid options: %s', options)
   130      elif len(options) == 0:
   131          return ''
   132      return sorted(options.values())[0][1]