github.com/web-platform-tests/wpt.fyi@v0.0.0-20240530210107-70cf978996f1/util/generate_testrun_index.py (about)

     1  #!/usr/bin/env python
     2  
     3  # Copyright 2017 The WPT Dashboard Project. All rights reserved.
     4  # Use of this source code is governed by a BSD-style license that can be
     5  # found in the LICENSE file.
     6  
     7  import json
     8  from google.cloud import storage
     9  
    10  """
    11  Scans all WPT results directories then generates and uploads an index.
    12  
    13  You must be logged into gcloud and a member of the wptdashboard project
    14  in order for this script to work.
    15  """
    16  
    17  GCP_PROJECT = 'wptdashboard'
    18  RESULTS_BUCKET = 'wptd'
    19  
    20  
    21  def main():
    22      storage_client = storage.Client(project=GCP_PROJECT)
    23      bucket = storage_client.get_bucket(RESULTS_BUCKET)
    24  
    25      # by_sha is an object where:
    26      # Key: a WPT commit SHA
    27      # Value: list of platform IDs the SHA was tested against
    28      by_sha = {}
    29  
    30      # by_platform is an object where:
    31      # Key: a platform ID
    32      # Value: list of WPT commit SHAs the platform was tested against
    33      by_platform = {}
    34  
    35      sha_directories = list_directory(bucket)
    36  
    37      for sha_directory in sha_directories:
    38          sha = sha_directory.replace('/', '')
    39          directories = list_directory(bucket, sha_directory)
    40          platform_directories = [
    41              prefix[len(sha_directory):].replace('/', '')
    42              for prefix in directories
    43          ]
    44  
    45          for platform in platform_directories:
    46              by_sha.setdefault(sha, [])
    47              by_sha[sha].append(platform)
    48  
    49              by_platform.setdefault(platform, [])
    50              by_platform[platform].append(sha)
    51  
    52      print('by_sha', by_sha)
    53      print('by_platform', by_platform)
    54  
    55      index = {
    56          'by_sha': by_sha,
    57          'by_platform': by_platform
    58      }
    59  
    60      filename = 'testruns-index.json'
    61      blob = bucket.blob(filename)
    62      blob.upload_from_string(json.dumps(index), content_type='application/json')
    63  
    64      print('Uploaded!')
    65      print('https://storage.googleapis.com/wptd/%s' % filename)
    66  
    67  
    68  def list_directory(bucket, prefix=None):
    69      iterator = bucket.list_blobs(delimiter='/', prefix=prefix)
    70      response = iterator._get_next_page_response()
    71      return response['prefixes']
    72  
    73  
    74  if __name__ == '__main__':
    75      main()