github.com/web-platform-tests/wpt.fyi@v0.0.0-20240530210107-70cf978996f1/results-processor/gsutil.py (about)

     1  # Copyright 2018 The WPT Dashboard Project. All rights reserved.
     2  # Use of this source code is governed by a BSD-style license that can be
     3  # found in the LICENSE file.
     4  
     5  import logging
     6  import re
     7  import subprocess
     8  
     9  
    10  _log = logging.getLogger(__name__)
    11  
    12  
    13  def _call(command):
    14      _log.info('EXEC: %s', ' '.join(command))
    15      subprocess.check_call(command)
    16  
    17  
    18  def split_gcs_path(gcs_path):
    19      """Splits /bucket/path into (bucket, path)."""
    20      match = re.match(r'/([^/]+)/(.*)', gcs_path)
    21      assert match
    22      return match.groups()
    23  
    24  
    25  def gs_to_public_url(gs_url):
    26      """Converts a gs:// URI to a HTTP URL."""
    27      assert gs_url.startswith('gs://')
    28      return gs_url.replace('gs://', 'https://storage.googleapis.com/', 1)
    29  
    30  
    31  def copy(path1, path2, gzipped=False, quiet=True):
    32      """Copies path1 to path2 with gsutil cp.
    33  
    34      Args:
    35          path1, path2: The source and destination paths.
    36          gzipped: Whether path1 is gzipped (if True, 'Content-Encoding:gzip'
    37              will be added to the headers).
    38          quiet: Whether to suppress command output (default True).
    39      """
    40      command = [
    41          'gsutil', '-m',
    42          '-o', 'GSUtil:parallel_process_count=16',
    43          '-o', 'GSUtil:parallel_thread_count=5',
    44      ]
    45      if quiet:
    46          command += ['-q']
    47      if gzipped:
    48          command += ['-h', 'Content-Encoding:gzip']
    49      command += ['cp', '-r', path1, path2]
    50      _call(command)