github.com/web-platform-tests/wpt.fyi@v0.0.0-20240530210107-70cf978996f1/results-processor/gsutil.py (about) 1 # Copyright 2018 The WPT Dashboard Project. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import logging 6 import re 7 import subprocess 8 9 10 _log = logging.getLogger(__name__) 11 12 13 def _call(command): 14 _log.info('EXEC: %s', ' '.join(command)) 15 subprocess.check_call(command) 16 17 18 def split_gcs_path(gcs_path): 19 """Splits /bucket/path into (bucket, path).""" 20 match = re.match(r'/([^/]+)/(.*)', gcs_path) 21 assert match 22 return match.groups() 23 24 25 def gs_to_public_url(gs_url): 26 """Converts a gs:// URI to a HTTP URL.""" 27 assert gs_url.startswith('gs://') 28 return gs_url.replace('gs://', 'https://storage.googleapis.com/', 1) 29 30 31 def copy(path1, path2, gzipped=False, quiet=True): 32 """Copies path1 to path2 with gsutil cp. 33 34 Args: 35 path1, path2: The source and destination paths. 36 gzipped: Whether path1 is gzipped (if True, 'Content-Encoding:gzip' 37 will be added to the headers). 38 quiet: Whether to suppress command output (default True). 39 """ 40 command = [ 41 'gsutil', '-m', 42 '-o', 'GSUtil:parallel_process_count=16', 43 '-o', 'GSUtil:parallel_thread_count=5', 44 ] 45 if quiet: 46 command += ['-q'] 47 if gzipped: 48 command += ['-h', 'Content-Encoding:gzip'] 49 command += ['cp', '-r', path1, path2] 50 _call(command)