k8s.io/test-infra@v0.0.0-20240520184403-27c6b4c223d8/experiment/edit-prowjobs.py (about)

     1  #!/usr/bin/env python3
     2  
     3  # Copyright 2021 The Kubernetes Authors.
     4  #
     5  # Licensed under the Apache License, Version 2.0 (the "License");
     6  # you may not use this file except in compliance with the License.
     7  # You may obtain a copy of the License at
     8  #
     9  #     http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14  # See the License for the specific language governing permissions and
    15  # limitations under the License.
    16  
    17  """Edit prowjobs en-masse by round-tripping them through ruamel.yaml
    18  
    19  This is not intended for general usage, because:
    20  - not all jobs can or should be edited
    21  - many jobs have different formatting, and we're not at a point where
    22    we can enforce formatting standards, so this is almost guaranteed
    23    to introduce formatting change noise
    24  - the idea is to manually edit this file with the specific edit to be
    25    done, rather that developing a general purpose language to do this
    26  """
    27  
    28  import argparse
    29  import glob
    30  import re
    31  
    32  from os import path
    33  
    34  import ruamel.yaml
    35  
    36  # Prow files that will be ignored
    37  EXCLUDED_JOB_CONFIGS = [
    38      # Ruamel won't be able to successfully dump fejta-bot-periodics
    39      # See https://bitbucket.org/ruamel/yaml/issues/258/applying-json-patch-breaks-comment
    40      "fejta-bot-periodics.yaml",
    41      # generated.yaml is generated by generate_tests.py, and will be overwritten.
    42      "generated.yaml",
    43  ]
    44  # A hilariously large line length to ensure we never line-wrap
    45  MAX_WIDTH = 2000000000
    46  
    47  
    48  
    49  def setup_yaml():
    50      # Setup the ruamel.yaml parser
    51      yaml = ruamel.yaml.YAML(typ='rt')
    52      yaml.preserve_quotes = True
    53      # GoogleCloudPlatform/ - no yaml.indent
    54      # bazelbuild/ - no yaml.indent
    55      # cadvisor/ - no yaml.indent
    56      # containerd/ - no yaml.indent
    57      # image-pushing/ - yaml.indent(mapping=2, sequence=4, offset=2)
    58      # kubernetes/ - yaml.indent(mapping=2) seems to cause the least change
    59      # kubernetes-client - TBD
    60      # kubernetes-csi - TBD
    61      # kubernetes-sigs - TBD
    62      yaml.indent(mapping=2, sequence=4, offset=2)
    63      yaml.width = MAX_WIDTH
    64      return yaml
    65  
    66  def edit_job_config(yaml, prow_job_file_name, force_rewrite=False):
    67      with open(prow_job_file_name, "r") as job_fp:
    68          prow_config = yaml.load(job_fp)
    69  
    70      def edit(job):
    71          edited = False
    72          name = job["name"]
    73          print(f'  handling job: {name}')
    74          annotations = job["annotations"]
    75          dashboard_list = re.split('[, ]+', annotations["testgrid-dashboards"])
    76          if 'sig-k8s-infra-gcb' not in dashboard_list:
    77              dashboard_list.append('sig-k8s-infra-gcb')
    78              annotations["testgrid-dashboards"] = ", ".join(dashboard_list)
    79              edited = True
    80          return edited
    81  
    82      should_rewrite = force_rewrite
    83  
    84      # For each presubmit, postsubmit, and periodic
    85      # presubmits -> <any repository> -> [{name: prowjob}]
    86      if "presubmits" in prow_config:
    87          for _, jobs in prow_config["presubmits"].items():
    88              for job in jobs:
    89                  if edit(job):
    90                      should_rewrite = True
    91  
    92      # postsubmits -> <any repository> -> [{name: prowjob}]
    93      if "postsubmits" in prow_config:
    94          for _, jobs in prow_config["postsubmits"].items():
    95              for job in jobs:
    96                  if edit(job):
    97                      should_rewrite = True
    98  
    99      # periodics -> [{name: prowjob}]
   100      if "periodics" in prow_config:
   101          for job in prow_config["periodics"]:
   102              if edit(job):
   103                  should_rewrite = True
   104  
   105      # Dump ProwConfig to prowJobFile
   106      if should_rewrite:
   107          print(f'  writing {prow_job_file_name}')
   108          with open(prow_job_file_name, "w") as job_fp:
   109              yaml.dump(prow_config, job_fp)
   110              job_fp.truncate()
   111  
   112  def main(prow_job_dir, force_rewrite):
   113      yaml = setup_yaml()
   114      for f in glob.glob(f'{prow_job_dir}/**/*.yaml', recursive=True):
   115          if path.basename(f) not in EXCLUDED_JOB_CONFIGS:
   116              try:
   117                  print(f'processing config: {f}')
   118                  edit_job_config(yaml, f, force_rewrite)
   119              except Exception as e:  # pylint: disable=broad-except
   120                  print(f'ERROR: could not edit {f}: {e}')
   121  
   122  if __name__ == '__main__':
   123      PARSER = argparse.ArgumentParser(
   124          description='Does things to prowjob configs')
   125      PARSER.add_argument(
   126          '--prow-job-dir',
   127          default='../config/jobs',
   128          help='Path to Prow Job Directory')
   129      PARSER.add_argument(
   130          '--force',
   131          default=True,
   132          help='Force rewrite of all job configs')
   133      ARGS = PARSER.parse_args()
   134  
   135      main(ARGS.prow_job_dir, ARGS.force)