sigs.k8s.io/cluster-api-provider-aws@v1.5.5/hack/boilerplate/boilerplate.py (about)

     1  #!/usr/bin/env python3
     2  
     3  # Copyright 2015 The Kubernetes Authors.
     4  #
     5  # Licensed under the Apache License, Version 2.0 (the "License");
     6  # you may not use this file except in compliance with the License.
     7  # You may obtain a copy of the License at
     8  #
     9  #     http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14  # See the License for the specific language governing permissions and
    15  # limitations under the License.
    16  
    17  from __future__ import print_function
    18  
    19  import argparse
    20  import datetime
    21  import difflib
    22  import glob
    23  import os
    24  import re
    25  import sys
    26  
    27  parser = argparse.ArgumentParser()
    28  parser.add_argument(
    29      "filenames",
    30      help="list of files to check, all files if unspecified",
    31      nargs='*')
    32  
    33  rootdir = os.path.dirname(__file__) + "/../../"
    34  rootdir = os.path.abspath(rootdir)
    35  parser.add_argument(
    36      "--rootdir", default=rootdir, help="root directory to examine")
    37  
    38  default_boilerplate_dir = os.path.join(rootdir, "hack/boilerplate")
    39  parser.add_argument(
    40      "--boilerplate-dir", default=default_boilerplate_dir)
    41  
    42  parser.add_argument(
    43      "-v", "--verbose",
    44      help="give verbose output regarding why a file does not pass",
    45      action="store_true")
    46  
    47  args = parser.parse_args()
    48  
    49  verbose_out = sys.stderr if args.verbose else open("/dev/null", "w")
    50  
    51  def get_refs():
    52      refs = {}
    53  
    54      for path in glob.glob(os.path.join(args.boilerplate_dir, "boilerplate.*.txt")):
    55          extension = os.path.basename(path).split(".")[1]
    56  
    57          ref_file = open(path, 'r')
    58          ref = ref_file.read().splitlines()
    59          ref_file.close()
    60          refs[extension] = ref
    61  
    62      return refs
    63  
    64  def is_generated_file(filename, data, regexs):
    65      for d in skipped_ungenerated_files:
    66          if d in filename:
    67              return False
    68  
    69      p = regexs["generated"]
    70      return p.search(data)
    71  
    72  def file_passes(filename, refs, regexs):
    73      try:
    74          f = open(filename, 'r')
    75      except Exception as exc:
    76          print("Unable to open %s: %s" % (filename, exc), file=verbose_out)
    77          return False
    78  
    79      data = f.read()
    80      f.close()
    81  
    82      # determine if the file is automatically generated
    83      generated = is_generated_file(filename, data, regexs)
    84  
    85      basename = os.path.basename(filename)
    86      extension = file_extension(filename)
    87      if generated:
    88          if extension == "go":
    89              extension = "generatego"
    90          elif extension == "bzl":
    91              extension = "generatebzl"
    92  
    93      if extension != "":
    94          ref = refs[extension]
    95      else:
    96          ref = refs[basename]
    97  
    98      # remove extra content from the top of files
    99      if extension == "go" or extension == "generatego":
   100          p = regexs["go_build_constraints_old"]
   101          (data, found) = p.subn("", data, 1)
   102          p = regexs["go_build_constraints"]
   103          (data, found) = p.subn("", data, 1)
   104      elif extension == "sh":
   105          p = regexs["shebang"]
   106          (data, found) = p.subn("", data, 1)
   107  
   108      data = data.splitlines()
   109  
   110      # if our test file is smaller than the reference it surely fails!
   111      if len(ref) > len(data):
   112          print('File %s smaller than reference (%d < %d)' %
   113                (filename, len(data), len(ref)),
   114                file=verbose_out)
   115          return False
   116  
   117      # trim our file to the same number of lines as the reference file
   118      data = data[:len(ref)]
   119  
   120      p = regexs["year"]
   121      for d in data:
   122          if p.search(d):
   123              if generated:
   124                  print('File %s has the YEAR field, but it should not be in generated file' % filename, file=verbose_out)
   125              else:
   126                  print('File %s has the YEAR field, but missing the year of date' % filename, file=verbose_out)
   127              return False
   128  
   129      if not generated:
   130          # Replace all occurrences of the regex "2014|2015|2016|2017|2018" with "YEAR"
   131          p = regexs["date"]
   132          for i, d in enumerate(data):
   133              (data[i], found) = p.subn('YEAR', d)
   134              if found != 0:
   135                  break
   136  
   137      # if we don't match the reference at this point, fail
   138      if ref != data:
   139          print("Header in %s does not match reference, diff:" % filename, file=verbose_out)
   140          if args.verbose:
   141              print(file=verbose_out)
   142              for line in difflib.unified_diff(ref, data, 'reference', filename, lineterm=''):
   143                  print(line, file=verbose_out)
   144              print(file=verbose_out)
   145          return False
   146  
   147      return True
   148  
   149  def file_extension(filename):
   150      return os.path.splitext(filename)[1].split(".")[-1].lower()
   151  
   152  skipped_dirs = ['Godeps', 'third_party', '_gopath', '_output', '.git', 'cluster/env.sh',
   153                  "vendor", "test/e2e/generated/bindata.go", "hack/boilerplate/test",
   154                  "pkg/kubectl/generated/bindata.go"]
   155  
   156  # list all the files contain 'DO NOT EDIT', but are not generated
   157  skipped_ungenerated_files = ['hack/lib/swagger.sh', 'hack/boilerplate/boilerplate.py']
   158  
   159  def normalize_files(files):
   160      newfiles = []
   161      for pathname in files:
   162          if any(x in pathname for x in skipped_dirs):
   163              continue
   164          newfiles.append(pathname)
   165      for i, pathname in enumerate(newfiles):
   166          if not os.path.isabs(pathname):
   167              newfiles[i] = os.path.join(args.rootdir, pathname)
   168      return newfiles
   169  
   170  def get_files(extensions):
   171      files = []
   172      if len(args.filenames) > 0:
   173          files = args.filenames
   174      else:
   175          for root, dirs, walkfiles in os.walk(args.rootdir):
   176              # don't visit certain dirs. This is just a performance improvement
   177              # as we would prune these later in normalize_files(). But doing it
   178              # cuts down the amount of filesystem walking we do and cuts down
   179              # the size of the file list
   180              for d in skipped_dirs:
   181                  if d in dirs:
   182                      dirs.remove(d)
   183  
   184              for name in walkfiles:
   185                  pathname = os.path.join(root, name)
   186                  files.append(pathname)
   187  
   188      files = normalize_files(files)
   189      outfiles = []
   190      for pathname in files:
   191          basename = os.path.basename(pathname)
   192          extension = file_extension(pathname)
   193          if extension in extensions or basename in extensions:
   194              outfiles.append(pathname)
   195      return outfiles
   196  
   197  def get_dates():
   198      years = datetime.datetime.now().year
   199      return '(%s)' % '|'.join((str(year) for year in range(2014, years+1)))
   200  
   201  def get_regexs():
   202      regexs = {}
   203      # Search for "YEAR" which exists in the boilerplate, but shouldn't in the real thing
   204      regexs["year"] = re.compile( 'YEAR' )
   205      # get_dates return 2014, 2015, 2016, 2017, or 2018 until the current year as a regex like: "(2014|2015|2016|2017|2018)";
   206      # company holder names can be anything
   207      regexs["date"] = re.compile(get_dates())
   208      # strip // +build \n\n build constraints
   209      regexs["go_build_constraints_old"] = re.compile(r"^(// \+build.*\n)+\n", re.MULTILINE)
   210      # strip // +build \n\n build constraints
   211      regexs["go_build_constraints"] = re.compile(r"^(//go.build.*\n)", re.MULTILINE)
   212      # strip #!.* from shell scripts
   213      regexs["shebang"] = re.compile(r"^(#!.*\n)\n*", re.MULTILINE)
   214      # Search for generated files
   215      regexs["generated"] = re.compile( 'DO NOT EDIT' )
   216      return regexs
   217  
   218  def main():
   219      regexs = get_regexs()
   220      refs = get_refs()
   221      filenames = get_files(refs.keys())
   222  
   223      for filename in filenames:
   224          if not file_passes(filename, refs, regexs):
   225              print(filename, file=sys.stdout)
   226  
   227      return 0
   228  
   229  if __name__ == "__main__":
   230    sys.exit(main())