github.com/minio/madmin-go/v2@v2.2.1/licenseheaders.py (about) 1 #!/usr/bin/env python 2 # encoding: utf-8 3 4 """A tool to change or add license headers in all supported files in or below a directory.""" 5 6 # Copyright (c) 2016-2018 Johann Petrak 7 # 8 # Permission is hereby granted, free of charge, to any person obtaining a copy 9 # of this software and associated documentation files (the "Software"), to deal 10 # in the Software without restriction, including without limitation the rights 11 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 12 # copies of the Software, and to permit persons to whom the Software is 13 # furnished to do so, subject to the following conditions: 14 # 15 # The above copyright notice and this permission notice shall be included in 16 # all copies or substantial portions of the Software. 17 # 18 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 24 # THE SOFTWARE. 25 26 import argparse 27 import fnmatch 28 import logging 29 import os 30 import sys 31 import stat 32 import contextlib 33 from shutil import copyfile 34 from string import Template 35 36 import regex as re 37 38 __version__ = '0.8.8' 39 __author__ = 'Johann Petrak' 40 __license__ = 'MIT' 41 42 LOGGER = logging.getLogger("licenseheaders_{}".format(__version__)) 43 44 45 default_dir = "." 46 default_encoding = "utf-8" 47 48 def update_c_style_comments(extensions): 49 return { 50 "extensions": extensions, 51 "keepFirst": None, 52 "blockCommentStartPattern": re.compile(r'^\s*/\*'), 53 "blockCommentEndPattern": re.compile(r'\*/\s*$'), 54 "lineCommentStartPattern": re.compile(r'^\s*//'), 55 "lineCommentEndPattern": None, 56 "headerStartLine": "/*\n", 57 "headerEndLine": " */\n", 58 "headerLinePrefix": " * ", 59 "headerLineSuffix": None, 60 } 61 62 def update_go_style_comments(extensions): 63 return { 64 "extensions": extensions, 65 "keepFirst": None, 66 "blockCommentStartPattern": re.compile(r'^\s*/\*'), 67 "blockCommentEndPattern": re.compile(r'\*/\s*$'), 68 "lineCommentStartPattern": re.compile(r'^\s*//'), 69 "lineCommentEndPattern": None, 70 "headerStartLine": "//\n", 71 "headerEndLine": "//\n", 72 "headerLinePrefix": "// ", 73 "headerLineSuffix": None, 74 } 75 76 # for each processing type, the detailed settings of how to process files of that type 77 TYPE_SETTINGS = { 78 # All the languages with C style comments: 79 "c": update_c_style_comments([".c", ".cc", ".h"]), 80 "cpp": update_c_style_comments([".cpp", ".hpp", ".cxx", ".hxx", ".ixx"]), 81 "csharp": update_c_style_comments([".cs", ".csx"]), 82 "d": update_c_style_comments([".d"]), 83 "go": update_go_style_comments([".go"]), 84 "groovy": update_c_style_comments([".groovy"]), 85 "java": update_c_style_comments([".java", ".jape"]), 86 "javascript": update_c_style_comments([".js", ".js", ".cjs", ".mjs"]), 87 "kotlin": update_c_style_comments([".kt", ".kts", ".ktm"]), 88 "objective-c": update_c_style_comments([".m", ".mm", ".M"]), 89 "php": update_c_style_comments([".php," ".phtml," ".php3," ".php4," ".php5," ".php7," ".phps," ".php-s," ".pht," ".phar"]), 90 "rust": update_c_style_comments([".rs"]), 91 "scala": update_c_style_comments([".scala"]), 92 "swift": update_c_style_comments([".swift"]), 93 "typescript": update_c_style_comments([".ts", ".tsx"]), 94 "script": { 95 "extensions": [".sh", ".csh", ".pl"], 96 "keepFirst": re.compile(r'^#!|^# -\*-'), 97 "blockCommentStartPattern": None, 98 "blockCommentEndPattern": None, 99 "lineCommentStartPattern": re.compile(r'^\s*#'), 100 "lineCommentEndPattern": None, 101 "headerStartLine": "##\n", 102 "headerEndLine": "##\n", 103 "headerLinePrefix": "## ", 104 "headerLineSuffix": None 105 }, 106 "perl": { 107 "extensions": [".pl"], 108 "keepFirst": re.compile(r'^#!|^# -\*-'), 109 "blockCommentStartPattern": None, 110 "blockCommentEndPattern": None, 111 "lineCommentStartPattern": re.compile(r'^\s*#'), 112 "lineCommentEndPattern": None, 113 "headerStartLine": "##\n", 114 "headerEndLine": "##\n", 115 "headerLinePrefix": "## ", 116 "headerLineSuffix": None 117 }, 118 "python": { 119 "extensions": [".py"], 120 "keepFirst": re.compile(r'^#!|^# +pylint|^# +-\*-|^# +coding|^# +encoding|^# +type|^# +flake8'), 121 "blockCommentStartPattern": None, 122 "blockCommentEndPattern": None, 123 "lineCommentStartPattern": re.compile(r'^\s*#'), 124 "lineCommentEndPattern": None, 125 "headerStartLine": None, 126 "headerEndLine": "\n", 127 "headerLinePrefix": "# ", 128 "headerLineSuffix": None 129 }, 130 "robot": { 131 "extensions": [".robot"], 132 "keepFirst": re.compile(r'^#!|^# +pylint|^# +-\*-|^# +coding|^# +encoding'), 133 "blockCommentStartPattern": None, 134 "blockCommentEndPattern": None, 135 "lineCommentStartPattern": re.compile(r'^\s*#'), 136 "lineCommentEndPattern": None, 137 "headerStartLine": None, 138 "headerEndLine": None, 139 "headerLinePrefix": "# ", 140 "headerLineSuffix": None 141 }, 142 "xml": { 143 "extensions": [".xml"], 144 "keepFirst": re.compile(r'^\s*<\?xml.*\?>'), 145 "blockCommentStartPattern": re.compile(r'^\s*<!--'), 146 "blockCommentEndPattern": re.compile(r'-->\s*$'), 147 "lineCommentStartPattern": None, 148 "lineCommentEndPattern": None, 149 "headerStartLine": "<!--\n", 150 "headerEndLine": " -->\n", 151 "headerLinePrefix": "-- ", 152 "headerLineSuffix": None 153 }, 154 "sql": { 155 "extensions": [".sql"], 156 "keepFirst": None, 157 "blockCommentStartPattern": None, # re.compile('^\s*/\*'), 158 "blockCommentEndPattern": None, # re.compile(r'\*/\s*$'), 159 "lineCommentStartPattern": re.compile(r'^\s*--'), 160 "lineCommentEndPattern": None, 161 "headerStartLine": "--\n", 162 "headerEndLine": "--\n", 163 "headerLinePrefix": "-- ", 164 "headerLineSuffix": None 165 }, 166 "cmake": { 167 "extensions": [], 168 "filenames": ["CMakeLists.txt"], 169 "keepFirst": None, 170 "blockCommentStartPattern": re.compile(r'^\s*#\[\['), 171 "blockCommentEndPattern": re.compile(r'\]\]\s*$'), 172 "lineCommentStartPattern": re.compile(r'\s*#'), 173 "lineCommentEndPattern": None, 174 "headerStartLine": "#[[\n", 175 "headerEndLine": "]]\n", 176 "headerLinePrefix": "", 177 "headerLineSuffix": None 178 }, 179 "markdown": { 180 "extensions": [".md"], 181 "keepFirst": None, 182 "blockCommentStartPattern": re.compile(r'^\s*<!--'), 183 "blockCommentEndPattern": re.compile(r'-->\s*$'), 184 "lineCommentStartPattern": None, 185 "lineCommentEndPattern": None, 186 "headerStartLine": "<!--\n", 187 "headerEndLine": "-->\n", 188 "headerLinePrefix": "", 189 "headerLineSuffix": None 190 }, 191 "ruby": { 192 "extensions": [".rb"], 193 "keepFirst": re.compile(r'^#!'), 194 "blockCommentStartPattern": re.compile('^=begin'), 195 "blockCommentEndPattern": re.compile(r'^=end'), 196 "lineCommentStartPattern": re.compile(r'^\s*#'), 197 "lineCommentEndPattern": None, 198 "headerStartLine": "##\n", 199 "headerEndLine": "##\n", 200 "headerLinePrefix": "## ", 201 "headerLineSuffix": None 202 }, 203 "vb": { 204 "extensions": [".vb"], 205 "keepFirst": None, 206 "blockCommentStartPattern": None, 207 "blockCommentEndPattern": None, 208 "lineCommentStartPattern": re.compile(r"^\s*\'"), 209 "lineCommentEndPattern": None, 210 "headerStartLine": None, 211 "headerEndLine": None, 212 "headerLinePrefix": "' ", 213 "headerLineSuffix": None 214 }, 215 "erlang": { 216 "extensions": [".erl", ".src", ".config", ".schema"], 217 "keepFirst": None, 218 "blockCommentStartPattern": None, 219 "blockCommentEndPattern": None, 220 "lineCommentStartPattern": None, 221 "lineCommentEndPattern": None, 222 "headerStartLine": "%% -*- erlang -*-\n%% %CopyrightBegin%\n%%\n", 223 "headerEndLine": "%%\n%% %CopyrightEnd%\n\n", 224 "headerLinePrefix": "%% ", 225 "headerLineSuffix": None, 226 }, 227 "html": { 228 "extensions": [".html"], 229 "keepFirst": re.compile(r'^\s*<\!DOCTYPE.*>'), 230 "blockCommentStartPattern": re.compile(r'^\s*<!--'), 231 "blockCommentEndPattern": re.compile(r'-->\s*$'), 232 "lineCommentStartPattern": None, 233 "lineCommentEndPattern": None, 234 "headerStartLine": "<!--\n", 235 "headerEndLine": "-->\n", 236 "headerLinePrefix": "-- ", 237 "headerLineSuffix": None 238 }, 239 "css": { 240 "extensions": [".css", ".scss", ".sass"], 241 "keepFirst": None, 242 "blockCommentStartPattern": re.compile(r'^\s*/\*'), 243 "blockCommentEndPattern": re.compile(r'\*/\s*$'), 244 "lineCommentStartPattern": None, 245 "lineCommentEndPattern": None, 246 "headerStartLine": "/*\n", 247 "headerEndLine": "*/\n", 248 "headerLinePrefix": None, 249 "headerLineSuffix": None 250 }, 251 "docker": { 252 "extensions": [".dockerfile"], 253 "filenames": ["Dockerfile"], 254 "keepFirst": None, 255 "blockCommentStartPattern": None, 256 "blockCommentEndPattern": None, 257 "lineCommentStartPattern": re.compile(r'^\s*#'), 258 "lineCommentEndPattern": None, 259 "headerStartLine": "##\n", 260 "headerEndLine": "##\n", 261 "headerLinePrefix": "## ", 262 "headerLineSuffix": None 263 }, 264 "yaml": { 265 "extensions": [".yaml", ".yml"], 266 "keepFirst": None, 267 "blockCommentStartPattern": None, 268 "blockCommentEndPattern": None, 269 "lineCommentStartPattern": re.compile(r'^\s*#'), 270 "lineCommentEndPattern": None, 271 "headerStartLine": "##\n", 272 "headerEndLine": "##\n", 273 "headerLinePrefix": "## ", 274 "headerLineSuffix": None 275 }, 276 "zig": { 277 "extensions": [".zig"], 278 "keepFirst": None, 279 "blockCommentStartPattern": None, 280 "blockCommentEndPattern": None, 281 "lineCommentStartPattern": re.compile(r'^\s*//'), 282 "lineCommentEndPattern": None, 283 "headerStartLine": "//!\n", 284 "headerEndLine": "//!\n", 285 "headerLinePrefix": "//! ", 286 "headerLineSuffix": None 287 }, 288 "proto": { 289 "extensions": [".proto"], 290 "keepFirst": None, 291 "blockCommentStartPattern": None, 292 "blockCommentEndPattern": None, 293 "lineCommentStartPattern": re.compile(r'^\s*//'), 294 "lineCommentEndPattern": None, 295 "headerStartLine": None, 296 "headerEndLine": None, 297 "headerLinePrefix": "// ", 298 "headerLineSuffix": None 299 }, 300 "terraform": { 301 "extensions": [".tf"], 302 "keepFirst": None, 303 "blockCommentStartPattern": None, 304 "blockCommentEndPattern": None, 305 "lineCommentStartPattern": re.compile(r'^\s*#'), 306 "lineCommentEndPattern": None, 307 "headerStartLine": "##\n", 308 "headerEndLine": "##\n", 309 "headerLinePrefix": "## ", 310 "headerLineSuffix": None 311 }, 312 "bat": { 313 "extensions": [".bat"], 314 "keepFirst": None, 315 "blockCommentStartPattern": None, 316 "blockCommentEndPattern": None, 317 "lineCommentStartPattern": re.compile(r'^\s*::'), 318 "lineCommentEndPattern": None, 319 "headerStartLine": "::\n", 320 "headerEndLine": "::\n", 321 "headerLinePrefix": ":: ", 322 "headerLineSuffix": None 323 }, 324 "ocaml": { 325 "extensions": [".ml", ".mli", ".mlg", ".v"], 326 "keepFirst": None, 327 "blockCommentStartPattern": re.compile(r'^\s*\(\*'), 328 "blockCommentEndPattern": re.compile(r'\*\)\s*$'), 329 "lineCommentStartPattern": None, 330 "lineCommentEndPattern": None, 331 "headerStartLine": "(*\n", 332 "headerEndLine": " *)\n", 333 "headerLinePrefix": " * ", 334 "headerLineSuffix": None 335 } 336 } 337 338 yearsPattern = re.compile( 339 r"(?<=Copyright\s*(?:\(\s*[Cc©]\s*\)\s*))?([0-9][0-9][0-9][0-9](?:-[0-9][0-9]?[0-9]?[0-9]?)?)", 340 re.IGNORECASE) 341 licensePattern = re.compile(r"license", re.IGNORECASE) 342 emptyPattern = re.compile(r'^\s*$') 343 344 # maps each extension to its processing type. Filled from tpeSettings during initialization 345 ext2type = {} 346 name2type = {} 347 patterns = [] 348 349 350 # class for dict args. Use --argname key1=val1,val2 key2=val3 key3=val4, val5 351 class DictArgs(argparse.Action): 352 def __call__(self, parser, namespace, values, option_string=None): 353 dict_args = {} 354 if not isinstance(values, (list,)): 355 values = (values,) 356 for value in values: 357 n, v = value.split("=") 358 if n not in TYPE_SETTINGS: 359 LOGGER.error("No valid language '%s' to add additional file extensions for" % n) 360 if v and "," in str(v): 361 dict_args[n] = v.split(",") 362 else: 363 dict_args[n] = list() 364 dict_args[n].append(str(v).strip()) 365 setattr(namespace, self.dest, dict_args) 366 367 368 def parse_command_line(argv): 369 """ 370 Parse command line argument. See -h option. 371 :param argv: the actual program arguments 372 :return: parsed arguments 373 """ 374 import textwrap 375 376 377 known_extensions = [ftype+":"+",".join(conf["extensions"]) for ftype, conf in TYPE_SETTINGS.items() if "extensions" in conf] 378 # known_extensions = [ext for ftype in typeSettings.values() for ext in ftype["extensions"]] 379 380 example = textwrap.dedent(""" 381 Known extensions: {0} 382 383 If -t/--tmpl is specified, that header is added to (or existing header replaced for) all source files of known type 384 If -t/--tmpl is not specified byt -y/--years is specified, all years in existing header files 385 are replaced with the years specified 386 387 Examples: 388 {1} -t lgpl-v3 -y 2012-2014 -o ThisNiceCompany -n ProjectName -u http://the.projectname.com 389 {1} -y 2012-2015 390 {1} -y 2012-2015 -d /dir/where/to/start/ 391 {1} -y 2012-2015 -d /dir/where/to/start/ --additional-extensions python=.j2 392 {1} -y 2012-2015 -d /dir/where/to/start/ --additional-extensions python=.j2,.tpl script=.txt 393 {1} -t .copyright.tmpl -cy 394 {1} -t .copyright.tmpl -cy -f some_file.cpp 395 """).format(known_extensions, os.path.basename(argv[0])) 396 formatter_class = argparse.RawDescriptionHelpFormatter 397 parser = argparse.ArgumentParser(description="Python license header updater", 398 epilog=example, 399 formatter_class=formatter_class) 400 parser.add_argument("-V", "--version", action="version", 401 version="%(prog)s {}".format(__version__)) 402 parser.add_argument("-v", "--verbose", dest="verbose_count", 403 action="count", default=0, 404 help="increases log verbosity (can be specified " 405 "1 to 3 times, default shows errors only)") 406 parser.add_argument("-d", "--dir", dest="dir", default=default_dir, 407 help="The directory to recursively process (default: {}).".format(default_dir)) 408 parser.add_argument("-f", "--files", dest="files", nargs='*', type=str, 409 help="The list of files to process. If not empty - will disable '--dir' option") 410 parser.add_argument("-b", action="store_true", 411 help="Back up all files which get changed to a copy with .bak added to the name") 412 parser.add_argument("-t", "--tmpl", dest="tmpl", default=None, 413 help="Template name or file to use.") 414 parser.add_argument("-s", "--settings", dest="settings", default=None, 415 help="Settings file to use.") 416 parser.add_argument("-y", "--years", dest="years", default=None, 417 help="Year or year range to use.") 418 parser.add_argument("-cy", "--current-year", dest="current_year", action="store_true", 419 help="Use today's year.") 420 parser.add_argument("-o", "--owner", dest="owner", default=None, 421 help="Name of copyright owner to use.") 422 parser.add_argument("-n", "--projname", dest="projectname", default=None, 423 help="Name of project to use.") 424 parser.add_argument("-u", "--projurl", dest="projecturl", default=None, 425 help="Url of project to use.") 426 parser.add_argument("--enc", nargs=1, dest="encoding", default=default_encoding, 427 help="Encoding of program files (default: {})".format(default_encoding)) 428 parser.add_argument("--dry", action="store_true", help="Only show what would get done, do not change any files") 429 parser.add_argument("--safesubst", action="store_true", 430 help="Do not raise error if template variables cannot be substituted.") 431 parser.add_argument("-D", "--debug", action="store_true", help="Enable debug messages (same as -v -v -v)") 432 parser.add_argument("-E","--ext", type=str, nargs="*", 433 help="If specified, restrict processing to the specified extension(s) only") 434 parser.add_argument("--additional-extensions", dest="additional_extensions", default=None, nargs="+", 435 help="Provide a comma-separated list of additional file extensions as value for a " 436 "specified language as key, each with a leading dot and no whitespace (default: None).", 437 action=DictArgs) 438 parser.add_argument("-x", "--exclude", type=str, nargs="*", 439 help="File path patterns to exclude") 440 parser.add_argument("--force-overwrite", action="store_true", dest="force_overwrite", 441 help="Try to include headers even in read-only files, given sufficient permissions. " 442 "File permissions are restored after successful header injection.") 443 arguments = parser.parse_args(argv[1:]) 444 445 # Sets log level to WARN going more verbose for each new -V. 446 loglevel = max(4 - arguments.verbose_count, 1) * 10 447 global LOGGER 448 LOGGER.setLevel(loglevel) 449 if arguments.debug: 450 LOGGER.setLevel(logging.DEBUG) 451 # fmt = logging.Formatter('%(asctime)s|%(levelname)s|%(name)s|%(message)s') 452 fmt = logging.Formatter('%(name)s %(levelname)s: %(message)s') 453 hndlr = logging.StreamHandler(sys.stderr) 454 hndlr.setFormatter(fmt) 455 LOGGER.addHandler(hndlr) 456 457 return arguments 458 459 460 def read_type_settings(path): 461 def handle_regex(setting, name): 462 if setting[name]: 463 setting[name] = re.compile(setting[name]) 464 else: 465 setting[name] = None 466 467 def handle_line(setting, name): 468 if setting[name]: 469 setting[name] = setting[name] 470 else: 471 setting[name] = None 472 473 settings = {} 474 475 import json 476 with open(path) as f: 477 data = json.load(f) 478 for key, value in data.items(): 479 for setting_name in ["keepFirst", "blockCommentStartPattern", "blockCommentEndPattern", "lineCommentStartPattern", "lineCommentEndPattern"]: 480 handle_regex(value, setting_name) 481 482 for setting_name in ["headerStartLine", "headerEndLine", "headerLinePrefix", "headerLineSuffix"]: 483 handle_line(value, setting_name) 484 485 settings[key] = value 486 487 return settings 488 489 def get_paths(fnpatterns, start_dir=default_dir): 490 """ 491 Retrieve files that match any of the glob patterns from the start_dir and below. 492 :param fnpatterns: the file name patterns 493 :param start_dir: directory where to start searching 494 :return: generator that returns one path after the other 495 """ 496 seen = set() 497 for root, dirs, files in os.walk(start_dir): 498 names = [] 499 for pattern in fnpatterns: 500 names += fnmatch.filter(files, pattern) 501 for name in names: 502 path = os.path.join(root, name) 503 if path in seen: 504 continue 505 seen.add(path) 506 yield path 507 508 def get_files(fnpatterns, files): 509 seen = set() 510 names = [] 511 for f in files: 512 file_name = os.path.basename(f) 513 for pattern in fnpatterns: 514 if fnmatch.filter([file_name], pattern): 515 names += [f] 516 517 for path in names: 518 if path in seen: 519 continue 520 seen.add(path) 521 yield path 522 523 def read_template(template_file, vardict, args): 524 """ 525 Read a template file replace variables from the dict and return the lines. 526 Throws exception if a variable cannot be replaced. 527 :param template_file: template file with variables 528 :param vardict: dictionary to replace variables with values 529 :param args: the program arguments 530 :return: lines of the template, with variables replaced 531 """ 532 with open(template_file, 'r') as f: 533 lines = f.readlines() 534 if args.safesubst: 535 lines = [Template(line).safe_substitute(vardict) for line in lines] 536 else: 537 lines = [Template(line).substitute(vardict) for line in lines] 538 return lines 539 540 541 def for_type(templatelines, ftype, settings): 542 """ 543 Format the template lines for the given ftype. 544 :param templatelines: the lines of the template text 545 :param ftype: file type 546 :return: header lines 547 """ 548 lines = [] 549 settings = settings[ftype] 550 header_start_line = settings["headerStartLine"] 551 header_end_line = settings["headerEndLine"] 552 header_line_prefix = settings["headerLinePrefix"] 553 header_line_suffix = settings["headerLineSuffix"] 554 if header_start_line is not None: 555 lines.append(header_start_line) 556 for line in templatelines: 557 tmp = line 558 if header_line_prefix is not None and line == '\n': 559 tmp = header_line_prefix.rstrip() + tmp 560 elif header_line_prefix is not None: 561 tmp = header_line_prefix + tmp 562 if header_line_suffix is not None: 563 tmp = tmp + header_line_suffix 564 lines.append(tmp) 565 if header_end_line is not None: 566 lines.append(header_end_line) 567 return lines 568 569 570 ## 571 def read_file(file, args, type_settings): 572 """ 573 Read a file and return a dictionary with the following elements: 574 :param file: the file to read 575 :param args: the options specified by the user 576 :return: a dictionary with the following entries or None if the file is not supported: 577 - skip: number of lines at the beginning to skip (always keep them when replacing or adding something) 578 can also be seen as the index of the first line not to skip 579 - headStart: index of first line of detected header, or None if non header detected 580 - headEnd: index of last line of detected header, or None 581 - yearsLine: index of line which contains the copyright years, or None 582 - haveLicense: found a line that matches a pattern that indicates this could be a license header 583 - settings: the type settings 584 """ 585 skip = 0 586 head_start = None 587 head_end = None 588 years_line = None 589 have_license = False 590 filename, extension = os.path.splitext(file) 591 LOGGER.debug("File name is %s", os.path.basename(filename)) 592 LOGGER.debug("File extension is %s", extension) 593 # if we have no entry in the mapping from extensions to processing type, return None 594 ftype = ext2type.get(extension) 595 LOGGER.debug("Type for this file is %s", ftype) 596 if not ftype: 597 ftype = name2type.get(os.path.basename(file)) 598 if not ftype: 599 return None 600 settings = type_settings.get(ftype) 601 if not os.access(file, os.R_OK): 602 LOGGER.error("File %s is not readable.", file) 603 with open(file, 'r', encoding=args.encoding) as f: 604 lines = f.readlines() 605 # now iterate throw the lines and try to determine the various indies 606 # first try to find the start of the header: skip over shebang or empty lines 607 keep_first = settings.get("keepFirst") 608 isBlockHeader = False 609 block_comment_start_pattern = settings.get("blockCommentStartPattern") 610 block_comment_end_pattern = settings.get("blockCommentEndPattern") 611 line_comment_start_pattern = settings.get("lineCommentStartPattern") 612 i = 0 613 LOGGER.info("Processing file {} as {}".format(file, ftype)) 614 for line in lines: 615 if (i == 0 or i == skip) and keep_first and keep_first.findall(line): 616 skip = i + 1 617 elif emptyPattern.findall(line): 618 pass 619 elif block_comment_start_pattern and block_comment_start_pattern.findall(line): 620 head_start = i 621 isBlockHeader = True 622 break 623 elif line_comment_start_pattern and line_comment_start_pattern.findall(line): 624 head_start = i 625 break 626 elif not block_comment_start_pattern and \ 627 line_comment_start_pattern and \ 628 line_comment_start_pattern.findall(line): 629 head_start = i 630 break 631 else: 632 # we have reached something else, so no header in this file 633 # logging.debug("Did not find the start giving up at line %s, line is >%s<",i,line) 634 return {"type": ftype, 635 "lines": lines, 636 "skip": skip, 637 "headStart": None, 638 "headEnd": None, 639 "yearsLine": None, 640 "settings": settings, 641 "haveLicense": have_license 642 } 643 i = i + 1 644 LOGGER.debug("Found preliminary start at {}, i={}, lines={}".format(head_start, i, len(lines))) 645 # now we have either reached the end, or we are at a line where a block start or line comment occurred 646 # if we have reached the end, return default dictionary without info 647 if i == len(lines): 648 LOGGER.debug("We have reached the end, did not find anything really") 649 return {"type": ftype, 650 "lines": lines, 651 "skip": skip, 652 "headStart": head_start, 653 "headEnd": head_end, 654 "yearsLine": years_line, 655 "settings": settings, 656 "haveLicense": have_license 657 } 658 # otherwise process the comment block until it ends 659 if isBlockHeader: 660 LOGGER.debug("Found comment start, process until end") 661 for j in range(i, len(lines)): 662 LOGGER.debug("Checking line {}".format(j)) 663 if licensePattern.findall(lines[j]): 664 have_license = True 665 elif block_comment_end_pattern.findall(lines[j]): 666 return {"type": ftype, 667 "lines": lines, 668 "skip": skip, 669 "headStart": head_start, 670 "headEnd": j, 671 "yearsLine": years_line, 672 "settings": settings, 673 "haveLicense": have_license 674 } 675 elif yearsPattern.findall(lines[j]): 676 have_license = True 677 years_line = j 678 # if we went through all the lines without finding an end, maybe we have some syntax error or some other 679 # unusual situation, so lets return no header 680 LOGGER.debug("Did not find the end of a block comment, returning no header") 681 return {"type": ftype, 682 "lines": lines, 683 "skip": skip, 684 "headStart": None, 685 "headEnd": None, 686 "yearsLine": None, 687 "settings": settings, 688 "haveLicense": have_license 689 } 690 else: 691 LOGGER.debug("ELSE1") 692 for j in range(i, len(lines)): 693 if line_comment_start_pattern.findall(lines[j]) and licensePattern.findall(lines[j]): 694 have_license = True 695 elif not line_comment_start_pattern.findall(lines[j]): 696 LOGGER.debug("ELSE2") 697 return {"type": ftype, 698 "lines": lines, 699 "skip": skip, 700 "headStart": i, 701 "headEnd": j - 1, 702 "yearsLine": years_line, 703 "settings": settings, 704 "haveLicense": have_license 705 } 706 elif yearsPattern.findall(lines[j]): 707 have_license = True 708 years_line = j 709 # if we went through all the lines without finding the end of the block, it could be that the whole 710 # file only consisted of the header, so lets return the last line index 711 LOGGER.debug("RETURN") 712 return {"type": ftype, 713 "lines": lines, 714 "skip": skip, 715 "headStart": i, 716 "headEnd": len(lines) - 1, 717 "yearsLine": years_line, 718 "settings": settings, 719 "haveLicense": have_license 720 } 721 722 723 def make_backup(file, arguments): 724 """ 725 Backup file by copying it to a file with the extension .bak appended to the name. 726 :param file: file to back up 727 :param arguments: program args, only backs up, if required by an option 728 :return: 729 """ 730 if arguments.b: 731 LOGGER.info("Backing up file {} to {}".format(file, file + ".bak")) 732 if not arguments.dry: 733 copyfile(file, file + ".bak") 734 735 736 class OpenAsWriteable(object): 737 """ 738 This contextmanager wraps standard open(file, 'w', encoding=...) using 739 arguments.encoding encoding. If file cannot be written (read-only file), 740 and if args.force_overwrite is set, try to alter the owner write flag before 741 yielding the file handle. On exit, file permissions are restored to original 742 permissions on __exit__ . If the file does not exist, or if it is read-only 743 and cannot be made writable (due to lacking user rights or force_overwrite 744 argument not being set), this contextmanager yields None on __enter__. 745 """ 746 747 def __init__(self, filename, arguments): 748 """ 749 Initialize an OpenAsWriteable context manager 750 :param filename: path to the file to open 751 :param arguments: program arguments 752 """ 753 self._filename = filename 754 self._arguments = arguments 755 self._file_handle = None 756 self._file_permissions = None 757 758 def __enter__(self): 759 """ 760 Yields a writable file handle when possible, else None. 761 """ 762 filename = self._filename 763 arguments = self._arguments 764 file_handle = None 765 file_permissions = None 766 767 if os.path.isfile(filename): 768 file_permissions = stat.S_IMODE(os.lstat(filename).st_mode) 769 770 if not os.access(filename, os.W_OK): 771 if arguments.force_overwrite: 772 try: 773 os.chmod(filename, file_permissions | stat.S_IWUSR) 774 except PermissionError: 775 LOGGER.warning("File {} cannot be made writable, it will be skipped.".format(filename)) 776 else: 777 LOGGER.warning("File {} is not writable, it will be skipped.".format(filename)) 778 779 if os.access(filename, os.W_OK): 780 file_handle = open(filename, 'w', encoding=arguments.encoding) 781 else: 782 LOGGER.warning("File {} does not exist, it will be skipped.".format(filename)) 783 784 self._file_handle = file_handle 785 self._file_permissions = file_permissions 786 787 return file_handle 788 789 def __exit__ (self, exc_type, exc_value, traceback): 790 """ 791 Restore back file permissions and close file handle (if any). 792 """ 793 if (self._file_handle is not None): 794 self._file_handle.close() 795 796 actual_permissions = stat.S_IMODE(os.lstat(self._filename).st_mode) 797 if (actual_permissions != self._file_permissions): 798 try: 799 os.chmod(self._filename, self._file_permissions) 800 except PermissionError: 801 LOGGER.error("File {} permissions could not be restored.".format(self._filename)) 802 803 self._file_handle = None 804 self._file_permissions = None 805 return True 806 807 808 @contextlib.contextmanager 809 def open_as_writable(file, arguments): 810 """ 811 Wrapper around OpenAsWriteable context manager. 812 """ 813 with OpenAsWriteable(file, arguments=arguments) as fw: 814 yield fw 815 816 817 def main(): 818 """Main function.""" 819 # LOGGER.addHandler(logging.StreamHandler(stream=sys.stderr)) 820 # init: create the ext2type mappings 821 arguments = parse_command_line(sys.argv) 822 additional_extensions = arguments.additional_extensions 823 824 type_settings = TYPE_SETTINGS 825 if arguments.settings: 826 type_settings = read_type_settings(arguments.settings) 827 828 for t in type_settings: 829 settings = type_settings[t] 830 exts = settings["extensions"] 831 if "filenames" in settings: 832 names = settings['filenames'] 833 else: 834 names = [] 835 # if additional file extensions are provided by the user, they are "merged" here: 836 if additional_extensions and t in additional_extensions: 837 for aext in additional_extensions[t]: 838 LOGGER.debug("Enable custom file extension '%s' for language '%s'" % (aext, t)) 839 exts.append(aext) 840 841 for ext in exts: 842 ext2type[ext] = t 843 patterns.append("*" + ext) 844 845 for name in names: 846 name2type[name] = t 847 patterns.append(name) 848 849 LOGGER.debug("Allowed file patterns %s" % patterns) 850 851 limit2exts = None 852 if arguments.ext is not None and len(arguments.ext) > 0: 853 limit2exts = arguments.ext 854 855 try: 856 error = False 857 template_lines = None 858 if arguments.dir is not default_dir and arguments.files: 859 LOGGER.error("Cannot use both '--dir' and '--files' options.") 860 error = True 861 862 if arguments.years and arguments.current_year: 863 LOGGER.error("Cannot use both '--years' and '--currentyear' options.") 864 error = True 865 866 years = arguments.years 867 if arguments.current_year: 868 import datetime 869 now = datetime.datetime.now() 870 years = str(now.year) 871 872 settings = {} 873 if years: 874 settings["years"] = years 875 if arguments.owner: 876 settings["owner"] = arguments.owner 877 if arguments.projectname: 878 settings["projectname"] = arguments.projectname 879 if arguments.projecturl: 880 settings["projecturl"] = arguments.projecturl 881 # if we have a template name specified, try to get or load the template 882 if arguments.tmpl: 883 opt_tmpl = arguments.tmpl 884 # first get all the names of our own templates 885 # for this get first the path of this file 886 templates_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "templates") 887 LOGGER.debug("File path: {}".format(os.path.abspath(__file__))) 888 # get all the templates in the templates directory 889 templates = [f for f in get_paths("*.tmpl", templates_dir)] 890 templates = [(os.path.splitext(os.path.basename(t))[0], t) for t in templates] 891 # filter by trying to match the name against what was specified 892 tmpls = [t for t in templates if opt_tmpl in t[0]] 893 # check if one of the matching template names is identical to the parameter, then take that one 894 tmpls_eq = [t for t in tmpls if opt_tmpl == t[0]] 895 if len(tmpls_eq) > 0: 896 tmpls = tmpls_eq 897 if len(tmpls) == 1: 898 tmpl_name = tmpls[0][0] 899 tmpl_file = tmpls[0][1] 900 LOGGER.info("Using template file {} for {}".format(tmpl_file, tmpl_name)) 901 template_lines = read_template(tmpl_file, settings, arguments) 902 else: 903 if len(tmpls) == 0: 904 # check if we can interpret the option as file 905 if os.path.isfile(opt_tmpl): 906 LOGGER.info("Using file {}".format(os.path.abspath(opt_tmpl))) 907 template_lines = read_template(os.path.abspath(opt_tmpl), settings, arguments) 908 else: 909 LOGGER.error("Not a built-in template and not a file, cannot proceed: {}".format(opt_tmpl)) 910 LOGGER.error("Built in templates: {}".format(", ".join([t[0] for t in templates]))) 911 error = True 912 else: 913 LOGGER.error("There are multiple matching template names: {}".format([t[0] for t in tmpls])) 914 error = True 915 else: 916 # no tmpl parameter 917 if not years: 918 LOGGER.error("No template specified and no years either, nothing to do (use -h option for usage info)") 919 error = True 920 if error: 921 return 1 922 else: 923 # logging.debug("Got template lines: %s",templateLines) 924 # now do the actual processing: if we did not get some error, we have a template loaded or 925 # no template at all 926 # if we have no template, then we will have the years. 927 # now process all the files and either replace the years or replace/add the header 928 if arguments.files: 929 LOGGER.debug("Processing files %s", arguments.files) 930 LOGGER.debug("Patterns: %s", patterns) 931 paths = get_files(patterns, arguments.files) 932 else: 933 LOGGER.debug("Processing directory %s", arguments.dir) 934 LOGGER.debug("Patterns: %s", patterns) 935 paths = get_paths(patterns, arguments.dir) 936 937 for file in paths: 938 LOGGER.debug("Considering file: {}".format(file)) 939 file = os.path.normpath(file) 940 if limit2exts is not None and not any([file.endswith(ext) for ext in limit2exts]): 941 LOGGER.info("Skipping file with non-matching extension: {}".format(file)) 942 continue 943 if arguments.exclude and any([fnmatch.fnmatch(file, pat) for pat in arguments.exclude]): 944 LOGGER.info("Ignoring file {}".format(file)) 945 continue 946 finfo = read_file(file, arguments, type_settings) 947 if not finfo: 948 LOGGER.debug("File not supported %s", file) 949 continue 950 # logging.debug("FINFO for the file: %s", finfo) 951 lines = finfo["lines"] 952 LOGGER.debug( 953 "Info for the file: headStart=%s, headEnd=%s, haveLicense=%s, skip=%s, len=%s, yearsline=%s", 954 finfo["headStart"], finfo["headEnd"], finfo["haveLicense"], finfo["skip"], len(lines), 955 finfo["yearsLine"]) 956 # if we have a template: replace or add 957 if template_lines: 958 make_backup(file, arguments) 959 if arguments.dry: 960 LOGGER.info("Would be updating changed file: {}".format(file)) 961 else: 962 with open_as_writable(file, arguments) as fw: 963 if (fw is not None): 964 # if we found a header, replace it 965 # otherwise, add it after the lines to skip 966 head_start = finfo["headStart"] 967 head_end = finfo["headEnd"] 968 have_license = finfo["haveLicense"] 969 ftype = finfo["type"] 970 skip = finfo["skip"] 971 if head_start is not None and head_end is not None and have_license: 972 LOGGER.debug("Replacing header in file {}".format(file)) 973 # first write the lines before the header 974 fw.writelines(lines[0:head_start]) 975 # now write the new header from the template lines 976 fw.writelines(for_type(template_lines, ftype, type_settings)) 977 # now write the rest of the lines 978 fw.writelines(lines[head_end + 1:]) 979 else: 980 LOGGER.debug("Adding header to file {}, skip={}".format(file, skip)) 981 fw.writelines(lines[0:skip]) 982 fw.writelines(for_type(template_lines, ftype, type_settings)) 983 if head_start is not None and not have_license: 984 # There is some header, but not license - add an empty line 985 fw.write("\n") 986 fw.writelines(lines[skip:]) 987 # TODO: optionally remove backup if all worked well? 988 else: 989 # no template lines, just update the line with the year, if we found a year 990 years_line = finfo["yearsLine"] 991 if years_line is not None: 992 make_backup(file, arguments) 993 if arguments.dry: 994 LOGGER.info("Would be updating year line in file {}".format(file)) 995 else: 996 with open_as_writable(file, arguments) as fw: 997 if (fw is not None): 998 LOGGER.debug("Updating years in file {} in line {}".format(file, years_line)) 999 fw.writelines(lines[0:years_line]) 1000 fw.write(yearsPattern.sub(years, lines[years_line])) 1001 fw.writelines(lines[years_line + 1:]) 1002 # TODO: optionally remove backup if all worked well 1003 return 0 1004 finally: 1005 logging.shutdown() 1006 1007 1008 if __name__ == "__main__": 1009 sys.exit(main())