vitess.io/vitess@v0.16.2/doc/vtctl_go_reference.py (about) 1 #!/usr/bin/python 2 3 # Copyright 2018 The Vitess Authors. 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 """ 18 """ 19 20 import json 21 import os 22 import optparse 23 import re 24 25 # TODO: Handle angle brackets that appear in command definitions -- 26 # e.g. ChangeSlaveType 27 28 29 # Traverse directory to get list of all files in the directory. 30 def get_all_files(directory, filenames): 31 os.chdir(directory) 32 for path, dirs, files in os.walk('.'): 33 for filename in files: 34 filenames[os.path.join(path, filename)] = True 35 return filenames 36 37 # This needs to produce the same anchor ID as the Markdown processor. 38 def anchor_id(heading): 39 return heading.lower().replace(' ', '-').replace(',', '') 40 41 def write_header(doc, commands): 42 doc.write('This reference guide explains the commands that the ' + 43 '<b>vtctl</b> tool supports. **vtctl** is a command-line tool ' + 44 'used to administer a Vitess cluster, and it allows a human ' + 45 'or application to easily interact with a Vitess ' + 46 'implementation.\n\nCommands are listed in the ' + 47 'following groups:\n\n') 48 for group in sorted(commands): 49 group_link = anchor_id(group) 50 doc.write('* [' + group + '](#' + group_link + ')\n') 51 doc.write('\n\n') 52 53 def write_footer(doc): 54 doc.write(' </body>\n') 55 doc.write('</html>\n') 56 57 def create_reference_doc(root_directory, commands, arg_definitions): 58 doc = open(root_directory + '/doc/vtctlReference.md', 'w') 59 write_header(doc, commands) 60 61 not_found_arguments = {} 62 for group in sorted(commands): 63 doc.write('## ' + group + '\n\n'); 64 for command in sorted(commands[group]): 65 if ('definition' in commands[group][command] and 66 commands[group][command]['definition'] != '' and 67 re.search(r'HIDDEN', commands[group][command]['definition'])): 68 print '\n\n****** ' + command + ' is hidden *******\n\n' 69 continue 70 command_link = anchor_id(command) 71 doc.write('* [' + command + '](#' + command_link + ')\n') 72 73 doc.write('\n') 74 75 for command in sorted(commands[group]): 76 if ('definition' in commands[group][command] and 77 commands[group][command]['definition'] != '' and 78 re.search(r'HIDDEN', commands[group][command]['definition'])): 79 continue 80 doc.write('### ' + command + '\n\n'); 81 if ('definition' in commands[group][command] and 82 commands[group][command]['definition'] != ''): 83 commands[group][command]['definition'] = ( 84 commands[group][command]['definition'].replace('<', '<')) 85 commands[group][command]['definition'] = ( 86 commands[group][command]['definition'].replace('>', '>')) 87 commands[group][command]['definition'] = ( 88 commands[group][command]['definition'].replace('</a>', 89 '</a>')) 90 commands[group][command]['definition'] = ( 91 commands[group][command]['definition'].replace('<a href', 92 '<a href')) 93 commands[group][command]['definition'] = ( 94 commands[group][command]['definition'].replace('">', '">')) 95 commands[group][command]['definition'] = ( 96 commands[group][command]['definition'].replace('"<br>', 97 '<br>')) 98 commands[group][command]['definition'] = ( 99 commands[group][command]['definition'].replace('\\n', '<br><br>')) 100 doc.write(commands[group][command]['definition'] + '\n\n') 101 102 if ('arguments' in commands[group][command] and 103 commands[group][command]['arguments']): 104 doc.write('#### Example\n\n') 105 arguments = commands[group][command]['arguments'].strip().strip( 106 '"').replace('<', '<') 107 arguments = arguments.strip().replace('>', '>') 108 doc.write('<pre class="command-example">%s %s</pre>\n\n' % 109 (command, arguments)) 110 111 if ('argument_list' in commands[group][command] and 112 'flags' in commands[group][command]['argument_list']): 113 flag_text = '' 114 for command_flag in sorted( 115 commands[group][command]['argument_list']['flags']): 116 flag = ( 117 commands[group][command]['argument_list']['flags'][command_flag]) 118 flag_text += ('| %s | %s | %s |\n' % (command_flag, flag['type'], 119 flag['definition'])) 120 121 if flag_text: 122 #flag_text = '<a name="' + command + '-flags"></a>\n\n' + 123 flag_text = ('| Name | Type | Definition |\n' + 124 '| :-------- | :--------- | :--------- |\n' + 125 flag_text + '\n') 126 doc.write('#### Flags\n\n' + flag_text + '\n') 127 128 if ('argument_list' in commands[group][command] and 129 'args' in commands[group][command]['argument_list']): 130 arg_text = '' 131 132 for arg in commands[group][command]['argument_list']['args']: 133 if 'name' in arg: 134 arg_name = arg['name'] 135 new_arg_name = arg['name'].replace('<', '').replace('>', '') 136 if (new_arg_name[0:len(new_arg_name) - 1] in arg_definitions and 137 re.search(r'\d', new_arg_name[-1])): 138 arg_name = '<' + new_arg_name[0:len(new_arg_name) - 1] + '>' 139 arg_name = arg_name.strip().replace('<', 'START_CODE_TAG<') 140 arg_name = arg_name.strip().replace('>', '>END_CODE_TAG') 141 arg_text += '* ' + arg_name + ' – ' 142 if 'required' in arg and arg['required']: 143 arg_text += 'Required.' 144 else: 145 arg_text += 'Optional.' 146 147 arg_values = None 148 temp_name = arg['name'].replace('<', '').replace('>', '') 149 if temp_name in arg_definitions: 150 arg_text += ' ' + arg_definitions[temp_name]['description'] 151 if 'list_items' in arg_definitions[temp_name]: 152 arg_values = arg_definitions[temp_name]['list_items'] 153 # Check if the argument name ends in a digit to catch things like 154 # keyspace1 being used to identify the first in a list of keyspaces. 155 elif (temp_name[0:len(temp_name) - 1] in arg_definitions and 156 re.search(r'\d', temp_name[-1])): 157 arg_length = len(temp_name) - 1 158 arg_text += (' ' + 159 arg_definitions[temp_name[0:arg_length]]['description']) 160 if 'list_items' in arg_definitions[temp_name[0:arg_length]]: 161 arg_values = ( 162 arg_definitions[temp_name[0:arg_length]]['list_items']) 163 else: 164 not_found_arguments[arg['name']] = True 165 if arg_values: 166 arg_text += '\n\n' 167 for arg_value in sorted(arg_values, key=lambda k: k['value']): 168 arg_text += (' * <code>' + arg_value['value'] + '</code> ' + 169 '– ' + arg_value['definition'] + '\n') 170 arg_text += '\n\n' 171 172 if 'multiple' in arg: 173 separation = 'space' 174 if 'hasComma' in arg: 175 separation = 'comma' 176 arg_text += (' To specify multiple values for this argument, ' + 177 'separate individual values with a ' + 178 separation + '.') 179 arg_text += '\n' 180 if arg_text: 181 arg_text = arg_text.replace('START_CODE_TAG', '<code>') 182 arg_text = arg_text.replace('END_CODE_TAG', '</code>') 183 doc.write('#### Arguments\n\n' + 184 arg_text + 185 '\n') 186 187 if 'errors' in commands[group][command]: 188 errors_text = '' 189 if 'ARG_COUNT' in commands[group][command]['errors']: 190 error = commands[group][command]['errors']['ARG_COUNT'] 191 message = re.sub(str(command), '<' + command + '>', error['message']) 192 #print 'message here' 193 #print message 194 #message = error['message'].replace(command, '<' + command + '>') 195 message = message.replace('<', 'START_CODE_TAG<') 196 message = message.replace('>', '>END_CODE_TAG') 197 errors_text += '* ' + message + ' ' 198 if (error['exact_count'] and 199 len(error['exact_count']) == 1 and 200 error['exact_count'][0] == '1'): 201 errors_text += ('This error occurs if the command is not called ' + 202 'with exactly one argument.') 203 elif error['exact_count'] and len(error['exact_count']) == 1: 204 errors_text += ('This error occurs if the command is not called ' + 205 'with exactly ' + error['exact_count'][0] + ' ' + 206 'arguments.') 207 elif error['exact_count']: 208 allowed_error_counts = ' or '.join(error['exact_count']) 209 errors_text += ('This error occurs if the command is not called ' + 210 'with exactly ' + allowed_error_counts + ' ' + 211 'arguments.') 212 elif error['min_count'] and error['max_count']: 213 errors_text += ('This error occurs if the command is not called ' + 214 'with between ' + error['min_count'] + ' and ' + 215 error['max_count'] + ' arguments.') 216 elif error['min_count'] and error['min_count'] == '1': 217 errors_text += ('This error occurs if the command is not called ' + 218 'with at least one argument.') 219 elif error['min_count']: 220 errors_text += ('This error occurs if the command is not called ' + 221 'with at least ' + error['min_count'] + ' ' + 222 'arguments.') 223 elif error['max_count']: 224 errors_text += ('This error occurs if the command is not called ' + 225 'with more than ' + error['max_count'] + ' ' + 226 'arguments.') 227 errors_text += '\n' 228 229 if 'other' in commands[group][command]['errors']: 230 #print 'other errors' 231 #print commands[group][command]['errors'] 232 for error in commands[group][command]['errors']['other']: 233 error = re.sub(str(command), '<' + command + '>', error) 234 if ('argument_list' in commands[group][command] and 235 'flags' in commands[group][command]['argument_list']): 236 for command_flag in sorted( 237 commands[group][command]['argument_list']['flags']): 238 error = re.sub(str(command_flag), '<' + command_flag + '>', 239 error) 240 241 error = error.replace('<', 'START_CODE_TAG<') 242 error = error.replace('>', '>END_CODE_TAG') 243 errors_text += '* ' + error + '\n' 244 245 if errors_text: 246 errors_text = errors_text.replace('START_CODE_TAG', '<code>') 247 errors_text = errors_text.replace('END_CODE_TAG', '</code>') 248 doc.write('#### Errors\n\n' + errors_text) 249 doc.write('\n\n') 250 #write_footer(doc) 251 doc.close() 252 #print json.dumps(not_found_arguments, sort_keys=True, indent=4) 253 return 254 255 def parse_arg_list(arguments, current_command): 256 last_char = '' 257 258 find_closing_square_bracket = False 259 has_comma = False 260 has_multiple = False 261 is_optional_argument = False 262 is_required_argument = False 263 current_argument = '' 264 265 new_arg_list = [] 266 arg_count = 0 267 char_count = 1 268 for char in arguments: 269 if (last_char == '' or last_char == ' ') and char == '[': 270 find_closing_square_bracket = True 271 elif (last_char == '[' and 272 find_closing_square_bracket and 273 char == '<'): 274 is_optional_argument = True 275 current_argument += char 276 elif find_closing_square_bracket: 277 if char == ']': 278 if is_optional_argument: 279 new_arg_list.append({'name': current_argument, 280 'has_comma': has_comma, 281 'has_multiple': has_multiple, 282 'required': False}) 283 arg_count += 1 284 find_closing_square_bracket = False 285 current_argument = '' 286 has_comma = False 287 has_multiple = False 288 elif is_optional_argument: 289 if char == ',': 290 has_comma = True 291 elif last_char == '.' and char == '.': 292 has_multiple = True 293 elif not has_comma: 294 current_argument += char 295 elif char == '<' and (last_char == '' or last_char == ' '): 296 is_required_argument = True 297 current_argument += char 298 elif char == ',': 299 has_comma = True 300 if last_char == '>': 301 new_arg_list[arg_count - 1]['hasComma'] = True 302 has_comma = False 303 elif is_required_argument: 304 current_argument += char 305 if char == '>' and current_argument: 306 if char_count == len(arguments[0]): 307 new_arg_list.append({'name': current_argument, 308 'required': True}) 309 arg_count += 1 310 is_required_argument = False 311 current_argument = '' 312 else: 313 next_char = 'x' 314 if current_command == 'Resolve': 315 if char_count < len(arguments[0]): 316 next_char = arguments[0][char_count:char_count + 1] 317 318 if next_char and not next_char == '.' and not next_char == ':': 319 new_arg_list.append({'name': current_argument, 320 'required': True}) 321 arg_count += 1 322 is_required_argument = False 323 current_argument = '' 324 elif (arg_count > 0 and 325 current_argument == '' and 326 last_char == '.' and 327 'multiple' in new_arg_list[arg_count - 1] and 328 char == ' '): 329 current_argument = '' 330 elif (arg_count > 0 and 331 current_argument == '' and 332 last_char == '.' and 333 char == '.'): 334 new_arg_list[arg_count - 1]['multiple'] = True 335 336 char_count += 1 337 last_char = char 338 return new_arg_list 339 340 def get_group_name_from_variable(file_path, variable_name): 341 vtctl_go_file = open(file_path, 'rU') 342 vtctl_go_data = vtctl_go_file.readlines() 343 vtctl_go_file.close() 344 345 for line in vtctl_go_data: 346 regex = r'const\s*' + re.escape(variable_name) + r'\s*=\s*\"([^\"]+)\"' 347 if re.search(regex, line): 348 return line.split('=')[1].strip().strip('"') 349 return variable_name 350 351 def main(root_directory): 352 arg_definitions = {} 353 commands = {} 354 command_groups = {} 355 error_counts = {} 356 functions = {} 357 358 # Read the .go files in the /vitess/go/vt/vtctl/ directory 359 vtctl_dir_path = root_directory + '/go/vt/vtctl/' 360 go_files = next(os.walk(vtctl_dir_path))[2] 361 for path in go_files: 362 if not path.endswith('.go'): 363 continue 364 vtctl_go_path = vtctl_dir_path + path 365 vtctl_go_file = open(vtctl_go_path, 'rU') 366 vtctl_go_data = vtctl_go_file.readlines() 367 vtctl_go_file.close() 368 369 add_command_syntax = False 370 get_commands = False 371 get_argument_definitions = False 372 get_wrong_arg_count_error = False 373 get_group_name = False 374 current_command_argument = '' 375 current_command_argument_value = '' 376 current_command = '' 377 current_function = '' 378 is_func_init = False 379 is_flag_section = False 380 381 # treat func init() same as var commands 382 # treat addCommand("Group Name"... same as command {... in vtctl.go group 383 # Reformat Generic Help command to same format as commands in backup.go 384 # and reparent.go. 385 # Add logic to capture command data from those commands. 386 for line in vtctl_go_data: 387 388 # skip comments and empty lines 389 if line.strip() == '' or line.strip().startswith('//'): 390 continue 391 392 if is_func_init and not is_flag_section and re.search(r'^if .+ {', line.strip()): 393 is_flag_section = True 394 elif is_func_init and not is_flag_section and line.strip() == 'servenv.OnRun(func() {': 395 pass 396 elif is_func_init and is_flag_section and line.strip() == 'return': 397 pass 398 elif is_func_init and is_flag_section and line.strip() == '}': 399 is_flag_section = False 400 elif is_func_init and (line.strip() == '}' or line.strip() == '})'): 401 is_func_init = False 402 elif get_commands: 403 # This line precedes a command group's name, e.g. "Tablets" or "Shards." 404 # Capture the group name on the next line. 405 if line.strip() == '{': 406 get_group_name = True 407 # Capture the name of a command group. 408 elif get_group_name: 409 # Regex to identify the group name. Line in code looks like: 410 # "Tablets", []command{ 411 find_group = re.findall(r'\"([^\"]+)\", \[\]\s*command\s*\{', line) 412 if find_group: 413 current_group = find_group[0] 414 if current_group not in commands: 415 commands[current_group] = {} 416 get_group_name = False 417 418 # First line of a command in addCommand syntax. This contains the 419 # name of the group that the command is in. Line in code looks like: 420 # addCommand{"Shards", command{ 421 elif re.search(r'^addCommand\(', line.strip()): 422 command_data = re.findall(r'addCommand\s*\(\s*([^\,]+),\s*command\{', 423 line) 424 if command_data: 425 current_group = command_data[0] 426 current_group_strip_quotes = current_group.strip('"') 427 if current_group == current_group_strip_quotes: 428 current_group = get_group_name_from_variable(vtctl_go_path, 429 current_group) 430 else: 431 current_group = current_group_strip_quotes 432 if current_group not in commands: 433 commands[current_group] = {} 434 add_command_syntax = True 435 436 elif add_command_syntax and is_func_init: 437 if not current_command: 438 current_command = line.strip().strip(',').strip('"') 439 commands[current_group][current_command] = { 440 'definition': '', 441 'argument_list': { 442 'flags': {}, 443 'args': [] 444 }, 445 'errors': {'other': []}} 446 command_groups[current_command] = current_group 447 elif 'function' not in commands[current_group][current_command]: 448 function = line.strip().strip(',') 449 commands[current_group][current_command]['function'] = function 450 functions[function] = current_command 451 elif 'arguments' not in commands[current_group][current_command]: 452 arguments = line.strip().strip(',') 453 commands[current_group][current_command]['arguments'] = arguments 454 if arguments: 455 new_arg_list = parse_arg_list(arguments, current_command) 456 commands[current_group][current_command]['argument_list']['args'] = new_arg_list 457 else: 458 definition_list = line.strip().split(' +') 459 for definition_part in definition_list: 460 definition = definition_part.strip().strip('})') 461 definition = definition.replace('}},', '') 462 definition = definition.replace('"', '') 463 commands[current_group][current_command]['definition'] += ( 464 definition) 465 if line.strip().endswith('})'): 466 current_command = '' 467 # Command definition ends with line ending in "},". 468 elif line.strip().endswith('})'): 469 current_command = '' 470 add_command_syntax = False 471 472 # First line of a command. This contains the command name and the 473 # function used to process the command. Line in code looks like: 474 # command{"ScrapTablet", commandScrapTablet, 475 elif re.search(r'^\{\s*\"[^\"]+\",\s*command[^\,]+\,', line.strip()): 476 command_data = re.findall(r'\{\s*\"([^\"]+)\",\s*([^\,]+)\,', 477 line) 478 if command_data: 479 # Capture the command name and associate it with its function. 480 # Create a data structure to contain information about the command 481 # and its processing rules. 482 current_command = command_data[0][0] 483 function = command_data[0][1] 484 commands[current_group][current_command] = { 485 'definition': '', 486 'argument_list': { 487 'flags': {}, 488 'args': [] 489 }, 490 'errors': {'other': []}} 491 492 # Associate the function with the command and vice versa. 493 # Associate the command with its group and vice versa. 494 commands[current_group][current_command]['function'] = function 495 command_groups[current_command] = current_group 496 functions[function] = current_command 497 498 # If code has identified a command name but has not identified 499 # arguments for that command, capture the next line and store it 500 # as the command arguments. 501 elif (current_command and 502 'arguments' not in commands[current_group][current_command]): 503 arguments = re.findall(r'\"([^\"]+)\"', line) 504 if arguments: 505 commands[current_group][current_command]['arguments'] = arguments[0] 506 last_char = '' 507 508 find_closing_square_bracket = False 509 has_comma = False 510 has_multiple = False 511 is_optional_argument = False 512 is_required_argument = False 513 current_argument = '' 514 515 new_arg_list = [] 516 arg_count = 0 517 char_count = 1 518 for char in arguments[0]: 519 if (last_char == '' or last_char == ' ') and char == '[': 520 find_closing_square_bracket = True 521 elif (last_char == '[' and 522 find_closing_square_bracket and 523 char == '<'): 524 is_optional_argument = True 525 current_argument += char 526 elif find_closing_square_bracket: 527 if char == ']': 528 if is_optional_argument: 529 new_arg_list.append({'name': current_argument, 530 'has_comma': has_comma, 531 'has_multiple': has_multiple, 532 'required': False}) 533 arg_count += 1 534 find_closing_square_bracket = False 535 current_argument = '' 536 has_comma = False 537 has_multiple = False 538 elif is_optional_argument: 539 if char == ',': 540 has_comma = True 541 elif last_char == '.' and char == '.': 542 has_multiple = True 543 elif not has_comma: 544 current_argument += char 545 elif char == '<' and (last_char == '' or last_char == ' '): 546 is_required_argument = True 547 current_argument += char 548 elif char == ',': 549 has_comma = True 550 if last_char == '>': 551 new_arg_list[arg_count - 1]['hasComma'] = True 552 has_comma = False 553 elif is_required_argument: 554 current_argument += char 555 if char == '>' and current_argument: 556 if char_count == len(arguments[0]): 557 new_arg_list.append({'name': current_argument, 558 'required': True}) 559 arg_count += 1 560 is_required_argument = False 561 current_argument = '' 562 else: 563 next_char = 'x' 564 if current_command == 'Resolve': 565 #print len(arguments[0]) 566 #print char_count 567 if char_count < len(arguments[0]): 568 next_char = arguments[0][char_count:char_count + 1] 569 #print next_char 570 571 #if char_count < (len(arguments[0]) - 1): 572 # print current_argument 573 # next_char = arguments[0][char_count + 1:char_count + 2] 574 # print next_char 575 if next_char and not next_char == '.' and not next_char == ':': 576 new_arg_list.append({'name': current_argument, 577 'required': True}) 578 arg_count += 1 579 is_required_argument = False 580 current_argument = '' 581 elif (arg_count > 0 and 582 current_argument == '' and 583 last_char == '.' and 584 'multiple' in new_arg_list[arg_count - 1] and 585 char == ' '): 586 current_argument = '' 587 elif (arg_count > 0 and 588 current_argument == '' and 589 last_char == '.' and 590 char == '.'): 591 new_arg_list[arg_count - 1]['multiple'] = True 592 #elif (arg_count > 0 and 593 # current_argument == '' and 594 # last_char == '.'): 595 596 char_count += 1 597 last_char = char 598 commands[current_group][current_command]['argument_list']['args'] = ( 599 new_arg_list) 600 601 else: 602 commands[current_group][current_command]['arguments'] = "" 603 # If code has identified a command and arguments, capture remaining lines 604 # as the command description. Assume the description ends at the line 605 # of code ending with "},". 606 elif current_command: 607 definition_list = line.rstrip('},').split(' +') 608 for definition_part in definition_list: 609 definition = definition_part.strip().strip('"') 610 #definition = definition.replace('\\n', '<br><br>') 611 definition = definition.replace('"},', '') 612 commands[current_group][current_command]['definition'] += definition 613 if line.strip().endswith('},'): 614 current_command = '' 615 # Command definition ends with line ending in "},". 616 elif line.strip().endswith('},'): 617 current_command = '' 618 619 # Capture information about a function that processes a command. 620 # Here, identify the function name. 621 elif re.search(r'^func ([^\)]+)\(', line.strip()): 622 current_function = '' 623 function_data = re.findall(r'func ([^\)]+)\(', line.strip()) 624 if function_data: 625 current_function = function_data[0] 626 627 elif current_function: 628 # Lines that contain this: 629 # = subFlags.... 630 # generally seem to contain descriptions of flags for the function. 631 # Capture the content type of the argument, its name, default value, 632 # and description. Associate these with the command that calls this 633 # function. 634 if re.search(r'\=\s*subFlags\.([^\(]+)\(\s*\"([^\"]+)\"\,' + 635 '([^\,]+)\,\s*\"([^\"]+)\"', line): 636 argument_data = re.findall(r'\=\s*subFlags\.([^\(]+)' + 637 '\(\s*\"([^\"]+)\"\,([^\,]+)\,\s*\"([^\"]+)\"', line) 638 if argument_data and current_function in functions: 639 [arg_type, arg_name, arg_default, arg_definition] = argument_data[0] 640 if arg_type == 'Bool': 641 arg_type = 'Boolean' 642 if arg_type == 'String' or arg_type == 'int': 643 arg_type = arg_type.lower() 644 645 arg_default = arg_default.strip().strip('"') 646 647 fcommand = functions[current_function] 648 fgroup = command_groups[fcommand] 649 commands[fgroup][fcommand]['argument_list']['flags'][arg_name] = { 650 'type': arg_type, 651 'default': arg_default, 652 'definition': arg_definition 653 } 654 655 elif re.search(r'\s*subFlags\.Var\(\s*([^\,]+)\,\s*\"([^\"]+)\"\,' + 656 '\s*\"([^\"]+)\"', line): 657 #print 'found subFlags.Var' 658 argument_data = re.findall(r'\s*subFlags\.Var\(\s*([^\,]+)\,' + 659 '\s*\"([^\"]+)\"\,\s*\"([^\"]+)\"', line) 660 #print argument_data 661 if argument_data and current_function in functions: 662 [arg_type, arg_name, arg_definition] = argument_data[0] 663 arg_type = 'string' # Var? 664 665 fcommand = functions[current_function] 666 fgroup = command_groups[fcommand] 667 commands[fgroup][fcommand]['argument_list']['flags'][arg_name] = { 668 'type': arg_type, 669 'definition': arg_definition 670 } 671 672 # Capture information for errors that indicate that the command 673 # was called with the incorrect number of arguments. Use the 674 # code to determine whether the code is looking for an exact number 675 # of arguments, a minimum number, a maximum number, etc. 676 elif re.search(r'if subFlags.NArg', line): 677 wrong_arg_data = re.findall( 678 r'subFlags.NArg\(\)\s*([\<\>\!\=]+)\s*(\d+)', line) 679 error_counts[current_function] = {'min': None, 680 'max': None, 681 'exact': []} 682 if wrong_arg_data: 683 get_wrong_arg_count_error = True 684 for wrong_arg_info in wrong_arg_data: 685 if wrong_arg_info[0] == '!=': 686 error_counts[current_function]['exact'].append( 687 wrong_arg_info[1]) 688 elif wrong_arg_info[0] == '<': 689 error_counts[current_function]['min'] = wrong_arg_info[1] 690 elif wrong_arg_info[0] == '>': 691 error_counts[current_function]['max'] = wrong_arg_info[1] 692 elif wrong_arg_info[0] == '==' and wrong_arg_info[1] == '0': 693 error_counts[current_function]['min'] = '1' 694 695 # Capture data about other errors that the command might yield. 696 # TODO: Capture other errors from other files, such as 697 # //depot/google3/third_party/golang/vitess/go/vt/topo/tablet.go 698 elif get_wrong_arg_count_error and re.search(r'fmt.Errorf', line): 699 get_wrong_arg_count_error = False 700 error_data = re.findall(r'fmt\.Errorf\(\"([^\"]+)\"\)', line) 701 if error_data and current_function in functions: 702 fcommand = functions[current_function] 703 fgroup = command_groups[fcommand] 704 commands[fgroup][fcommand]['errors']['ARG_COUNT'] = { 705 'exact_count': error_counts[current_function]['exact'], 706 'min_count': error_counts[current_function]['min'], 707 'max_count': error_counts[current_function]['max'], 708 'message': error_data[0] 709 } 710 elif line.strip().endswith('}') or line.strip().endswith('{'): 711 get_wrong_arg_count_error = False 712 elif current_function in functions and re.search(r'fmt.Errorf', line): 713 error_data = re.findall(r'fmt\.Errorf\(\"([^\"]+)\"', line) 714 if error_data: 715 fcommand = functions[current_function] 716 fgroup = command_groups[fcommand] 717 if (error_data[0] not in 718 commands[fgroup][fcommand]['errors']['other']): 719 if ('ARG_COUNT' in commands[fgroup][fcommand]['errors'] and 720 error_data[0] != 721 commands[fgroup][fcommand]['errors']['ARG_COUNT']['message']): 722 commands[fgroup][fcommand]['errors']['other'].append( 723 error_data[0]) 724 elif 'ARG_COUNT' not in commands[fgroup][fcommand]['errors']: 725 commands[fgroup][fcommand]['errors']['other'].append( 726 error_data[0]) 727 728 729 # This line indicates that commands are starting. No need to capture 730 # stuff before here. 731 elif re.search(r'^var commands =', line.strip()): 732 get_commands = True 733 734 elif re.search(r'^func init\(\)', line.strip()): 735 get_commands = True 736 is_func_init = True 737 738 if get_argument_definitions: 739 if line.strip() == '*/': 740 get_argument_definitions = False 741 elif line.startswith('-'): 742 definition_data = re.findall(r'^\-\s*([^\:]+)\:\s*(.*)', line) 743 if definition_data: 744 arg_name_array = definition_data[0][0].split('(internal)') 745 current_command_argument = arg_name_array[0].strip() 746 arg_definitions[current_command_argument] = {} 747 if len(arg_name_array) > 1: 748 arg_definitions[current_command_argument]['internal'] = True 749 arg_definitions[current_command_argument]['description'] = ( 750 definition_data[0][1].strip()) 751 current_command_argument_value = '' 752 elif current_command_argument and line.lstrip()[0:2] == '--': 753 if 'list_items' not in arg_definitions[current_command_argument]: 754 arg_definitions[current_command_argument]['list_items'] = [] 755 arg_value_data = re.findall(r'^\--\s*([^\:]+)\:\s*(.*)', line.strip()) 756 if arg_value_data: 757 current_command_argument_value = arg_value_data[0][0] 758 argument_definition = arg_value_data[0][1] 759 arg_definitions[current_command_argument]['list_items'].append({ 760 'value': current_command_argument_value, 761 'definition': argument_definition}) 762 elif current_command_argument_value: 763 list_length = (len( 764 arg_definitions[current_command_argument]['list_items']) - 1) 765 arg_definitions[current_command_argument][ 766 'list_items'][list_length]['definition'] += ' ' + line.strip() 767 elif line and current_command_argument: 768 arg_definitions[current_command_argument]['description'] += ' ' + line.strip() 769 770 elif re.search(r'^COMMAND ARGUMENT DEFINITIONS', line.strip()): 771 get_argument_definitions = True 772 773 # Handle arguments that have different names but same meaning 774 new_arg_definitions = {} 775 modifiers = ['destination', 'new master', 'original', 'parent', 'served', 776 'source', 'target'] 777 778 for defined_argument in arg_definitions: 779 argument_list = defined_argument.split(',') 780 if len(argument_list) > 1: 781 for argument_list_item in argument_list: 782 new_arg_definitions[argument_list_item.strip()] = ( 783 arg_definitions[defined_argument]) 784 for modifier in modifiers: 785 new_arg_definitions[modifier + ' ' + argument_list_item.strip()] = ( 786 arg_definitions[defined_argument]) 787 else: 788 new_arg_definitions[defined_argument] = arg_definitions[defined_argument] 789 for modifier in modifiers: 790 new_arg_definitions[modifier + ' ' + defined_argument] = ( 791 arg_definitions[defined_argument]) 792 793 #print json.dumps(new_arg_definitions, sort_keys=True, indent=4) 794 #print json.dumps(commands["Generic"], sort_keys=True, indent=4) 795 796 create_reference_doc(root_directory, commands, new_arg_definitions) 797 798 return 799 800 if __name__ == '__main__': 801 802 parser = optparse.OptionParser() 803 parser.add_option('-r', '--root-directory', default='..', 804 help='root directory for the vitess github tree') 805 (options, args) = parser.parse_args() 806 main(options.root_directory)