vitess.io/vitess@v0.16.2/doc/vitess_api_reference.py (about) 1 #!/usr/bin/python 2 3 # Copyright 2018 The Vitess Authors. 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 """ 18 """ 19 20 import json 21 import os 22 import optparse 23 import pprint 24 import re 25 26 #def print_api_summary(doc, service_summary): 27 # doc.write(service_summary + '\n\n') 28 29 def print_method_summary(doc, proto_contents, methods): 30 for method in sorted(methods, key=lambda k: k['name']): 31 method_group_info = method['comment'].split(' API group: ') 32 if len(method_group_info) > 1: 33 method['group'] = method_group_info[1] 34 method['comment'] = method_group_info[0] 35 else: 36 method['group'] = 'Uncategorized' 37 38 doc.write('This document describes Vitess API methods that enable your ' + 39 'client application to more easily talk to your storage system ' + 40 'to query data. API methods are grouped into the following ' + 41 'categories:\n\n') 42 last_group = '' 43 for group in proto_contents['group-ordering']: 44 for method in sorted(methods, key=lambda k: (k['group'], k['name'])): 45 if (group.lower() == method['group'].lower() and 46 not method['group'] == last_group): 47 if not method['group'] == last_group: 48 doc.write('* [' + method['group'] + ']' + 49 '(#' + method['group'].replace(' ', '-').lower() + ')\n') 50 last_group = method['group'] 51 doc.write('\n\n') 52 53 doc.write('The following table lists the methods in each group and links ' + 54 'to more detail about each method:\n\n') 55 doc.write('<table id="api-method-summary">\n') 56 57 last_group = '' 58 for group in proto_contents['group-ordering']: 59 for method in sorted(methods, key=lambda k: (k['group'], k['name'])): 60 if (group.lower() == method['group'].lower() and 61 not method['group'] == last_group): 62 print_method_summary_group_row(doc, method['group']) 63 last_group = method['group'] 64 print_method_summary_row(doc, method) 65 elif group.lower() == method['group'].lower(): 66 print_method_summary_row(doc, method) 67 doc.write('</table>\n') 68 69 def print_method_summary_group_row(doc, group_name): 70 doc.write('<tr><td class="api-method-summary-group" colspan="2">' + 71 group_name + '</td></tr>\n') 72 73 def print_method_summary_row(doc, method): 74 doc.write('<tr>\n') 75 doc.write('<td><code><a href="#' + method['name'].lower() + '">' + 76 method['name'] + '</a></code></td>\n<td>') 77 if 'comment' in method and method['comment']: 78 doc.write(method['comment']) 79 doc.write('</td>\n') 80 doc.write('</tr>\n') 81 82 def recursively_add_objects(new_objects, method_file, obj, 83 properties, proto_contents): 84 if obj in new_objects: 85 return 86 87 [op_enum_file, op_enum] = get_op_item(proto_contents, obj, 'enums') 88 [op_file, op_method] = get_op_item(proto_contents, obj, 'messages') 89 90 if properties: 91 if method_file not in new_objects: 92 new_objects[method_file] = {'messages': {}} 93 elif 'messages' not in new_objects[method_file]: 94 new_objects[method_file]['messages'] = {} 95 new_objects[method_file]['messages'][obj] = 1 96 for prop in properties: 97 type_list = prop['type'].split('.') 98 if len(type_list) == 1: 99 if (method_file in proto_contents and 100 'messages' in proto_contents[method_file]): 101 lil_pc = proto_contents[method_file]['messages'] 102 if type_list[0] in lil_pc and 'properties' in lil_pc[type_list[0]]: 103 new_objects = recursively_add_objects(new_objects, method_file, 104 prop['type'], lil_pc[type_list[0]]['properties'], 105 proto_contents) 106 elif (obj in lil_pc and 107 'messages' in lil_pc[obj] and 108 type_list[0] in lil_pc[obj]['messages'] and 109 'properties' in lil_pc[obj]['messages'][type_list[0]]): 110 new_objects = recursively_add_objects(new_objects, method_file, 111 prop['type'], 112 lil_pc[obj]['messages'][type_list[0]]['properties'], 113 proto_contents) 114 else: 115 [op_file, op_method] = get_op_item(proto_contents, obj, 'messages') 116 [op_enum_file, op_enum] = get_op_item(proto_contents, obj, 'enums') 117 118 if (op_file and 119 op_file in proto_contents and 120 'messages' in proto_contents[op_file] and 121 type_list[1] in proto_contents[op_file]['messages'] and 122 'properties' in proto_contents[op_file]['messages'][type_list[1]]): 123 new_objects = recursively_add_objects(new_objects, op_file, type_list[1], 124 proto_contents[op_file]['messages'][type_list[1]]['properties'], 125 proto_contents) 126 elif (op_enum_file and 127 op_enum_file in proto_contents and 128 'enums' in proto_contents[op_enum_file] and 129 type_list[1] in proto_contents[op_enum_file]['enums']): 130 if not op_enum_file in new_objects['enums']: 131 new_objects['enums'][op_enum_file] = {} 132 new_objects['enums'][op_enum_file][type_list[1]] = ( 133 proto_contents[op_enum_file]['enums'][type_list[1]]) 134 return new_objects 135 136 def print_method_details(doc, proto_contents, proto, methods, objects): 137 138 last_group = '' 139 for group in proto_contents['group-ordering']: 140 for method in sorted(methods, key=lambda k: (k['group'], k['name'])): 141 if (group.lower() == method['group'].lower() and 142 not method['group'] == last_group): 143 doc.write('##' + method['group'] + '\n') 144 last_group = method['group'] 145 print_method_detail_header(doc, method) 146 print_method_detail_request(doc, proto_contents, proto, method) 147 print_method_detail_response(doc, proto_contents, proto, method) 148 elif group.lower() == method['group'].lower(): 149 print_method_detail_header(doc, method) 150 print_method_detail_request(doc, proto_contents, proto, method) 151 print_method_detail_response(doc, proto_contents, proto, method) 152 new_objects = {} 153 for obj in sorted(objects): 154 type_list = obj.split('.') 155 if len(type_list) == 1: 156 method_file = objects[obj]['methods'][0]['method_file'] 157 if (method_file in proto_contents and 158 'messages' in proto_contents[method_file] and 159 obj in proto_contents[method_file]['messages'] and 160 'properties' in proto_contents[method_file]['messages'][obj]): 161 new_objects = recursively_add_objects(new_objects, method_file, obj, 162 proto_contents[method_file]['messages'][obj]['properties'], 163 proto_contents) 164 165 else: 166 [op_file, op_method] = get_op_item(proto_contents, obj, 'messages') 167 [op_enum_file, op_enum] = get_op_item(proto_contents, obj, 'enums') 168 if (op_file and 169 op_file in proto_contents and 170 'messages' in proto_contents[op_file] and 171 type_list[1] in proto_contents[op_file]['messages'] and 172 'properties' in proto_contents[op_file]['messages'][type_list[1]]): 173 new_objects = recursively_add_objects(new_objects, op_file, type_list[1], 174 proto_contents[op_file]['messages'][type_list[1]]['properties'], 175 proto_contents) 176 elif (op_enum_file and 177 op_enum_file in proto_contents and 178 'enums' in proto_contents[op_enum_file] and 179 type_list[1] in proto_contents[op_enum_file]['enums']): 180 if not op_enum_file in new_objects: 181 new_objects[op_enum_file] = {'enums':{}} 182 elif not 'enums' in new_objects[op_enum_file]: 183 new_objects[op_enum_file]['enums'] = {} 184 new_objects[op_enum_file]['enums'][type_list[1]] = ( 185 proto_contents[op_enum_file]['enums'][type_list[1]]) 186 187 #print json.dumps(new_objects, sort_keys=True, indent=2) 188 print_nested_objects(doc, new_objects, proto_contents) 189 190 def print_nested_objects(doc, objects, proto_contents): 191 doc.write('## Enums\n\n') 192 for obj in sorted(objects): 193 if obj == 'vtgate.proto': 194 print_proto_enums(doc, proto_contents, obj, objects, 195 {'strip-proto-name': 1}) 196 for obj in sorted(objects): 197 if not obj == 'vtgate.proto': 198 print_proto_enums(doc, proto_contents, obj, objects, {}) 199 doc.write('## Messages\n\n') 200 for obj in sorted(objects): 201 if obj == 'vtgate.proto': 202 print_proto_messages(doc, proto_contents, obj, objects, 203 {'strip-proto-name': 1}) 204 for obj in sorted(objects): 205 if not obj == 'vtgate.proto': 206 print_proto_messages(doc, proto_contents, obj, objects, {}) 207 208 def print_message_detail_header(doc, proto, message_details, message_name, 209 options): 210 header_size = '###' 211 if 'header-size' in options: 212 header_size = options['header-size'] 213 214 message = (proto.replace('.proto', '').replace('.pb.go', '') + 215 '.' + message_name) 216 if (options and 217 'strip-proto-name' in options and 218 options['strip-proto-name']): 219 message = message_name 220 elif options and 'add-method-name' in options and 'method-name' in options: 221 message = options['method-name'] + '.' + message_name 222 doc.write(header_size + ' ' + message + '\n\n') 223 224 if 'comment' in message_details and message_details['comment']: 225 doc.write(message_details['comment'].strip() + '\n\n') 226 227 def print_method_detail_header(doc, method): 228 doc.write('### ' + method['name'] + '\n\n') 229 if 'comment' in method and method['comment']: 230 doc.write(method['comment'] + '\n\n') 231 232 def print_properties_header(doc, header, table_headers): 233 if header: 234 doc.write('##### ' + header + '\n\n') 235 if table_headers: 236 doc.write('| ') 237 for field in table_headers: 238 doc.write(field + ' |') 239 doc.write('\n') 240 for field in table_headers: 241 doc.write('| :-------- ') 242 doc.write('\n') 243 244 def print_property_row(doc, proto_contents, proto, method_file, method, prop): 245 # Print property/parameter name 246 if 'name' in prop: 247 doc.write('| <code>' + prop['name'] + '</code> ') 248 249 method_in_messages = False 250 enum_in_messages = False 251 for key in proto_contents[proto]['messages']: 252 if key == method: 253 method_in_messages = True 254 for key in proto_contents[proto]['enums']: 255 if key == prop['type']: 256 enum_in_messages = True 257 258 # Print property/parameter type 259 if 'type' in prop and prop['type']: 260 doc.write('<br>') 261 prop_text = '' 262 if prop['type'][0:5] == 'map <': 263 map_value = prop['type'].split(',')[1].split('>')[0].strip() 264 if (map_value and 265 map_value in proto_contents[proto]['messages']): 266 prop_text = (prop['type'].split(',')[0] + ', [' + map_value + ']' + 267 '(#' + proto.lower().replace('.proto', '') + '.' + 268 map_value.lower() + ')' + '>') 269 prop_text = prop_text.replace('<', '<').replace('>', '>') 270 else: 271 type_list = prop['type'].split('.') 272 if len(type_list) == 2: 273 prop_text = ('[' + prop['type'] + '](#' + type_list[0] + '.' + 274 type_list[1].lower() + ')') 275 elif (method_file and 276 prop['type'] in proto_contents[method_file]['messages']): 277 if method_file == 'vtgate.proto': 278 prop_text = '[' + prop['type'] + '](#' + prop['type'].lower() + ')' 279 else: 280 prop_text = ('[' + prop['type'] + 281 '](#' + proto.lower().replace('.proto', '') + '.' + 282 prop['type'].lower() + ')') 283 elif enum_in_messages: 284 prop_text = ('[' + prop['type'] + ']' + 285 '(#' + proto.lower().replace('.proto', '') + '.' + 286 prop['type'].lower() + ')') 287 elif (method_file and 288 prop['type'] in proto_contents[method_file]['enums']): 289 prop_text = '[' + prop['type'] + '](#' + prop['type'].lower() + ')' 290 elif (method_in_messages and 291 'messages' in proto_contents[proto]['messages'][method]): 292 if prop['type'] in proto_contents[proto]['messages'][method]['messages']: 293 prop_text = '[' + prop['type'] + '](#' + method.lower() + '.' + prop['type'].lower() + ')' 294 elif prop['type'] in proto_contents[proto]['messages'][method]['enums']: 295 prop_text = '[' + prop['type'] + '](#' + method.lower() + '.' + prop['type'].lower() + ')' 296 else: 297 prop_text = prop['type'] 298 299 else: 300 prop_text = prop['type'] 301 bad_text = True 302 for p in proto_contents: 303 if 'messages' in proto_contents[p] and proto_contents[p]['messages']: 304 for m in proto_contents[p]['messages']: 305 if ('messages' in proto_contents[p]['messages'][m] and 306 proto_contents[p]['messages'][m]['messages'] and 307 prop['type'] in proto_contents[p]['messages'][m]['messages']): 308 prop_text = ('[' + prop['type'] + ']' + 309 '(#' + m.lower() + '.' + prop['type'].lower() + ')') 310 bad_text = False 311 if bad_text and isinstance(method, basestring): 312 for p in proto_contents: 313 if ('messages' in proto_contents[p] and 314 proto_contents[p]['messages']): 315 for m in proto_contents[p]['messages']: 316 if ('messages' in proto_contents[p]['messages'][m] and 317 proto_contents[p]['messages'][m]['messages'] and 318 method in proto_contents[p]['messages'][m]['messages'] and 319 'enums' in proto_contents[p]['messages'][m]['messages'][method] and 320 prop['type'] in proto_contents[p]['messages'][m]['messages'][method]['enums']): 321 prop_text = ('[' + prop['type'] + ']' + 322 '(#' + m.lower() + '.' + method.lower() + '.' + 323 prop['type'].lower() + ')') 324 325 if 'status' in prop and prop['status'] == 'repeated': 326 prop_text = 'list <' + prop_text + '>' 327 doc.write(prop_text) 328 329 # Print property/parameter definition. 330 331 if 'type' in prop and prop['type']: 332 # If type contains period -- e.g. vtrpc.CallerId -- then it refers to 333 # a message in another proto. We want to print the comment identifying 334 # that message. In that case, the link field should also link to a doc 335 # for that proto. 336 type_list = prop['type'].split('.') 337 [op_file, op_method] = get_op_item(proto_contents, prop['type'], 338 'messages') 339 [op_enum_file, op_enum] = get_op_item(proto_contents, prop['type'], 'enums') 340 if op_method: 341 doc.write('| ' + op_method['comment'].strip()) 342 elif op_enum: 343 doc.write('| ' + op_enum['comment'].strip()) 344 elif (method_file and 345 prop['type'] in proto_contents[method_file]['messages']): 346 if ('comment' in proto_contents[method_file]['messages'][prop['type']] and 347 proto_contents[method_file]['messages'][prop['type']]['comment']): 348 doc.write('| ' + 349 proto_contents[method_file]['messages'][prop['type']]['comment'].strip()) 350 else: 351 doc.write('|') 352 elif 'comment' in prop and prop['comment']: 353 doc.write('| ' + prop['comment'].strip()) 354 else: 355 doc.write('|') 356 elif 'comment' in prop and prop['comment']: 357 doc.write('| ' + prop['comment'].strip()) 358 else: 359 doc.write('|') 360 doc.write(' |\n') 361 362 def get_op_item(proto_contents, item, item_type): 363 item_list = item.split('.') 364 if len(item_list) == 2: 365 item_file = item_list[0] + '.proto' 366 item_enum = item_list[1] 367 if item_file in proto_contents: 368 if item_type in proto_contents[item_file]: 369 if item_enum in proto_contents[item_file][item_type]: 370 return item_file, proto_contents[item_file][item_type][item_enum] 371 else: 372 return [None, None] 373 else: 374 return [None, None] 375 else: 376 return [None, None] 377 else: 378 return [None, None] 379 380 def print_method_detail_request(doc, proto_contents, proto, method): 381 if method['request']: 382 [op_file, op_method] = get_op_item(proto_contents, method['request'], 383 'messages') 384 if (op_method and 385 'comment' in op_method and 386 op_method['comment']): 387 doc.write('#### Request\n\n') 388 doc.write(op_method['comment'] + '\n\n') 389 print_properties_header(doc, 'Parameters', ['Name', 'Description']) 390 for prop in op_method['properties']: 391 print_property_row(doc, proto_contents, proto, op_file, op_method, prop) 392 doc.write('\n') 393 394 if 'messages' in op_method and op_method['messages']: 395 doc.write('#### Messages\n\n') 396 for message in sorted(op_method['messages']): 397 print_proto_message(doc, proto, proto_contents, 398 op_method['messages'][message], message, 399 {'header-size': '#####', 400 'add-method-name': 1, 401 'method-name': method['request'].split('.')[1]}) 402 403 def print_method_detail_response(doc, proto_contents, proto, method): 404 if method['response']: 405 [op_file, op_method] = get_op_item(proto_contents, 406 method['response'].replace('stream ', ''), 'messages') 407 if (op_method and 408 'comment' in op_method and 409 op_method['comment']): 410 doc.write('#### Response\n\n') 411 doc.write(op_method['comment'] + '\n\n') 412 print_properties_header(doc, 'Properties', ['Name', 'Description']) 413 for prop in op_method['properties']: 414 print_property_row(doc, proto_contents, proto, op_file, op_method, prop) 415 doc.write('\n') 416 417 if 'messages' in op_method and op_method['messages']: 418 doc.write('#### Messages\n\n') 419 for message in sorted(op_method['messages']): 420 print_proto_message(doc, proto, proto_contents, 421 op_method['messages'][message], message, 422 {'header-size': '#####', 423 'add-method-name': 1, 424 'method-name': method['response'].split('.')[1]}) 425 426 def print_proto_file_definition(doc, proto_contents, proto): 427 if ('file_definition' in proto_contents[proto] and 428 proto_contents[proto]['file_definition']): 429 doc.write(proto_contents[proto]['file_definition'].strip() + '\n\n') 430 431 def print_proto_enum(doc, enum_details, enum_name, proto, options): 432 header_size = '###' 433 if 'header-size' in options: 434 header_size = options['header-size'] 435 # Print name of enum as header 436 enum_header = proto.replace('.proto', '') + '.' + enum_name 437 if options and 'add-method-name' in options and 'method-name' in options: 438 enum_header = options['method-name'] + '.' + enum_name 439 elif (options and 440 'strip-proto-name' in options and 441 options['strip-proto-name']): 442 enum_header = enum_name 443 444 doc.write(header_size + ' ' + enum_header + '\n\n') 445 446 if 'comment' in enum_details and enum_details['comment']: 447 doc.write(enum_details['comment'] + '\n\n') 448 449 print_properties_header(doc, None, ['Name', 'Value', 'Description']) 450 for value in enum_details['values']: 451 # Print enum text 452 if 'text' in value: 453 doc.write('| <code>' + value['text'] + '</code> ') 454 else: 455 doc.write('| ') 456 457 # Print enum value 458 if 'value' in value and value['value']: 459 doc.write('| <code>' + value['value'] + '</code> ') 460 else: 461 doc.write('| ') 462 463 # Print enum value description 464 if 'comment' in value and value['comment']: 465 doc.write('| ' + value['comment'].strip() + ' ') 466 else: 467 doc.write('| ') 468 469 doc.write(' |\n') 470 doc.write('\n') 471 472 def print_proto_message(doc, proto, proto_contents, message_details, message, 473 options): 474 475 print_message_detail_header(doc, proto, message_details, message, options) 476 477 if 'header-size' in options: 478 doc.write('<em>Properties</em>\n\n') 479 else: 480 doc.write('#### Properties\n\n') 481 print_properties_header(doc, None, ['Name', 'Description']) 482 for prop in message_details['properties']: 483 print_property_row(doc, proto_contents, proto, proto, message, prop) 484 doc.write('\n') 485 486 option_method_name = message 487 if 'method-name' in options: 488 option_method_name = options['method-name'] + '.' + message 489 490 if 'enums' in message_details and message_details['enums']: 491 doc.write('#### Enums\n\n') 492 for enum in sorted(message_details['enums']): 493 print_proto_enum(doc, message_details['enums'][enum], enum, proto, 494 {'header-size': '#####', 495 'add-method-name': 1, 496 'method-name': option_method_name}) 497 498 if 'messages' in message_details and message_details['messages']: 499 doc.write('#### Messages\n\n') 500 for child_message in sorted(message_details['messages']): 501 print_proto_message(doc, proto, proto_contents, 502 message_details['messages'][child_message], child_message, 503 {'header-size': '#####', 504 'add-method-name': 1, 505 'method-name': option_method_name}) 506 507 def print_proto_messages(doc, proto_contents, proto, objects_to_print, options): 508 if 'messages' in proto_contents[proto] and proto_contents[proto]['messages']: 509 for message in sorted(proto_contents[proto]['messages']): 510 if ('messages' in objects_to_print[proto] and 511 message in objects_to_print[proto]['messages']): 512 print_proto_message(doc, proto, proto_contents, 513 proto_contents[proto]['messages'][message], 514 message, options) 515 516 def print_proto_enums(doc, proto_contents, proto, objects_to_print, options): 517 if 'enums' in proto_contents[proto] and proto_contents[proto]['enums']: 518 for enum in sorted(proto_contents[proto]['enums']): 519 if ('enums' in objects_to_print[proto] and 520 enum in objects_to_print[proto]['enums']): 521 print_proto_enum(doc, proto_contents[proto]['enums'][enum], enum, 522 proto, options) 523 524 def create_reference_doc(proto_directory, doc_directory, proto_contents, 525 addl_types): 526 for proto in proto_contents: 527 if 'service' in proto_contents[proto]: 528 if (proto_contents[proto]['service']['name'] and 529 proto_contents[proto]['service']['name'] == 'Vitess'): 530 doc = open(doc_directory + 'VitessApi.md', 'w') 531 532 #if proto_contents[proto]['file_definition']: 533 # print_api_summary(doc, proto_contents[proto]['file_definition']) 534 535 if proto_contents[proto]['service']['methods']: 536 print_method_summary(doc, proto_contents, 537 proto_contents[proto]['service']['methods']) 538 539 if proto_contents[proto]['service']['methods']: 540 print_method_details(doc, proto_contents, proto, 541 proto_contents[proto]['service']['methods'], 542 addl_types) 543 doc.close() 544 return 545 546 def parse_method_details(line): 547 details = re.findall(r'rpc ([^\(]+)\(([^\)]+)\) returns \(([^\)]+)', line) 548 if details: 549 return {'name': details[0][0], 550 'request': details[0][1], 551 'response': details[0][2]} 552 return {} 553 554 def get_enum_struct(comment): 555 return {'comment': comment, 556 'values': []} 557 558 def get_message_struct(comment): 559 return {'comment': comment, 560 'enums': {}, 561 'messages': {}, 562 'properties': []} 563 564 def add_property(message, prop_data, prop_type, comment): 565 message['properties'].append({'type': prop_type, 566 'name': prop_data[0][2], 567 'position': prop_data[0][3], 568 'status': prop_data[0][0], 569 'comment': comment}) 570 return message 571 572 def build_property_type_list(types, proto_contents, method): 573 [op_file, op_method] = get_op_item(proto_contents, method, 'messages') 574 if op_method and 'properties' in op_method: 575 for prop in op_method['properties']: 576 if 'map' in prop['type']: 577 map_fields = re.findall(r'map\s*<([^\,]+)\,\s*([^\>]+)', prop['type']) 578 if map_fields: 579 for x in range(0,2): 580 if '.' in map_fields[0][x]: 581 types.append(map_fields[0][x]) 582 elif map_fields[0][x][0].isupper(): 583 types.append(op_file.replace('.proto', '') + '.' + 584 map_fields[0][x]) 585 elif '.' in prop['type']: 586 types.append(prop['type']) 587 elif prop['type'][0].isupper(): 588 if prop['type'] in proto_contents[op_file]['messages']: 589 types.append(op_file.replace('.proto', '') + '.' + prop['type']) 590 elif prop['type'] in proto_contents[op_file]['enums']: 591 types.append(op_file.replace('.proto', '') + '.' + prop['type']) 592 else: 593 for message in proto_contents[op_file]['messages']: 594 if 'messages' in proto_contents[op_file]['messages'][message]: 595 child_messages = ( 596 proto_contents[op_file]['messages'][message]['messages']) 597 if (prop['type'] in child_messages and 598 'properties' in child_messages[prop['type']]): 599 for child_prop in child_messages[prop['type']]['properties']: 600 if '.' in child_prop['type']: 601 types.append(child_prop['type']) 602 return types 603 604 def main(proto_directory, doc_directory): 605 arg_definitions = {} 606 commands = {} 607 command_groups = {} 608 error_counts = {} 609 functions = {} 610 611 # Read the .go files in the /vitess/go/vt/vtctl/ directory 612 api_file_path = proto_directory 613 proto_dirs = next(os.walk(api_file_path))[2] 614 proto_lines = {} 615 proto_contents = {} 616 for path in proto_dirs: 617 if not path.endswith('.proto'): 618 continue 619 api_proto_file = open(proto_directory + path, 'rU') 620 proto_lines[path.replace('pb.go', 'proto')] = api_proto_file.readlines() 621 api_proto_file.close() 622 623 # parse .proto files 624 for path in proto_lines: 625 comment = '' 626 enum_values = [] 627 inside_service = '' 628 current_message = {} 629 current_top_level_message = {} 630 current_hierarchy = [] 631 current_struct = '' 632 syntax_specified = False 633 proto_contents[path] = {'file_definition': '', 634 'imports': [], 635 'enums': {}, 636 'messages': {}, 637 'methods': {}, 638 'service': {'name': '', 639 'methods': []} 640 } 641 for original_line in proto_lines[path]: 642 line = original_line.strip() 643 if line[0:8] == 'syntax =': 644 syntax_specified = True 645 continue 646 if line[0:2] == '//' and not syntax_specified: 647 proto_contents[path]['file_definition'] += (' ' + line[2:].strip()) 648 continue 649 elif line[0:2] == '//': 650 if 'TODO' not in line: 651 comment += ' ' + line[2:].strip() 652 elif line[0:6] == 'import': 653 import_file = line[6:].strip().rstrip(';').strip('"').split('/').pop() 654 proto_contents[path]['imports'].append(import_file) 655 elif line[0:8] == 'service ': 656 service = line[8:].strip().rstrip('{').strip() 657 proto_contents[path]['service']['name'] = service 658 inside_service = service 659 comment = '' 660 elif inside_service: 661 if line[0:4] == 'rpc ': 662 method_details = parse_method_details(line) 663 if method_details: 664 if comment: 665 method_details['comment'] = comment.strip() 666 proto_contents[path]['service']['methods'].append(method_details) 667 comment = '' 668 669 elif line == '}': 670 item_to_add = current_hierarchy.pop().split('-') 671 if item_to_add[0] == 'enum': 672 current_enum['values'] = enum_values 673 enum_values = [] 674 if len(current_hierarchy) > 0: 675 go_back_to_struct = current_hierarchy[-1].split('-')[0] 676 if go_back_to_struct == 'topLevelMessage': 677 current_top_level_message['enums'][item_to_add[1]] = current_enum 678 elif go_back_to_struct == 'message': 679 current_message['enums'][item_to_add[1]] = current_enum 680 current_struct = go_back_to_struct 681 else: 682 if current_struct == 'enum': 683 proto_contents[path]['enums'][item_to_add[1]] = current_enum 684 current_struct = '' 685 elif item_to_add[0] == 'message': 686 current_top_level_message['messages'][item_to_add[1]] = ( 687 current_message) 688 current_struct = current_hierarchy[-1].split('-')[0] 689 elif item_to_add[0] == 'topLevelMessage': 690 proto_contents[path]['messages'][item_to_add[1]] = ( 691 current_top_level_message) 692 current_struct = '' 693 elif original_line[0:8] == 'message ': 694 message = line[8:].strip().rstrip('{').strip() 695 current_top_level_message = get_message_struct(comment) 696 comment = '' 697 current_hierarchy.append('topLevelMessage-' + message) 698 current_struct = 'topLevelMessage' 699 elif line[0:8] == 'message ': 700 message = line[8:].strip().rstrip('{').strip() 701 current_message = get_message_struct(comment) 702 current_hierarchy.append('message-' + message) 703 current_struct = 'message' 704 elif line[0:5] == 'enum ': 705 enum = line[5:].strip().rstrip('{').strip() 706 current_enum = get_enum_struct(comment) 707 current_hierarchy.append('enum-' + enum) 708 current_struct = 'enum' 709 comment = '' 710 elif current_struct == 'enum': 711 enum_value_data = re.findall(r'([a-zA-Z0-9_]+)\s*=\s*(\d+)', line) 712 if enum_value_data: 713 enum_values.append({'comment': comment, 714 'text': enum_value_data[0][0], 715 'value': enum_value_data[0][1]}) 716 comment = '' 717 718 else: 719 prop_data = re.findall(r'(optional|repeated|required)?\s*([\w\.\_]+)\s+([\w\.\_]+)\s*=\s*(\d+)', line) 720 if prop_data: 721 if current_struct == 'topLevelMessage': 722 current_top_level_message = add_property(current_top_level_message, 723 prop_data, prop_data[0][1], 724 comment) 725 elif current_struct == 'message': 726 current_message = add_property(current_message, prop_data, 727 prop_data[0][1], comment) 728 comment = '' 729 else: 730 prop_data = re.findall(r'(optional|repeated|required)?\s*map\s*\<([^\>]+)\>\s+([\w\.\_]+)\s*=\s*(\d+)', line) 731 if prop_data: 732 prop_type = 'map <' + prop_data[0][1] + '>' 733 if current_struct == 'topLevelMessage': 734 current_top_level_message = add_property( 735 current_top_level_message, prop_data, prop_type, comment) 736 elif current_struct == 'message': 737 current_message = add_property(current_message, prop_data, 738 prop_type, comment) 739 comment = '' 740 741 #print json.dumps(proto_contents, sort_keys=True, indent=2) 742 methods = [] 743 types = [] 744 for method in proto_contents['vtgateservice.proto']['service']['methods']: 745 methods.append(method['request']) 746 methods.append(method['response'].replace('stream ', '')) 747 for method in methods: 748 types = build_property_type_list(types, proto_contents, method) 749 types = list(set(types)) 750 751 type_length = len(types) 752 new_type_length = 100 753 for x in range(0, 10): 754 #while type_length < new_type_length: 755 for prop_type in types: 756 types = build_property_type_list(types, proto_contents, prop_type) 757 types = list(set(types)) 758 new_type_length = len(types) 759 760 proto_contents['group-ordering'] = ['Range-based Sharding', 761 'Transactions', 762 'Custom Sharding', 763 'Map Reduce', 764 'Topology', 765 'v3 API (alpha)'] 766 767 create_reference_doc(proto_directory, doc_directory, proto_contents, types) 768 769 return 770 771 if __name__ == '__main__': 772 parser = optparse.OptionParser() 773 parser.add_option('-p', '--proto-directory', default='../proto/', 774 help='The root directory for the Vitess GitHub tree') 775 parser.add_option('-d', '--doc-directory', default='', 776 help='The directory where the documentation resides.') 777 (options,args) = parser.parse_args() 778 main(options.proto_directory, options.doc_directory)