github.com/tiagovtristao/plz@v13.4.0+incompatible/src/parse/rules/proto_rules.build_defs (about) 1 """Build rules for compiling protocol buffers & gRPC service stubs. 2 3 Note that these are some of the most complex of our built-in build rules, 4 because of their cross-language nature. Each proto_library rule declares a set of 5 sub-rules to run protoc & the appropriate java_library, go_library rules etc. Users 6 shouldn't worry about those sub-rules and just declare a dependency directly on 7 the proto_library rule to get its appropriate outputs. 8 9 It is possible to add extra languages to these for generation. This is accomplished 10 via the 'languages' argument; this can be simply a list of languages to build, but 11 can also be a mapping of language name -> definition of how to build it. The definition 12 should be the return value of proto_language. 13 """ 14 15 _DEFAULT_GRPC_LABELS = ['grpc'] 16 17 18 def proto_library(name:str, srcs:list, deps:list=[], visibility:list=None, labels:list&features&tags=[], 19 languages:list|dict=None, test_only:bool&testonly=False, root_dir:str='', protoc_flags:list=[]): 20 """Compile a .proto file to generated code for various languages. 21 22 Args: 23 name (str): Name of the rule 24 srcs (list): Input .proto files. 25 deps (list): Dependencies 26 visibility (list): Visibility specification for the rule. 27 labels (list): List of labels to apply to this rule. 28 languages (list | dict): List of languages to generate rules for, chosen from the set {cc, py, go, java, js}. 29 Alternatively, a dict mapping the language name to a definition of how to build it 30 (see proto_language for more details of the values). 31 test_only (bool): If True, can only be used in test rules. 32 root_dir (str): The directory that the protos are compiled relative to. Useful if your 33 proto files have import statements that are not relative to the repo root. 34 protoc_flags (list): Additional flags to pass to protoc. Note that these are inherited by 35 further rules that depend on this one (because in nearly all cases that 36 will be necessary for them to build too). 37 """ 38 languages = _merge_dicts(languages or CONFIG.PROTO_LANGUAGES, proto_languages()) 39 40 # We detect output names for normal sources, but will have to do a post-build rule for 41 # any input rules. We could just do that for everything but it's nicer to avoid them 42 # when possible since they obscure what's going on with the build graph. 43 file_srcs = [src for src in srcs if src[0] not in [':', '/']] 44 need_post_build = file_srcs != srcs 45 provides = {'proto': f':_{name}#proto'} 46 47 lang_plugins = sorted(languages.items()) 48 plugins = [plugin for _, plugin in lang_plugins] 49 file_extensions = [] 50 outs = {ext_lang: [src.replace('.proto', ext) for src in file_srcs for ext in exts] 51 if plugin['use_file_names'] else [] 52 for language, plugin in lang_plugins for ext_lang, exts in plugin['extensions'].items()} 53 flags = [' '.join(plugin['protoc_flags']) for plugin in plugins] + protoc_flags 54 tools = {lang: plugin.get('tools') for lang, plugin in lang_plugins} 55 tools['protoc'] = [CONFIG.PROTOC_TOOL] 56 cmd = '$TOOLS_PROTOC ' + ' '.join(flags) 57 if root_dir: 58 cmd = 'export RD="%s"; cd $RD; %s ${SRCS//$RD\\//} && cd $TMP_DIR' % (root_dir, cmd.replace('$TMP_DIR', '.')) 59 else: 60 cmd += ' ${SRCS}' 61 cmds = [cmd, '(mv -f ${PKG}/* .; true)'] 62 63 # protoc_flags are applied transitively to dependent rules via labels. 64 labels += ['protoc:' + flag for flag in protoc_flags] 65 66 # TODO(pebers): genericise this bit? 67 if 'go' in languages: 68 base_path = get_base_path() 69 diff_pkg = basename(base_path) != name 70 if CONFIG.GO_IMPORT_PATH: 71 base_path = join_path(CONFIG.GO_IMPORT_PATH, base_path) 72 labels += [f'proto:go-map: {base_path}/{src}={base_path}/{name}' for src in srcs 73 if not src.startswith(':') and not src.startswith('/') and 74 (src != (name + '.proto') or len(srcs) > 1 or diff_pkg)] 75 76 # Figure out which languages we need to detect output files for. 77 # This always happens for Java, and will be needed for any other language where the inputs aren't plain files. 78 post_build = None 79 search_extensions = [(lang, exts) for plugin in plugins 80 for lang, exts in sorted(plugin['extensions'].items()) 81 if need_post_build or not plugin['use_file_names']] 82 if search_extensions: 83 all_exts = [ext for _, exts in search_extensions for ext in exts] 84 cmds.append('find . %s | sort' % ' -or '.join(['-name "*%s"' % ext for ext in all_exts])) 85 post_build = _annotate_outs(search_extensions) 86 87 # Plugins can declare their own pre-build functions. If there are any, we need to apply them all in sequence. 88 pre_build_functions = [plugin['pre_build'] for plugin in plugins if plugin['pre_build']] 89 pre_build_functions.append(_collect_transitive_labels) 90 pre_build = lambda rule: [fn(rule) for fn in pre_build_functions] 91 protoc_rule = build_rule( 92 name = name, 93 tag = 'protoc', 94 srcs = srcs, 95 outs = outs, 96 cmd = ' && '.join(cmds), 97 deps = deps, 98 tools = tools, 99 requires = ['proto'], 100 pre_build = pre_build, 101 post_build = post_build, 102 labels = labels, 103 needs_transitive_deps = True, 104 test_only = test_only, 105 visibility = visibility, 106 ) 107 108 for language, plugin in lang_plugins: 109 lang_name = f'_{name}#{language}' 110 provides[language] = plugin['func']( 111 name = lang_name, 112 srcs = [f'{protoc_rule}|{language}'], 113 deps = deps + plugin['deps'], 114 test_only = test_only 115 ) or (':' + lang_name) 116 # TODO(pebers): find a way of genericising this too... 117 if language == 'cc': 118 provides['cc_hdrs'] = provides['cc'].replace('#cc', '#cc_hdrs') 119 120 # This simply collects the sources, it's used for other proto_library rules to depend on. 121 filegroup( 122 name = f'_{name}#proto', 123 srcs = srcs, 124 visibility = visibility, 125 exported_deps = deps, 126 labels = labels, 127 requires = ['proto'], 128 output_is_complete = False, 129 test_only = test_only, 130 ) 131 # This is the final rule that directs dependencies to the appropriate language. 132 filegroup( 133 name = name, 134 deps = provides.values(), 135 provides = provides, 136 visibility = visibility, 137 labels = labels, 138 test_only = test_only, 139 ) 140 141 142 def grpc_library(name:str, srcs:list, deps:list=None, visibility:list=None, languages:list|dict=None, 143 labels:list&features&tags=[], test_only:bool&testonly=False, root_dir:str='', protoc_flags:list=None): 144 """Defines a rule for a grpc library. 145 146 Args: 147 name (str): Name of the rule 148 srcs (list): Input .proto files. 149 deps (list): Dependencies (other grpc_library or proto_library rules) 150 visibility (list): Visibility specification for the rule. 151 languages (list | dict): List of languages to generate rules for, chosen from the set {cc, py, go, java}. 152 Alternatively, a dict mapping the language name to a definition of how to build it 153 (see proto_language for more details of the values). 154 labels (list): List of labels to apply to this rule. 155 test_only (bool): If True, this rule can only be used by test rules. 156 root_dir (str): The directory that the protos are compiled relative to. Useful if your 157 proto files have import statements that are not relative to the repo root. 158 protoc_flags (list): Additional flags to pass to protoc. 159 """ 160 proto_library( 161 name = name, 162 srcs = srcs, 163 deps = deps, 164 languages = _merge_dicts(languages or CONFIG.PROTO_LANGUAGES, grpc_languages()), 165 visibility = visibility, 166 labels = labels + _DEFAULT_GRPC_LABELS, 167 test_only = test_only, 168 root_dir = root_dir, 169 protoc_flags = protoc_flags, 170 ) 171 172 173 def _go_path_mapping(grpc): 174 """Used to update the Go path mapping; by default it doesn't really import in the way we want.""" 175 grpc_plugin = 'plugins=grpc,' if grpc else '' 176 def _map_go_paths(rule_name): 177 mapping = ',M'.join(get_labels(rule_name, 'proto:go-map:')) 178 cmd = get_command(rule_name) 179 new_cmd = cmd.replace('--go_out=paths=source_relative:', f'--go_out=paths=source_relative,{grpc_plugin}M{mapping}:') 180 set_command(rule_name, new_cmd) 181 return _map_go_paths 182 183 184 def proto_language(language:str, extensions:list|dict, func:function, use_file_names:bool=True, protoc_flags:list=None, 185 tools:list=None, deps:list=None, pre_build:function=None, proto_language:str=''): 186 """Returns the definition of how to build a particular language for proto_library. 187 188 Args: 189 language (str): Name of the language (as we would name it). 190 extensions (list | dict): File extensions that will get generated. 191 func (function): Function defining how to build the rule. It will receive the following arguments: 192 name: Suggested name of the rule. 193 srcs: Source files, as generated by protoc. 194 deps: Suggested dependencies. 195 test_only: True if the original rule was marked as test_only. 196 It should return the name of any rule that it wants added to the final list of provides. 197 use_file_names (bool): True if the output file names are normally predictable. 198 This is the case for most languages but not e.g. Java where they depend on the 199 declarations in the proto file. If False we'll attempt to detect them. 200 protoc_flags (list): Additional flags for the protoc invocation for this rule. 201 tools (list): Additional tools to apply to this rule. 202 deps (list): Additional dependencies to apply to this rule. 203 pre_build (function): Definition of pre-build function to apply to this language. 204 proto_language (str): Name of the language (as protoc would name it). Defaults to the same as language. 205 """ 206 return { 207 'language': language, 208 'proto_language': proto_language or language, 209 'extensions': {language: extensions} if isinstance(extensions, list) else extensions, 210 'func': func, 211 'use_file_names': use_file_names, 212 'protoc_flags': protoc_flags or [], 213 'tools': tools or [], 214 'deps': deps or [], 215 'pre_build': pre_build, 216 } 217 218 219 def _parent_rule(name): 220 """Returns the parent rule, i.e. strips the leading _ and trailing #hashtag.""" 221 before, _, _ = name.partition('#') 222 return before.lstrip('_') 223 224 225 def _annotate_outs(extensions): 226 """Used to collect output files when we can't determine them without running the rule. 227 228 For Java this is always the case because their location depends on the java_package option 229 defined in the .proto file. For other languages we might not know if the sources come from 230 another rule. 231 """ 232 def _annotate_outs(rule_name, output): 233 for out in output: 234 for lang, exts in extensions: 235 for ext in exts: 236 if out.endswith(ext): 237 add_out(rule_name, lang, out.lstrip('./')) 238 return _annotate_outs 239 240 241 def _merge_dicts(a, b): 242 """Merges dictionary a into dictionary b, overwriting where a's values are not None.""" 243 if not isinstance(a, dict): 244 return {x: b[x] for x in a} # Languages can be passed as just a list. 245 return {k: v or b[k] for k, v in a.items()} 246 247 248 def _collect_transitive_labels(rule): 249 """Defines a pre-build function that updates a build command with transitive protoc flags.""" 250 labels = get_labels(rule, 'protoc:') 251 if labels: 252 cmd = get_command(rule) 253 set_command(rule, cmd.replace('$TOOLS_PROTOC ', '$TOOLS_PROTOC %s ' % ' '.join(labels))) 254 255 256 def proto_languages(): 257 """Returns the known set of proto language definitions. 258 259 Due to technical reasons this can't just be a global (if you must know: the lambdas need 260 to bind to the set of globals for the BUILD file, not the set when we load the rules). 261 TODO(pebers): This seems a bit ugly and might be slow if we're creating a lot of temporaries. 262 Find a way to persist these... 263 """ 264 return { 265 'cc': proto_language( 266 language = 'cc', 267 proto_language = 'cpp', 268 extensions = {'cc': ['.pb.cc'], 'cc_hdrs': ['.pb.h']}, 269 func = lambda name, srcs, deps, test_only: cc_library( 270 name = name, 271 srcs = srcs, 272 hdrs = [srcs[0] + '_hdrs'], 273 deps = deps, 274 test_only = test_only, 275 pkg_config_libs = ['protobuf'], 276 compiler_flags = ['-I$PKG'], 277 ), 278 protoc_flags = ['--cpp_out=$TMP_DIR'], 279 ), 280 'java': proto_language( 281 language = 'java', 282 extensions = ['.java'], 283 use_file_names = False, 284 func = lambda name, srcs, deps, test_only: java_library( 285 name = name, 286 srcs = srcs, 287 exported_deps = deps, 288 test_only = test_only, 289 labels = ['proto'], 290 ), 291 protoc_flags = ['--java_out=$TMP_DIR'], 292 deps = [CONFIG.PROTO_JAVA_DEP], 293 ), 294 'go': proto_language( 295 language = 'go', 296 extensions = ['.pb.go'], 297 func = lambda name, srcs, deps, test_only: go_library( 298 name = name, 299 srcs = srcs, 300 out = _parent_rule(name) + '.a', 301 deps = deps, 302 test_only = test_only, 303 ), 304 protoc_flags = ['--go_out=paths=source_relative:$TMP_DIR', '--plugin=protoc-gen-go=$TOOLS_GO'], 305 tools = [CONFIG.PROTOC_GO_PLUGIN], 306 deps = [CONFIG.PROTO_GO_DEP], 307 pre_build = _go_path_mapping(False), 308 ), 309 'js': proto_language( 310 language = 'js', 311 extensions = ['_pb.js'], 312 func = lambda name, srcs, deps, test_only: filegroup( 313 name = name, 314 srcs = srcs, 315 deps = deps, 316 test_only = test_only, 317 requires = ['js'], 318 output_is_complete = False, 319 ), 320 protoc_flags = ['--js_out=import_style=commonjs,binary:$TMP_DIR'], 321 deps = [CONFIG.PROTO_JS_DEP], 322 ), 323 'py': proto_language( 324 language = 'py', 325 proto_language = 'python', 326 extensions = ['_pb2.py'], 327 func = python_library, 328 protoc_flags = ['--python_out=$TMP_DIR'], 329 deps = [CONFIG.PROTO_PYTHON_DEP], 330 ), 331 } 332 333 334 def grpc_languages(): 335 """Returns the predefined set of gRPC languages.""" 336 return { 337 'cc': proto_language( 338 language = 'cc', 339 proto_language = 'cpp', 340 extensions = {'cc': ['.pb.cc', '.grpc.pb.cc'], 'cc_hdrs': ['.pb.h', '.grpc.pb.h']}, 341 func = lambda name, srcs, deps, test_only: cc_library( 342 name = name, 343 srcs = srcs, 344 hdrs = [srcs[0] + '_hdrs'], 345 deps = deps, 346 test_only = test_only, 347 pkg_config_libs = ['grpc++', 'grpc', 'protobuf'], 348 compiler_flags = ['-I$PKG', '-Wno-unused-parameter'], # Generated gRPC code is not robust to this. 349 ), 350 protoc_flags = ['--cpp_out=$TMP_DIR', '--plugin=protoc-gen-grpc-cc=$TOOLS_CC', '--grpc-cc_out=$TMP_DIR'], 351 tools = [CONFIG.GRPC_CC_PLUGIN], 352 ), 353 'py': proto_language( 354 language = 'py', 355 proto_language = 'python', 356 extensions = ['_pb2.py', '_pb2_grpc.py'], 357 func = python_library, 358 protoc_flags = ['--python_out=$TMP_DIR', '--plugin=protoc-gen-grpc-python=$TOOLS_PY', '--grpc-python_out=$TMP_DIR'], 359 tools = [CONFIG.GRPC_PYTHON_PLUGIN], 360 deps = [CONFIG.PROTO_PYTHON_DEP, CONFIG.GRPC_PYTHON_DEP], 361 ), 362 'java': proto_language( 363 language = 'java', 364 extensions = ['.java'], 365 use_file_names = False, 366 func = lambda name, srcs, deps, test_only: java_library( 367 name = name, 368 srcs = srcs, 369 exported_deps = deps, 370 test_only = test_only, 371 labels = ['proto'], 372 ), 373 protoc_flags = ['--java_out=$TMP_DIR', '--plugin=protoc-gen-grpc-java=$TOOLS_JAVA', '--grpc-java_out=$TMP_DIR'], 374 tools = [CONFIG.GRPC_JAVA_PLUGIN], 375 deps = [CONFIG.GRPC_JAVA_DEP, CONFIG.PROTO_JAVA_DEP], 376 ), 377 'go': proto_language( 378 language = 'go', 379 extensions = ['.pb.go'], 380 func = lambda name, srcs, deps, test_only: go_library( 381 name = name, 382 srcs = srcs, 383 out = _parent_rule(name) + '.a', 384 deps = deps, 385 test_only = test_only, 386 ), 387 protoc_flags = ['--go_out=paths=source_relative:$TMP_DIR', '--plugin=protoc-gen-go=$TOOLS_GO'], 388 tools = [CONFIG.PROTOC_GO_PLUGIN], 389 deps = [CONFIG.PROTO_GO_DEP, CONFIG.GRPC_GO_DEP], 390 pre_build = _go_path_mapping(True), 391 ), 392 # We don't really support grpc-js right now, so this is the same as proto-js. 393 'js': proto_language( 394 language = 'js', 395 extensions = ['_pb.js'], 396 func = lambda name, srcs, deps, test_only: filegroup( 397 name = name, 398 srcs = srcs, 399 deps = deps, 400 test_only = test_only, 401 requires = ['js'], 402 output_is_complete = False, 403 ), 404 protoc_flags = ['--js_out=import_style=commonjs,binary:$TMP_DIR'], 405 deps = [CONFIG.PROTO_JS_DEP], 406 ), 407 } 408 409 410 def protoc_binary(name, version, hashes=None, deps=None, visibility=None): 411 """Downloads a precompiled protoc binary. 412 413 You will obviously need to choose a version that is available on Github - there aren't 414 necessarily protoc downloads for every protobuf release. 415 416 Args: 417 name (str): Name of the rule 418 version (str): Version of protoc to download (e.g. '3.4.0'). 419 hashes (list): Hashes to verify the download against. 420 deps (list): Any other dependencies 421 visibility (list): Visibility of the rule. 422 """ 423 download_rule = remote_file( 424 name = name, 425 _tag = 'download', 426 url = f'https://github.com/google/protobuf/releases/download/v{version}/protoc-{version}-$XOS-$XARCH.zip', 427 out = f'protoc-{version}.zip', 428 hashes = hashes, 429 deps = deps, 430 ) 431 return genrule( 432 name = name, 433 srcs = [download_rule], 434 outs = ['protoc'], 435 tools = [CONFIG.JARCAT_TOOL], 436 binary = True, 437 cmd = '$TOOL x $SRCS bin/protoc', 438 visibility = visibility, 439 )