github.com/stackb/rules_proto@v0.0.0-20240221195024-5428336c51f1/rules/proto/proto_repository.bzl (about) 1 """proto_repostitory.bzl provides the proto_repository rule.""" 2 3 # Copyright 2014 The Bazel Authors. All rights reserved. 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 # buildifier: disable=bzl-visibility 18 load("//rules/private:execution.bzl", "env_execute", "executable_extension") 19 load("@bazel_gazelle//internal:go_repository_cache.bzl", "read_cache_env") 20 load("@bazel_tools//tools/build_defs/repo:utils.bzl", "read_netrc", "use_netrc") 21 22 # We can't disable timeouts on Bazel, but we can set them to large values. 23 _GO_REPOSITORY_TIMEOUT = 86400 24 25 # Inspired by @bazel_tools//tools/build_defs/repo:http.bzl 26 def _get_auth(ctx, urls): 27 """Given the list of URLs obtain the correct auth dict.""" 28 netrcfile = "" 29 if "NETRC" in ctx.os.environ: 30 netrcfile = ctx.os.environ["NETRC"] 31 elif ctx.os.name.startswith("windows"): 32 if "USERPROFILE" in ctx.os.environ: 33 netrcfile = "%s/_netrc" % (ctx.os.environ["USERPROFILE"]) 34 elif "HOME" in ctx.os.environ: 35 netrcfile = "%s/.netrc" % (ctx.os.environ["HOME"]) 36 37 if netrcfile and ctx.path(netrcfile).exists: 38 netrc = read_netrc(ctx, netrcfile) 39 return use_netrc(netrc, urls, {}) 40 41 return {} 42 43 def _proto_repository_impl(ctx): 44 # TODO(#549): vcs repositories are not cached and still need to be fetched. 45 # Download the repository or module. 46 fetch_repo_args = None 47 48 if ctx.attr.urls: 49 # HTTP mode 50 for key in ("commit", "tag", "vcs", "remote", "version", "sum", "replace"): 51 if getattr(ctx.attr, key): 52 fail("cannot specifiy both urls and %s" % key, key) 53 ctx.download_and_extract( 54 url = ctx.attr.urls, 55 sha256 = ctx.attr.sha256, 56 stripPrefix = ctx.attr.strip_prefix, 57 type = ctx.attr.type, 58 auth = _get_auth(ctx, ctx.attr.urls), 59 ) 60 elif ctx.attr.commit or ctx.attr.tag: 61 # repository mode 62 rev = None 63 if ctx.attr.commit: 64 rev = ctx.attr.commit 65 rev_key = "commit" 66 elif ctx.attr.tag: 67 rev = ctx.attr.tag 68 rev_key = "tag" 69 for key in ("urls", "strip_prefix", "type", "sha256", "version", "sum", "replace"): 70 if getattr(ctx.attr, key): 71 fail("cannot specify both %s and %s" % (rev_key, key), key) 72 73 if ctx.attr.vcs and not ctx.attr.remote: 74 fail("if vcs is specified, remote must also be") 75 76 fetch_repo_args = ["-dest", ctx.path(""), "-importpath", ctx.attr.importpath] 77 if ctx.attr.remote: 78 fetch_repo_args.extend(["--remote", ctx.attr.remote]) 79 if rev: 80 fetch_repo_args.extend(["--rev", rev]) 81 if ctx.attr.vcs: 82 fetch_repo_args.extend(["--vcs", ctx.attr.vcs]) 83 elif ctx.attr.version: 84 # module mode 85 for key in ("urls", "strip_prefix", "type", "sha256", "commit", "tag", "vcs", "remote"): 86 if getattr(ctx.attr, key): 87 fail("cannot specify both version and %s" % key) 88 if not ctx.attr.sum: 89 fail("if version is specified, sum must also be") 90 91 fetch_path = ctx.attr.replace if ctx.attr.replace else ctx.attr.importpath 92 fetch_repo_args = [ 93 "-dest=" + str(ctx.path("")), 94 "-importpath=" + fetch_path, 95 "-version=" + ctx.attr.version, 96 "-sum=" + ctx.attr.sum, 97 ] 98 else: 99 fail("one of urls, commit, tag, or importpath must be specified") 100 101 env = read_cache_env(ctx, str(ctx.path(Label("@bazel_gazelle_go_repository_cache//:go.env")))) 102 env_keys = [ 103 # Respect user proxy and sumdb settings for privacy. 104 # TODO(jayconrod): gazelle in go_repository mode should probably 105 # not go out to the network at all. This means *the build* 106 # goes out to the network. We tolerate this for downloading 107 # archives, but finding module roots is a bit much. 108 "GOPROXY", 109 "GONOPROXY", 110 "GOPRIVATE", 111 "GOSUMDB", 112 "GONOSUMDB", 113 114 # PATH is needed to locate git and other vcs tools. 115 "PATH", 116 117 # HOME is needed to locate vcs configuration files (.gitconfig). 118 "HOME", 119 120 # Settings below are used by vcs tools. 121 "SSH_AUTH_SOCK", 122 "SSL_CERT_FILE", 123 "SSL_CERT_DIR", 124 "HTTP_PROXY", 125 "HTTPS_PROXY", 126 "NO_PROXY", 127 "http_proxy", 128 "https_proxy", 129 "no_proxy", 130 "GIT_SSL_CAINFO", 131 "GIT_SSH", 132 "GIT_SSH_COMMAND", 133 ] 134 env.update({k: ctx.os.environ[k] for k in env_keys if k in ctx.os.environ}) 135 136 if fetch_repo_args: 137 # Disable sumdb in fetch_repo. In module mode, the sum is a mandatory 138 # attribute of go_repository, so we don't need to look it up. 139 fetch_repo_env = dict(env) 140 fetch_repo_env["GOSUMDB"] = "off" 141 142 # Override external GO111MODULE, because it is needed by module mode, no-op in repository mode 143 fetch_repo_env["GO111MODULE"] = "on" 144 145 fetch_repo = str(ctx.path(Label("@bazel_gazelle_go_repository_tools//:bin/fetch_repo{}".format(executable_extension(ctx))))) 146 result = env_execute( 147 ctx, 148 [fetch_repo] + fetch_repo_args, 149 environment = fetch_repo_env, 150 timeout = _GO_REPOSITORY_TIMEOUT, 151 ) 152 if result.return_code: 153 fail("failed to fetch %s: %s" % (ctx.name, result.stderr)) 154 if result.stderr: 155 # buildifier: disable=print 156 print("fetch_repo: " + result.stderr) 157 158 # Repositories are fetched. Determine if build file generation is needed. 159 build_file_names = ctx.attr.build_file_name.split(",") 160 existing_build_file = "" 161 for name in build_file_names: 162 path = ctx.path(name) 163 if path.exists and not env_execute(ctx, ["test", "-f", path]).return_code: 164 existing_build_file = name 165 break 166 167 generate = (ctx.attr.build_file_generation == "on" or (not existing_build_file and ctx.attr.build_file_generation == "auto")) 168 169 # remove any existing build files 170 if ctx.attr.build_file_expunge: 171 cmd = ["find", ".", "-type", "f", "("] 172 for i, name in enumerate(build_file_names): 173 cmd += ["-name", name] 174 if i + 1 < len(build_file_names): 175 cmd.append("-o") 176 cmd += [")", "-print", "-exec", "rm", "{}", "+"] 177 result = env_execute(ctx, cmd, environment = env) 178 if result.return_code: 179 fail("failed to expunge build files: " + result.stderr) 180 181 # remove any other files 182 if ctx.attr.deleted_files: 183 result = env_execute(ctx, ["rm"] + [str(ctx.path(f)) for f in ctx.attr.deleted_files], environment = env) 184 if result.return_code: 185 fail("failed to remove deleted files: " + result.stderr) 186 187 if generate: 188 # Build file generation is needed. 189 build_file_name = existing_build_file or build_file_names[0] 190 191 # Populate Gazelle directive at root build file and 192 lines = ["# " + d for d in ctx.attr.build_directives] + [ 193 'load("@build_stack_rules_proto//rules:proto_repository_info.bzl", "proto_repository_info")', 194 'exports_files(["%s"])' % ctx.attr.imports_out, 195 "proto_repository_info(", 196 " name = 'proto_repository',", 197 " commit = '%s'," % ctx.attr.commit, 198 " tag = '%s'," % ctx.attr.tag, 199 " vcs = '%s'," % ctx.attr.vcs, 200 " urls = %s," % ctx.attr.urls, 201 " sha256 = '%s'," % ctx.attr.sha256, 202 " strip_prefix = '%s'," % ctx.attr.strip_prefix, 203 " source_host = '%s'," % ctx.attr.source_host, 204 " source_owner = '%s'," % ctx.attr.source_owner, 205 " source_repo = '%s'," % ctx.attr.source_repo, 206 " source_prefix = '%s'," % ctx.attr.source_prefix, 207 " source_commit = '%s'," % ctx.attr.source_commit, 208 " visibility = ['//visibility:public'],", 209 ")", 210 ] 211 ctx.file( 212 build_file_name, 213 "\n".join(lines), 214 ) 215 216 # Run Gazelle 217 _gazelle = "@proto_repository_tools//:bin/gazelle{}".format(executable_extension(ctx)) 218 gazelle = ctx.path(Label(_gazelle)) 219 cmd = [ 220 gazelle, 221 "-go_repository_mode", 222 "-mode", 223 "fix", 224 "-repo_root", 225 ctx.path(""), 226 "-repo_config", 227 ctx.path(ctx.attr.build_config), 228 "-proto_repo_name", 229 ctx.name, 230 ] 231 if ctx.attr.version: 232 cmd.append("-go_repository_module_mode") 233 if ctx.attr.importpath: 234 cmd.extend(["-go_prefix", ctx.attr.importpath]) 235 if ctx.attr.build_file_name: 236 cmd.extend(["-build_file_name", ctx.attr.build_file_name]) 237 if ctx.attr.build_tags: 238 cmd.extend(["-build_tags", ",".join(ctx.attr.build_tags)]) 239 if ctx.attr.build_external: 240 cmd.extend(["-external", ctx.attr.build_external]) 241 if ctx.attr.build_file_proto_mode: 242 cmd.extend(["-proto", ctx.attr.build_file_proto_mode]) 243 if ctx.attr.build_naming_convention: 244 cmd.extend(["-go_naming_convention", ctx.attr.build_naming_convention]) 245 246 # protobuf extension 247 if ctx.attr.languages: 248 cmd.extend(["-lang", ",".join(ctx.attr.languages)]) 249 if ctx.attr.cfgs: 250 cfgs = ",".join([str(ctx.path(f).realpath) for f in ctx.attr.cfgs]) 251 cmd.extend(["-proto_configs", cfgs]) 252 if ctx.attr.imports_out: 253 cmd.extend(["-proto_imports_out", ctx.path(ctx.attr.imports_out)]) 254 if ctx.attr.imports: 255 protoimports = ",".join([str(ctx.path(lbl).realpath) for lbl in ctx.attr.imports]) 256 cmd.extend(["-proto_imports_in", protoimports]) 257 if ctx.attr.reresolve_known_proto_imports: 258 cmd.extend(["-reresolve_known_proto_imports"]) 259 260 cmd.extend(ctx.attr.build_extra_args) 261 cmd.append(ctx.path("")) 262 263 # for arg in cmd: 264 # print("gazelle cmd arg", arg) 265 result = env_execute(ctx, cmd, environment = env, timeout = _GO_REPOSITORY_TIMEOUT) 266 if result.return_code: 267 fail("failed to generate BUILD files: %s" % ( 268 result.stderr, 269 )) 270 if result.stderr: 271 # buildifier: disable=print 272 print("%s: %s" % (ctx.name, result.stderr)) 273 274 # Apply patches if necessary. 275 patch(ctx) 276 277 go_repository = repository_rule( 278 implementation = _proto_repository_impl, 279 attrs = { 280 281 # Fundamental attributes of a go repository 282 "importpath": attr.string(mandatory = False), # True in go_repository 283 284 # Attributes for a repository that should be checked out from VCS 285 "commit": attr.string(), 286 "tag": attr.string(), 287 "vcs": attr.string( 288 default = "", 289 values = [ 290 "", 291 "git", 292 "hg", 293 "svn", 294 "bzr", 295 ], 296 ), 297 "remote": attr.string(), 298 299 # Attributes for a repository that should be downloaded via HTTP. 300 "urls": attr.string_list(), 301 "strip_prefix": attr.string(), 302 "type": attr.string(), 303 "sha256": attr.string(), 304 305 # Attributes for a repository that is publically hosted by github 306 "source_host": attr.string(default = "github.com"), 307 "source_owner": attr.string(), 308 "source_repo": attr.string(), 309 "source_prefix": attr.string(), 310 "source_commit": attr.string(), 311 312 # Attributes for a module that should be downloaded with the Go toolchain. 313 "version": attr.string(), 314 "sum": attr.string(), 315 "replace": attr.string(), 316 317 # Attributes for a repository that needs automatic build file generation 318 "build_external": attr.string( 319 values = [ 320 "", 321 "external", 322 "vendored", 323 ], 324 ), 325 "build_file_name": attr.string(default = "BUILD.bazel,BUILD"), 326 "build_file_generation": attr.string( 327 default = "auto", 328 values = [ 329 "auto", 330 "off", 331 "on", 332 ], 333 ), 334 "build_naming_convention": attr.string( 335 values = [ 336 "go_default_library", 337 "import", 338 "import_alias", 339 ], 340 default = "import_alias", 341 ), 342 "build_tags": attr.string_list(), 343 "build_file_proto_mode": attr.string( 344 values = [ 345 "", 346 "file", 347 "default", 348 "package", 349 "disable", 350 "disable_global", 351 "legacy", 352 ], 353 ), 354 "build_extra_args": attr.string_list(), 355 "build_config": attr.label(default = "@bazel_gazelle_go_repository_config//:WORKSPACE"), 356 "build_directives": attr.string_list(default = []), 357 358 # Patches to apply after running gazelle. 359 "patches": attr.label_list(), 360 "patch_tool": attr.string(default = "patch"), 361 "patch_args": attr.string_list(default = ["-p0"]), 362 "patch_cmds": attr.string_list(default = []), 363 364 # protobuf extension specific configuration 365 "build_file_expunge": attr.bool(), 366 "languages": attr.string_list(), 367 "cfgs": attr.label_list(allow_files = True), 368 "imports": attr.label_list( 369 allow_files = True, 370 ), 371 "imports_out": attr.string(default = "imports.csv"), 372 "deleted_files": attr.string_list(), 373 "reresolve_known_proto_imports": attr.bool(), 374 }, 375 ) 376 377 def proto_repository(**kwargs): 378 kwargs.setdefault("languages", ["proto", "protobuf"]) 379 kwargs.setdefault("build_file_expunge", True) 380 kwargs.setdefault("build_file_generation", "on") 381 go_repository(**kwargs) 382 383 # Copied from @bazel_tools//tools/build_defs/repo:utils.bzl 384 def patch(ctx): 385 """Implementation of patching an already extracted repository 386 387 Args: 388 ctx: the context object 389 """ 390 bash_exe = ctx.os.environ["BAZEL_SH"] if "BAZEL_SH" in ctx.os.environ else "bash" 391 for patchfile in ctx.attr.patches: 392 command = "{patchtool} {patch_args} < {patchfile}".format( 393 patchtool = ctx.attr.patch_tool, 394 patchfile = ctx.path(patchfile), 395 patch_args = " ".join([ 396 "'%s'" % arg 397 for arg in ctx.attr.patch_args 398 ]), 399 ) 400 st = ctx.execute([bash_exe, "-c", command]) 401 if st.return_code: 402 fail("Error applying patch %s:\n%s%s" % 403 (str(patchfile), st.stderr, st.stdout)) 404 for cmd in ctx.attr.patch_cmds: 405 st = ctx.execute([bash_exe, "-c", cmd]) 406 if st.return_code: 407 fail("Error applying patch command %s:\n%s%s" % 408 (cmd, st.stdout, st.stderr)) 409 410 def github_proto_repository(name, owner, repo, commit, prefix = "", host = "github.com", build_file_expunge = True, build_file_proto_mode = "file", **kwargs): 411 """github_proto_repository is a macro for a proto_repository hosted at github.com 412 413 Args: 414 name: the name of the rule 415 owner: the github owner (e.g. 'protocolbuffers') 416 repo: the github repo name (e.g. 'protobuf') 417 prefix: the strip_prefix value for the repo (e.g. 'src') 418 host: the source host (default 'github.com') 419 commit: the git commit (required for this macro) 420 build_file_expunge: defaults to true for this macro. 421 build_file_proto_mode: defaults to 'file' for this macro. 422 **kwargs: the kwargs accumulator 423 424 """ 425 strip_prefix = "%s-%s" % (repo, commit) 426 if prefix: 427 strip_prefix += "/" + prefix 428 429 proto_repository( 430 name = name, 431 source_host = host, 432 source_owner = owner, 433 source_repo = repo, 434 source_commit = commit, 435 source_prefix = prefix, 436 strip_prefix = strip_prefix, 437 build_file_expunge = build_file_expunge, 438 build_file_proto_mode = build_file_proto_mode, 439 urls = ["https://%s/%s/%s/archive/%s.tar.gz" % (host, owner, repo, commit)], 440 **kwargs 441 )