go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/lucicfg/starlark/stdlib/internal/luci/rules/builder.star (about) 1 # Copyright 2018 The LUCI Authors. 2 # 3 # Licensed under the Apache License, Version 2.0 (the "License"); 4 # you may not use this file except in compliance with the License. 5 # You may obtain a copy of the License at 6 # 7 # http://www.apache.org/licenses/LICENSE-2.0 8 # 9 # Unless required by applicable law or agreed to in writing, software 10 # distributed under the License is distributed on an "AS IS" BASIS, 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 # See the License for the specific language governing permissions and 13 # limitations under the License. 14 15 """Defines luci.builder(...) rule.""" 16 17 load("@stdlib//internal/experiments.star", "experiments") 18 load("@stdlib//internal/graph.star", "graph") 19 load("@stdlib//internal/lucicfg.star", "lucicfg") 20 load("@stdlib//internal/validate.star", "validate") 21 load("@stdlib//internal/luci/common.star", "builder_ref", "keys", "triggerer") 22 load("@stdlib//internal/luci/lib/resultdb.star", "resultdb", "resultdbimpl") 23 load("@stdlib//internal/luci/lib/scheduler.star", "schedulerimpl") 24 load("@stdlib//internal/luci/lib/swarming.star", "swarming") 25 load("@stdlib//internal/luci/rules/binding.star", "binding") 26 load("@stdlib//internal/luci/rules/bucket_constraints.star", "bucket_constraints") 27 28 def _generate_builder( 29 ctx, # @unused 30 *, 31 name = None, 32 bucket = None, 33 description_html = None, 34 35 # Execution environment parameters. 36 properties = None, 37 allowed_property_overrides = None, 38 service_account = None, 39 caches = None, 40 execution_timeout = None, 41 grace_period = None, 42 heartbeat_timeout = None, 43 44 # Scheduling parameters. 45 dimensions = None, 46 priority = None, 47 swarming_host = None, 48 swarming_tags = None, 49 expiration_timeout = None, 50 wait_for_capacity = None, 51 retriable = None, 52 53 # LUCI Scheduler parameters. 54 schedule = None, 55 triggering_policy = None, 56 57 # Tweaks. 58 build_numbers = None, 59 experimental = None, 60 experiments = None, 61 task_template_canary_percentage = None, 62 repo = None, 63 64 # Results. 65 resultdb_settings = None, 66 test_presentation = None, 67 68 # TaskBackend. 69 backend = None, 70 backend_alt = None, 71 72 # led build adjustments. 73 shadow_service_account = None, 74 shadow_pool = None, 75 shadow_properties = None, 76 shadow_dimensions = None, 77 78 # Builder health indicators 79 contact_team_email = None, 80 81 # Dynamic builder template. 82 dynamic = False): 83 """Helper function for defining a generic builder. 84 85 Shared function by luci.builder(...) and luci.dynamic_builder_template(...). 86 87 Args: 88 ctx: the implicit rule context, see lucicfg.rule(...). 89 name: name of the builder, will show up in UIs and logs. 90 Required if `dynamic` is False. 91 bucket: a bucket the builder is in, see luci.bucket(...) rule. Required. 92 description_html: description of the builder, will show up in UIs. See 93 https://pkg.go.dev/go.chromium.org/luci/common/data/text/sanitizehtml 94 for the list of allowed HTML elements. 95 properties: a dict with string keys and JSON-serializable values, defining 96 properties to pass to the executable. Supports the module-scoped 97 defaults. They are merged (non-recursively) with the explicitly passed 98 properties. 99 allowed_property_overrides: a list of top-level property keys that can 100 be overridden by users calling the buildbucket ScheduleBuild RPC. If 101 this is set exactly to ['*'], ScheduleBuild is allowed to override 102 any properties. Only property keys which are populated via the `properties` 103 parameter here (or via the module-scoped defaults) are allowed. 104 service_account: an email of a service account to run the executable 105 under: the executable (and various tools it calls, e.g. gsutil) will be 106 able to make outbound HTTP calls that have an OAuth access token 107 belonging to this service account (provided it is registered with LUCI). 108 Supports the module-scoped default. 109 caches: a list of swarming.cache(...) objects describing Swarming named 110 caches that should be present on the bot. See swarming.cache(...) doc 111 for more details. Supports the module-scoped defaults. They are joined 112 with the explicitly passed caches. 113 execution_timeout: how long to wait for a running build to finish before 114 forcefully aborting it and marking the build as timed out. If None, 115 defer the decision to Buildbucket service. Supports the module-scoped 116 default. 117 grace_period: how long to wait after the expiration of `execution_timeout` 118 or after a Cancel event, before the build is forcefully shut down. Your 119 build can use this time as a 'last gasp' to do quick actions like 120 killing child processes, cleaning resources, etc. Supports the 121 module-scoped default. 122 heartbeat_timeout: How long Buildbucket should wait for a running build to 123 send any updates before forcefully fail it with `INFRA_FAILURE`. If 124 None, Buildbucket won't check the heartbeat timeout. This field only 125 takes effect for builds that don't have Buildbucket managing their 126 underlying backend tasks, namely the ones on TaskBackendLite. E.g. 127 builds running on Swarming don't need to set this. 128 129 dimensions: a dict with swarming dimensions, indicating requirements for 130 a bot to execute the build. Keys are strings (e.g. `os`), and values 131 are either strings (e.g. `Linux`), swarming.dimension(...) objects (for 132 defining expiring dimensions) or lists of thereof. Supports the 133 module-scoped defaults. They are merged (non-recursively) with the 134 explicitly passed dimensions. 135 priority: int [1-255] or None, indicating swarming task priority, lower is 136 more important. If None, defer the decision to Buildbucket service. 137 Supports the module-scoped default. 138 swarming_host: appspot hostname of a Swarming service to use for this 139 builder instead of the default specified in luci.project(...). Use with 140 great caution. Supports the module-scoped default. 141 swarming_tags: Deprecated. Used only to enable 142 "vpython:native-python-wrapper" and does not actually propagate to 143 Swarming. A list of tags (`k:v` strings). 144 expiration_timeout: how long to wait for a build to be picked up by a 145 matching bot (based on `dimensions`) before canceling the build and 146 marking it as expired. If None, defer the decision to Buildbucket 147 service. Supports the module-scoped default. 148 wait_for_capacity: tell swarming to wait for `expiration_timeout` even if 149 it has never seen a bot whose dimensions are a superset of the requested 150 dimensions. This is useful if this builder has bots whose dimensions 151 are mutated dynamically. Supports the module-scoped default. 152 retriable: control if the builds on the builder can be retried. Supports 153 the module-scoped default. 154 155 schedule: string with a cron schedule that describes when to run this 156 builder. See [Defining cron schedules](#schedules-doc) for the expected 157 format of this field. If None, the builder will not be running 158 periodically. 159 triggering_policy: scheduler.policy(...) struct with a configuration that 160 defines when and how LUCI Scheduler should launch new builds in response 161 to triggering requests from luci.gitiles_poller(...) or from 162 EmitTriggers API. Does not apply to builds started directly through 163 Buildbucket. By default, only one concurrent build is allowed and while 164 it runs, triggering requests accumulate in a queue. Once the build 165 finishes, if the queue is not empty, a new build starts right away, 166 "consuming" all pending requests. See scheduler.policy(...) doc for more 167 details. Supports the module-scoped default. 168 169 build_numbers: if True, generate monotonically increasing contiguous 170 numbers for each build, unique within the builder. If None, defer the 171 decision to Buildbucket service. Supports the module-scoped default. 172 experimental: if True, by default a new build in this builder will be 173 marked as experimental. This is seen from the executable and it may 174 behave differently (e.g. avoiding any side-effects). If None, defer the 175 decision to Buildbucket service. Supports the module-scoped default. 176 experiments: a dict that maps experiment name to percentage chance that it 177 will apply to builds generated from this builder. Keys are strings, 178 and values are integers from 0 to 100. This is unrelated to 179 lucicfg.enable_experiment(...). 180 task_template_canary_percentage: int [0-100] or None, indicating 181 percentage of builds that should use a canary swarming task template. 182 If None, defer the decision to Buildbucket service. Supports the 183 module-scoped default. 184 repo: URL of a primary git repository (starting with `https://`) 185 associated with the builder, if known. It is in particular important 186 when using luci.notifier(...) to let LUCI know what git history it 187 should use to chronologically order builds on this builder. If unknown, 188 builds will be ordered by creation time. If unset, will be taken from 189 the configuration of luci.gitiles_poller(...) that trigger this builder 190 if they all poll the same repo. 191 192 resultdb_settings: A buildbucket_pb.BuilderConfig.ResultDB, such as one 193 created with resultdb.settings(...). A configuration that defines if 194 Buildbucket:ResultDB integration should be enabled for this builder and 195 which results to export to BigQuery. 196 test_presentation: A resultdb.test_presentation(...) struct. A 197 configuration that defines how tests should be rendered in the UI. 198 199 backend: the name of the task backend defined via luci.task_backend(...). 200 Supports the module-scoped default. 201 202 backend_alt: the name of the alternative task backend defined via 203 luci.task_backend(...). Supports the module-scoped default. 204 205 shadow_service_account: If set, the led builds created for this Builder 206 will instead use this service account. This is useful to allow users to 207 automatically have their testing builds assume a service account which 208 is different than your production service account. 209 When specified, the shadow_service_account will also be included into 210 the shadow bucket's constraints (see luci.bucket_constraints(...)). 211 Which also means it will be granted the 212 `role/buildbucket.builderServiceAccount` role in the shadow bucket realm. 213 shadow_pool: If set, the led builds created for this Builder will instead 214 be set to use this alternate pool instead. This would allow you to grant 215 users the ability to create led builds in the alternate pool without 216 allowing them to create builds in the production pool. 217 When specified, the shadow_pool will also be included into 218 the shadow bucket's constraints (see luci.bucket_constraints(...)) and 219 a "pool:<shadow_pool>" dimension will be automatically added to 220 shadow_dimensions. 221 shadow_properties: If set, the led builds created for this Builder will 222 override the top-level input properties with the same keys. 223 shadow_dimensions: If set, the led builds created for this Builder will 224 override the dimensions with the same keys. Note: for historical reasons 225 pool can be set individually. If a "pool:<shadow_pool>" dimension is 226 included here, it would have the same effect as setting shadow_pool. 227 shadow_dimensions support dimensions with None values. It's useful for 228 led builds to remove some dimensions the production builds use. 229 230 contact_team_email: the owning team's contact email. This team is responsible for fixing 231 any builder health issues (see BuilderConfig.ContactTeamEmail). 232 233 dynamic: Flag for if to generate a dynamic_builder_template or a pre-defined builder. 234 """ 235 if dynamic: 236 if name != "" and name != None: 237 fail("name must be unset for dynamic builder template") 238 else: 239 name = validate.string("name", name) 240 bucket_key = keys.bucket(bucket) 241 242 # TODO(vadimsh): Validators here and in lucicfg.rule(..., defaults = ...) 243 # are duplicated. There's probably a way to avoid this by introducing a 244 # Schema object. 245 props = { 246 "name": name, 247 "bucket": bucket_key.id, 248 "realm": bucket_key.id, 249 "description_html": validate.string("description_html", description_html, required = False), 250 "project": "", # means "whatever is being defined right now" 251 "properties": validate.str_dict("properties", properties), 252 "allowed_property_overrides": validate.str_list("allowed_property_overrides", allowed_property_overrides), 253 "service_account": validate.string("service_account", service_account, required = False), 254 "caches": swarming.validate_caches("caches", caches), 255 "execution_timeout": validate.duration("execution_timeout", execution_timeout, required = False), 256 "grace_period": validate.duration("grace_period", grace_period, required = False), 257 "heartbeat_timeout": validate.duration("heartbeat_timeout", heartbeat_timeout, required = False), 258 "dimensions": swarming.validate_dimensions("dimensions", dimensions, allow_none = True), 259 "priority": validate.int("priority", priority, min = 1, max = 255, required = False), 260 "swarming_host": validate.string("swarming_host", swarming_host, required = False), 261 "swarming_tags": swarming.validate_tags("swarming_tags", swarming_tags), 262 "expiration_timeout": validate.duration("expiration_timeout", expiration_timeout, required = False), 263 "wait_for_capacity": validate.bool("wait_for_capacity", wait_for_capacity, required = False), 264 "retriable": validate.bool("retriable", retriable, required = False), 265 "schedule": validate.string("schedule", schedule, required = False), 266 "triggering_policy": schedulerimpl.validate_policy("triggering_policy", triggering_policy, required = False), 267 "build_numbers": validate.bool("build_numbers", build_numbers, required = False), 268 "experimental": validate.bool("experimental", experimental, required = False), 269 "experiments": _validate_experiments("experiments", experiments, allow_none = True), 270 "task_template_canary_percentage": validate.int("task_template_canary_percentage", task_template_canary_percentage, min = 0, max = 100, required = False), 271 "repo": validate.repo_url("repo", repo, required = False), 272 "resultdb": resultdb.validate_settings("settings", resultdb_settings), 273 "test_presentation": resultdb.validate_test_presentation("test_presentation", test_presentation), 274 "backend": keys.task_backend(backend) if backend != None else None, 275 "backend_alt": keys.task_backend(backend_alt) if backend_alt != None else None, 276 "shadow_service_account": validate.string("shadow_service_account", shadow_service_account, required = False), 277 "shadow_pool": validate.string("shadow_pool", shadow_pool, required = False), 278 "shadow_properties": validate.str_dict("shadow_properties", shadow_properties, required = False), 279 "shadow_dimensions": swarming.validate_dimensions("shadow_dimensions", shadow_dimensions, allow_none = True), 280 "contact_team_email": validate.email("contact_team_email", contact_team_email, required = False), 281 } 282 283 # Merge explicitly passed properties with the module-scoped defaults. 284 for k, prop_val in props.items(): 285 var = getattr(ctx.defaults, k, None) 286 def_val = var.get() if var else None 287 if def_val == None: 288 continue 289 if k in ("properties", "dimensions", "experiments"): 290 props[k] = _merge_dicts(def_val, prop_val) 291 elif k in ("allowed_property_overrides", "caches", "swarming_tags"): 292 props[k] = _merge_lists(def_val, prop_val) 293 elif prop_val == None: 294 props[k] = def_val 295 296 # Check to see if allowed_property_overrides is allowing override for 297 # properties not supplied. 298 if props["allowed_property_overrides"] and props["allowed_property_overrides"] != ["*"]: 299 # Do a de-duplication pass 300 props["allowed_property_overrides"] = sorted(set(props["allowed_property_overrides"])) 301 for override in props["allowed_property_overrides"]: 302 if "*" in override: 303 fail("allowed_property_overrides does not support wildcards: %r" % override) 304 elif override not in props["properties"]: 305 fail("%r listed in allowed_property_overrides but not in properties" % override) 306 307 test_presentation = props.pop("test_presentation") 308 309 # To reduce noise in the properties, set the test presentation config only 310 # when it's not the default value. 311 if test_presentation != None and test_presentation != resultdb.test_presentation(): 312 # Copy the properties dictionary so we won't modify the original value, 313 # which could be immutable if no default value was provided. 314 props["properties"] = dict(props["properties"]) 315 props["properties"]["$recipe_engine/resultdb/test_presentation"] = resultdbimpl.test_presentation_to_dict(test_presentation) 316 317 # Properties and shadow_properties should be JSON-serializable. 318 # The only way to check is to try to serialize. We do it here (instead of 319 # generators.star) to get a more informative stack trace. 320 _ = to_json(props["properties"]) # @unused 321 _ = to_json(props["shadow_properties"]) # @unused 322 323 # There should be no dimensions and experiments with value None after 324 # merging. 325 swarming.validate_dimensions("dimensions", props["dimensions"], allow_none = False) 326 _validate_experiments("experiments", props["experiments"], allow_none = False) 327 328 # Update shadow_pool or shadow_dimensions. 329 if shadow_dimensions: 330 pools_in_dimensions = [p.value for p in props["shadow_dimensions"].get("pool", [])] 331 if shadow_pool: 332 if len(pools_in_dimensions) > 0: 333 for p in pools_in_dimensions: 334 if shadow_pool != p: 335 fail("shadow_pool and pool dimension in shadow_dimensions should have the same value") 336 else: 337 props["shadow_dimensions"]["pool"] = [swarming.dimension(shadow_pool)] 338 elif len(pools_in_dimensions) == 1: 339 props["shadow_pool"] = pools_in_dimensions[0] 340 elif shadow_pool: 341 props["shadow_dimensions"] = { 342 "pool": [swarming.dimension(shadow_pool)], 343 } 344 345 # Setup a binding that allows the service account to be used for builds 346 # in the bucket's realm. 347 if props["service_account"]: 348 binding( 349 realm = bucket_key.id, 350 roles = "role/buildbucket.builderServiceAccount", 351 users = props["service_account"], 352 ) 353 354 if _apply_builder_config_as_bucket_constraints.is_enabled(): 355 # Implicitly add constraints to this builder's bucket. 356 pools = [p.value for p in props["dimensions"].get("pool", [])] 357 service_accounts = [props["service_account"]] if props["service_account"] else [] 358 if pools or service_accounts: 359 bucket_constraints( 360 bucket = bucket_key.id, 361 pools = pools, 362 service_accounts = service_accounts, 363 ) 364 return props 365 366 # Enables the application of a builder's config (more specifically pool and 367 # service_account) to its bucket as constraints. 368 _apply_builder_config_as_bucket_constraints = experiments.register("crbug.com/1338648") 369 370 def _builder( 371 ctx, # @unused 372 *, 373 name = None, 374 bucket = None, 375 description_html = None, 376 executable = None, 377 378 # Execution environment parameters. 379 properties = None, 380 allowed_property_overrides = None, 381 service_account = None, 382 caches = None, 383 execution_timeout = None, 384 grace_period = None, 385 heartbeat_timeout = None, 386 387 # Scheduling parameters. 388 dimensions = None, 389 priority = None, 390 swarming_host = None, 391 swarming_tags = None, 392 expiration_timeout = None, 393 wait_for_capacity = None, 394 retriable = None, 395 396 # LUCI Scheduler parameters. 397 schedule = None, 398 triggering_policy = None, 399 400 # Tweaks. 401 build_numbers = None, 402 experimental = None, 403 experiments = None, 404 task_template_canary_percentage = None, 405 repo = None, 406 407 # Results. 408 resultdb_settings = None, 409 test_presentation = None, 410 411 # TaskBackend. 412 backend = None, 413 backend_alt = None, 414 415 # led build adjustments. 416 shadow_service_account = None, 417 shadow_pool = None, 418 shadow_properties = None, 419 shadow_dimensions = None, 420 421 # Relations. 422 triggers = None, 423 triggered_by = None, 424 notifies = None, 425 426 # Builder health indicators 427 contact_team_email = None): 428 """Defines a generic builder. 429 430 It runs some executable (usually a recipe) in some requested environment, 431 passing it a struct with given properties. It is launched whenever something 432 triggers it (a poller or some other builder, or maybe some external actor 433 via Buildbucket or LUCI Scheduler APIs). 434 435 The full unique builder name (as expected by Buildbucket RPC interface) is 436 a pair `(<project>, <bucket>/<name>)`, but within a single project config 437 this builder can be referred to either via its bucket-scoped name (i.e. 438 `<bucket>/<name>`) or just via it's name alone (i.e. `<name>`), if this 439 doesn't introduce ambiguities. 440 441 The definition of what can *potentially* trigger what is defined through 442 `triggers` and `triggered_by` fields. They specify how to prepare ACLs and 443 other configuration of services that execute builds. If builder **A** is 444 defined as "triggers builder **B**", it means all services should expect 445 **A** builds to trigger **B** builds via LUCI Scheduler's EmitTriggers RPC 446 or via Buildbucket's ScheduleBuild RPC, but the actual triggering is still 447 the responsibility of **A**'s executable. 448 449 There's a caveat though: only Scheduler ACLs are auto-generated by the 450 config generator when one builder triggers another, because each Scheduler 451 job has its own ACL and we can precisely configure who's allowed to trigger 452 this job. Buildbucket ACLs are left unchanged, since they apply to an entire 453 bucket, and making a large scale change like that (without really knowing 454 whether Buildbucket API will be used) is dangerous. If the executable 455 triggers other builds directly through Buildbucket, it is the responsibility 456 of the config author (you) to correctly specify Buildbucket ACLs, for 457 example by adding the corresponding service account to the bucket ACLs: 458 459 ```python 460 luci.bucket( 461 ... 462 acls = [ 463 ... 464 acl.entry(acl.BUILDBUCKET_TRIGGERER, <builder service account>), 465 ... 466 ], 467 ) 468 ``` 469 470 This is not necessary if the executable uses Scheduler API instead of 471 Buildbucket. 472 473 Args: 474 ctx: the implicit rule context, see lucicfg.rule(...). 475 name: name of the builder, will show up in UIs and logs. Required. 476 bucket: a bucket the builder is in, see luci.bucket(...) rule. Required. 477 description_html: description of the builder, will show up in UIs. See 478 https://pkg.go.dev/go.chromium.org/luci/common/data/text/sanitizehtml 479 for the list of allowed HTML elements. 480 executable: an executable to run, e.g. a luci.recipe(...) or 481 luci.executable(...). Required. 482 properties: a dict with string keys and JSON-serializable values, defining 483 properties to pass to the executable. Supports the module-scoped 484 defaults. They are merged (non-recursively) with the explicitly passed 485 properties. 486 allowed_property_overrides: a list of top-level property keys that can 487 be overridden by users calling the buildbucket ScheduleBuild RPC. If 488 this is set exactly to ['*'], ScheduleBuild is allowed to override 489 any properties. Only property keys which are populated via the `properties` 490 parameter here (or via the module-scoped defaults) are allowed. 491 service_account: an email of a service account to run the executable 492 under: the executable (and various tools it calls, e.g. gsutil) will be 493 able to make outbound HTTP calls that have an OAuth access token 494 belonging to this service account (provided it is registered with LUCI). 495 Supports the module-scoped default. 496 caches: a list of swarming.cache(...) objects describing Swarming named 497 caches that should be present on the bot. See swarming.cache(...) doc 498 for more details. Supports the module-scoped defaults. They are joined 499 with the explicitly passed caches. 500 execution_timeout: how long to wait for a running build to finish before 501 forcefully aborting it and marking the build as timed out. If None, 502 defer the decision to Buildbucket service. Supports the module-scoped 503 default. 504 grace_period: how long to wait after the expiration of `execution_timeout` 505 or after a Cancel event, before the build is forcefully shut down. Your 506 build can use this time as a 'last gasp' to do quick actions like 507 killing child processes, cleaning resources, etc. Supports the 508 module-scoped default. 509 heartbeat_timeout: How long Buildbucket should wait for a running build to 510 send any updates before forcefully fail it with `INFRA_FAILURE`. If 511 None, Buildbucket won't check the heartbeat timeout. This field only 512 takes effect for builds that don't have Buildbucket managing their 513 underlying backend tasks, namely the ones on TaskBackendLite. E.g. 514 builds running on Swarming don't need to set this. 515 516 dimensions: a dict with swarming dimensions, indicating requirements for 517 a bot to execute the build. Keys are strings (e.g. `os`), and values 518 are either strings (e.g. `Linux`), swarming.dimension(...) objects (for 519 defining expiring dimensions) or lists of thereof. Supports the 520 module-scoped defaults. They are merged (non-recursively) with the 521 explicitly passed dimensions. 522 priority: int [1-255] or None, indicating swarming task priority, lower is 523 more important. If None, defer the decision to Buildbucket service. 524 Supports the module-scoped default. 525 swarming_host: appspot hostname of a Swarming service to use for this 526 builder instead of the default specified in luci.project(...). Use with 527 great caution. Supports the module-scoped default. 528 swarming_tags: Deprecated. Used only to enable 529 "vpython:native-python-wrapper" and does not actually propagate to 530 Swarming. A list of tags (`k:v` strings). 531 expiration_timeout: how long to wait for a build to be picked up by a 532 matching bot (based on `dimensions`) before canceling the build and 533 marking it as expired. If None, defer the decision to Buildbucket 534 service. Supports the module-scoped default. 535 wait_for_capacity: tell swarming to wait for `expiration_timeout` even if 536 it has never seen a bot whose dimensions are a superset of the requested 537 dimensions. This is useful if this builder has bots whose dimensions 538 are mutated dynamically. Supports the module-scoped default. 539 retriable: control if the builds on the builder can be retried. Supports 540 the module-scoped default. 541 542 schedule: string with a cron schedule that describes when to run this 543 builder. See [Defining cron schedules](#schedules-doc) for the expected 544 format of this field. If None, the builder will not be running 545 periodically. 546 triggering_policy: scheduler.policy(...) struct with a configuration that 547 defines when and how LUCI Scheduler should launch new builds in response 548 to triggering requests from luci.gitiles_poller(...) or from 549 EmitTriggers API. Does not apply to builds started directly through 550 Buildbucket. By default, only one concurrent build is allowed and while 551 it runs, triggering requests accumulate in a queue. Once the build 552 finishes, if the queue is not empty, a new build starts right away, 553 "consuming" all pending requests. See scheduler.policy(...) doc for more 554 details. Supports the module-scoped default. 555 556 build_numbers: if True, generate monotonically increasing contiguous 557 numbers for each build, unique within the builder. If None, defer the 558 decision to Buildbucket service. Supports the module-scoped default. 559 experimental: if True, by default a new build in this builder will be 560 marked as experimental. This is seen from the executable and it may 561 behave differently (e.g. avoiding any side-effects). If None, defer the 562 decision to Buildbucket service. Supports the module-scoped default. 563 experiments: a dict that maps experiment name to percentage chance that it 564 will apply to builds generated from this builder. Keys are strings, 565 and values are integers from 0 to 100. This is unrelated to 566 lucicfg.enable_experiment(...). 567 task_template_canary_percentage: int [0-100] or None, indicating 568 percentage of builds that should use a canary swarming task template. 569 If None, defer the decision to Buildbucket service. Supports the 570 module-scoped default. 571 repo: URL of a primary git repository (starting with `https://`) 572 associated with the builder, if known. It is in particular important 573 when using luci.notifier(...) to let LUCI know what git history it 574 should use to chronologically order builds on this builder. If unknown, 575 builds will be ordered by creation time. If unset, will be taken from 576 the configuration of luci.gitiles_poller(...) that trigger this builder 577 if they all poll the same repo. 578 579 resultdb_settings: A buildbucket_pb.BuilderConfig.ResultDB, such as one 580 created with resultdb.settings(...). A configuration that defines if 581 Buildbucket:ResultDB integration should be enabled for this builder and 582 which results to export to BigQuery. 583 test_presentation: A resultdb.test_presentation(...) struct. A 584 configuration that defines how tests should be rendered in the UI. 585 586 backend: the name of the task backend defined via luci.task_backend(...). 587 Supports the module-scoped default. 588 589 backend_alt: the name of the alternative task backend defined via 590 luci.task_backend(...). Supports the module-scoped default. 591 592 shadow_service_account: If set, the led builds created for this Builder 593 will instead use this service account. This is useful to allow users to 594 automatically have their testing builds assume a service account which 595 is different than your production service account. 596 When specified, the shadow_service_account will also be included into 597 the shadow bucket's constraints (see luci.bucket_constraints(...)). 598 Which also means it will be granted the 599 `role/buildbucket.builderServiceAccount` role in the shadow bucket realm. 600 shadow_pool: If set, the led builds created for this Builder will instead 601 be set to use this alternate pool instead. This would allow you to grant 602 users the ability to create led builds in the alternate pool without 603 allowing them to create builds in the production pool. 604 When specified, the shadow_pool will also be included into 605 the shadow bucket's constraints (see luci.bucket_constraints(...)) and 606 a "pool:<shadow_pool>" dimension will be automatically added to 607 shadow_dimensions. 608 shadow_properties: If set, the led builds created for this Builder will 609 override the top-level input properties with the same keys. 610 shadow_dimensions: If set, the led builds created for this Builder will 611 override the dimensions with the same keys. Note: for historical reasons 612 pool can be set individually. If a "pool:<shadow_pool>" dimension is 613 included here, it would have the same effect as setting shadow_pool. 614 shadow_dimensions support dimensions with None values. It's useful for 615 led builds to remove some dimensions the production builds use. 616 617 triggers: builders this builder triggers. 618 triggered_by: builders or pollers this builder is triggered by. 619 notifies: list of luci.notifier(...) or luci.tree_closer(...) the builder 620 notifies when it changes its status. This relation can also be defined 621 via `notified_by` field in luci.notifier(...) or luci.tree_closer(...). 622 623 contact_team_email: the owning team's contact email. This team is responsible for fixing 624 any builder health issues (see BuilderConfig.ContactTeamEmail). 625 """ 626 627 name = validate.string("name", name) 628 bucket_key = keys.bucket(bucket) 629 executable_key = keys.executable(executable) 630 631 props = _generate_builder( 632 ctx, 633 name = name, 634 bucket = bucket, 635 description_html = description_html, 636 properties = properties, 637 allowed_property_overrides = allowed_property_overrides, 638 service_account = service_account, 639 caches = caches, 640 execution_timeout = execution_timeout, 641 grace_period = grace_period, 642 heartbeat_timeout = heartbeat_timeout, 643 644 # Scheduling parameters. 645 dimensions = dimensions, 646 priority = priority, 647 swarming_host = swarming_host, 648 swarming_tags = swarming_tags, 649 expiration_timeout = expiration_timeout, 650 wait_for_capacity = wait_for_capacity, 651 retriable = retriable, 652 653 # LUCI Scheduler parameters. 654 schedule = schedule, 655 triggering_policy = triggering_policy, 656 657 # Tweaks. 658 build_numbers = build_numbers, 659 experimental = experimental, 660 experiments = experiments, 661 task_template_canary_percentage = task_template_canary_percentage, 662 repo = repo, 663 664 # Results. 665 resultdb_settings = resultdb_settings, 666 test_presentation = test_presentation, 667 668 # TaskBackend. 669 backend = backend, 670 backend_alt = backend_alt, 671 672 # led build adjustments. 673 shadow_service_account = shadow_service_account, 674 shadow_pool = shadow_pool, 675 shadow_properties = shadow_properties, 676 shadow_dimensions = shadow_dimensions, 677 678 # Builder health indicators 679 contact_team_email = contact_team_email, 680 ) 681 682 # Add a node that carries the full definition of the builder. 683 builder_key = keys.builder(bucket_key.id, name) 684 graph.add_node(builder_key, props = props) 685 graph.add_edge(bucket_key, builder_key) 686 graph.add_edge(builder_key, executable_key) 687 if props["backend"]: 688 graph.add_edge(builder_key, props["backend"]) 689 if props["backend_alt"]: 690 graph.add_edge(builder_key, props["backend_alt"]) 691 692 # Allow this builder to be referenced from other nodes via its bucket-scoped 693 # name and via a global (perhaps ambiguous) name. See builder_ref.add(...). 694 # Ambiguity is checked during the graph traversal via 695 # builder_ref.follow(...). 696 builder_ref_key = builder_ref.add(builder_key) 697 698 # Setup nodes that indicate this builder can be referenced in 'triggered_by' 699 # relations (either via its bucket-scoped name or via its global name). 700 triggerer_key = triggerer.add(builder_key) 701 702 # Link to builders triggered by this builder. 703 for t in validate.list("triggers", triggers): 704 graph.add_edge( 705 parent = triggerer_key, 706 child = keys.builder_ref(t), 707 title = "triggers", 708 ) 709 710 # And link to nodes this builder is triggered by. 711 for t in validate.list("triggered_by", triggered_by): 712 graph.add_edge( 713 parent = keys.triggerer(t), 714 child = builder_ref_key, 715 title = "triggered_by", 716 ) 717 718 # Subscribe notifiers/tree closers to this builder. 719 for n in validate.list("notifies", notifies): 720 graph.add_edge( 721 parent = keys.notifiable(n), 722 child = builder_ref_key, 723 title = "notifies", 724 ) 725 return graph.keyset(builder_key, builder_ref_key, triggerer_key) 726 727 def _merge_dicts(defaults, extra): 728 out = dict(defaults.items()) 729 for k, v in extra.items(): 730 if v != None: 731 out[k] = v 732 return out 733 734 def _merge_lists(defaults, extra): 735 return defaults + extra 736 737 def _validate_experiments(attr, val, allow_none = False): 738 """Validates that the value is a dict of {string: int[1-100]} 739 740 Args: 741 attr: field name with this value, for error messages. 742 val: a value to validate. 743 allow_none: True to also allow None as dict values. 744 745 Returns: 746 The validated dict or {}. 747 """ 748 if val == None: 749 return {} 750 751 validate.str_dict(attr, val) 752 753 for k, percent in val.items(): 754 if percent == None and allow_none: 755 continue 756 perc_type = type(percent) 757 if perc_type != "int": 758 fail("bad %r: got %s for key %s, want int from 0-100" % (attr, perc_type, k)) 759 if percent < 0 or percent > 100: 760 fail("bad %r: %d should be between 0-100" % (attr, percent)) 761 762 return val 763 764 builder = lucicfg.rule( 765 impl = _builder, 766 defaults = validate.vars_with_validators({ 767 "properties": validate.str_dict, 768 "allowed_property_overrides": validate.str_list, 769 "service_account": validate.string, 770 "caches": swarming.validate_caches, 771 "execution_timeout": validate.duration, 772 "grace_period": validate.duration, 773 "heartbeat_timeout": validate.duration, 774 "dimensions": swarming.validate_dimensions, 775 "priority": lambda attr, val: validate.int(attr, val, min = 1, max = 255), 776 "swarming_host": validate.string, 777 "swarming_tags": swarming.validate_tags, 778 "expiration_timeout": validate.duration, 779 "wait_for_capacity": validate.bool, 780 "retriable": validate.bool, 781 "triggering_policy": schedulerimpl.validate_policy, 782 "build_numbers": validate.bool, 783 "experimental": validate.bool, 784 "experiments": _validate_experiments, 785 "task_template_canary_percentage": lambda attr, val: validate.int(attr, val, min = 0, max = 100), 786 "resultdb": resultdb.validate_settings, 787 "test_presentation": resultdb.validate_test_presentation, 788 "backend": lambda _attr, val: keys.task_backend(val), 789 "backend_alt": lambda _attr, val: keys.task_backend(val), 790 }), 791 ) 792 793 builderimpl = struct( 794 generate_builder = _generate_builder, 795 )