github.com/sercand/please@v13.4.0+incompatible/src/parse/rules/misc_rules.build_defs (about)

     1  """Miscellaneous rules that aren't language-specific."""
     2  
     3  
     4  def genrule(name:str, cmd:str|list|dict, srcs:list|dict=None, out:str=None, outs:list|dict=None, deps:list=None,
     5              labels:list&features&tags=None, visibility:list=None, building_description:str='Building...',
     6              hashes:list=None, timeout:int=0, binary:bool=False, sandbox:bool=None,
     7              needs_transitive_deps:bool=False, output_is_complete:bool=True, test_only:bool&testonly=False,
     8              secrets:list=None, requires:list=None, provides:dict=None, pre_build:function=None,
     9              post_build:function=None, tools:list|dict=None):
    10      """A general build rule which allows the user to specify a command.
    11  
    12      Args:
    13        name (str): Name of the rule
    14        cmd (str | dict | list): Command to run.
    15             If given as a string, it's a single command that's used always (the most common case).
    16             If given as a list, it's a sequence of commands run one after another if the
    17             preceding command is successful (i.e. it's equivalent to ' && '.join(cmd)).
    18             If given as a dict, the keys correspond to the build config to run and the values
    19             are the command to run in that config. The typical use case here is 'opt' vs. 'dbg'
    20             but arbitrary names can be given and specified with `plz build -c name`.
    21             See https://please.build/build_rules.html for more information about the sequence
    22             replacements and environment variables that are made available to this rule.
    23        srcs (list | dict): Sources of this rule. Can be a list of files or rules, or a dict of names
    24                            to lists. In the latter case they can be accessed separately which is useful
    25                            to be able to refer to them distinctly in the command.
    26        outs (list | dict): Outputs of this rule. All are files relative to this package.
    27                            If given as a dict, it declares a series of named outputs which work
    28                            similarly to named srcs; they have separate environment variables and
    29                            can be accessed directly by other rules.
    30        out (str): A single output of this rule, as a string. Discouraged in favour of 'outs'.
    31        deps (list): Dependencies of this rule.
    32        tools (list | dict): Tools used to build this rule; similar to srcs but are not copied to the
    33                             temporary build directory. Should be accessed via $(exe //path/to:tool)
    34                             or similar.
    35                             Entries that are not build labels are assumed to be system-level commands
    36                             and resolved within the path given in the config file (note that the
    37                             value of $PATH in the outside environment is not propagated to the build
    38                             rule).
    39                             If tools is given as a dict then the keys of the dict name the various
    40                             tools and they can be accessed with $TOOLS_KEY.
    41        secrets (list): Files containing secrets (credentials etc) used to build this rule. These are
    42                        all absolute paths (beginning with / or ~) and are not copied to the build
    43                        directory. They can be accessed through the environment variable $SECRETS.
    44                        They don't contribute to the key used to retrieve outputs from the cache; this
    45                        means it's possible for one machine to build a target with the secret and then
    46                        share the output with others. That also implies that secrets are things like
    47                        credentials or signing keys and shouldn't be copied directly to outputs, otherwise
    48                        they might become more widely known.
    49        labels (list): Labels to apply to this rule.
    50        visibility (list): Visibility declaration of this rule
    51        building_description (str): Description to display to the user while the rule is building.
    52        hashes (list): List of hashes; if given the outputs must match one of these. They can be
    53                optionally preceded by their method. Currently the only supported method is sha1.
    54        timeout (int): Maximum time in seconds this rule can run for before being killed.
    55        binary (bool): True to mark a rule that produces a runnable output. Its output will be placed into
    56                plz-out/bin instead of plz-out/gen and can be run with 'plz run'. Binary rules
    57                can only have a single output.
    58        sandbox (bool): If True, the build action is sandboxed within a separate network and process
    59                        namespace. Only works on Linux and requires plz_sandbox to be installed
    60                        separately.
    61                        If False, it opts the target out of sandboxing when that is turned on.
    62        needs_transitive_deps (bool): If True, all transitive dependencies of the rule will be made
    63                               available to it when it builds (although see below...). By default
    64                               rules only get their immediate dependencies.
    65        output_is_complete (bool): If this is true then the rule blocks downwards searches of transitive
    66                            dependencies by other rules (ie. it will be available to them, but not
    67                            its dependencies as well).
    68        test_only (bool): If True it can only be used by test rules.
    69        requires (list): A list of arbitrary strings that define kinds of output that this rule might want.
    70                  See 'provides' for more detail; it's mostly useful to match up rules with multiple
    71                  kinds of output with ones that only need one of them, eg. a proto_library with
    72                  a python_library that doesn't want the C++ or Java proto outputs.
    73                  Entries in 'requires' are also implicitly labels on the rule.
    74        provides (dict): A map of arbitrary strings to dependencies of the rule that provide some specific
    75                  type of thing. For example:
    76                    provides = {'py': ':python_rule', 'go': ':go_rule'},
    77                  A Python rule would have requires = ['py'] and so if it depended on a rule like
    78                  this it would pick up a dependency on :python_rule instead. See the proto rules
    79                  for an example of where this is useful.
    80                  Note that the keys of provides and entries in requires are arbitrary and
    81                  have no effect until a matched pair meet one another.
    82        pre_build (function): A function to be executed immediately before the rule builds. It receives one
    83                   argument, the name of the building rule. This is mostly useful to interrogate
    84                   the metadata of dependent rules which isn't generally available at parse time;
    85                   see the get_labels function for a motivating example.
    86        post_build (function): A function to be executed immediately after the rule builds. It receives two
    87                    arguments, the rule name and its command line output.
    88                    This is significantly more useful than the pre_build function, it can be used
    89                    to dynamically create new rules based on the output of another.
    90      """
    91      if out and outs:
    92          raise TypeError('Can\'t specify both "out" and "outs".')
    93      return build_rule(
    94          name = name,
    95          srcs = srcs,
    96          outs = [out] if out else outs,
    97          cmd = ' && '.join(cmd) if isinstance(cmd, list) else cmd,
    98          deps = deps,
    99          tools = tools,
   100          secrets = secrets,
   101          labels = labels,
   102          visibility  =  visibility,
   103          output_is_complete = output_is_complete,
   104          building_description = building_description,
   105          hashes = hashes,
   106          post_build = post_build,
   107          binary = binary,
   108          sandbox = sandbox,
   109          build_timeout = timeout,
   110          needs_transitive_deps = needs_transitive_deps,
   111          requires = requires,
   112          provides = provides,
   113          test_only = test_only,
   114      )
   115  
   116  
   117  def gentest(name:str, test_cmd:str|dict, labels:list&features&tags=None, cmd:str|dict=None, srcs:list|dict=None, outs:list=None,
   118              deps:list=None, tools:list|dict=None, data:list=None, visibility:list=None, timeout:int=0,
   119              needs_transitive_deps:bool=False, flaky:bool|int=0, secrets:list=None, no_test_output:bool=False,
   120              test_outputs:list=None, output_is_complete:bool=True, requires:list=None,
   121              container:bool|dict=False, sandbox:bool=None, size:str=''):
   122      """A rule which creates a test with an arbitrary command.
   123  
   124      The command must return zero on success and nonzero on failure. Test results are written
   125      to test.results (or not if no_test_output is True).
   126      Most arguments are similar to genrule() so we cover them in less detail here.
   127  
   128      Args:
   129        name (str): Name of the rule
   130        test_cmd (str | dict): Command to run for the test. It works similarly to the cmd attribute;
   131                               see genrule for a more detailed discussion of its properties.
   132                               The command should exit with 0 when successful, and nonzero otherwise.
   133                               Normally this will correspond to the test results that are output,
   134                               unless no_test_output is True in which case this is the only thing
   135                               that determines success / failure.
   136        labels (list): Labels to apply to this test.
   137        cmd (str | dict): Command to run to build the test.
   138        srcs (list | dict): Source files for this rule.
   139        outs (list): Output files of this rule.
   140        deps (list): Dependencies of this rule.
   141        tools (list): Tools used to build this rule; similar to srcs but are not copied to the temporary
   142                      build directory. Should be accessed via $(exe //path/to:tool) or similar.
   143        secrets (list): Secrets available to this rule while building.
   144        data (list): Runtime data files for the test.
   145        visibility (list): Visibility declaration of this rule.
   146        timeout (int): Length of time in seconds to allow the test to run for before killing it.
   147        needs_transitive_deps (bool): True if building the rule requires all transitive dependencies to
   148                               be made available.
   149        flaky (bool | int): If true the test will be marked as flaky and automatically retried.
   150        no_test_output (bool): If true the test is not expected to write any output results, it will
   151                               pass if the command exits with 0 and fail otherwise.
   152        test_outputs (list): List of optional additional test outputs.
   153        output_is_complete (bool): If this is true then the rule blocks downwards searches of transitive
   154                            dependencies by other rules.
   155        requires (list): Kinds of output from other rules that this one requires.
   156        container (bool | dict): If true the test is run in a container (eg. Docker).
   157        sandbox (bool): If True, the test is run within a sandbox that restricts some cgroups
   158                        including networking, process, IPC, etc. Only has an effect on Linux.
   159                        If this is on by default then tests can opt out by setting this to False.
   160        size (str): Test size (enormous, large, medium or small).
   161      """
   162      timeout, labels = _test_size_and_timeout(size, timeout, labels)
   163      return build_rule(
   164          name = name,
   165          srcs = srcs,
   166          outs = outs,
   167          deps = deps,
   168          data = data,
   169          tools = tools,
   170          secrets = secrets,
   171          test_cmd  =  test_cmd,
   172          cmd = cmd,
   173          visibility = visibility,
   174          output_is_complete = output_is_complete,
   175          labels = labels,
   176          binary = True,
   177          test = True,
   178          test_timeout = timeout,
   179          needs_transitive_deps = needs_transitive_deps,
   180          requires = requires,
   181          container = container,
   182          test_sandbox = sandbox,
   183          no_test_output = no_test_output,
   184          test_outputs = test_outputs,
   185          flaky = flaky,
   186      )
   187  
   188  
   189  def export_file(name:str, src:str, visibility:list=None, binary:bool=False, test_only:bool&testonly=False):
   190      """Essentially a single-file alias for filegroup.
   191  
   192      Args:
   193        name (str): Name of the rule
   194        src (str): Source file for the rule
   195        visibility (list): Visibility declaration
   196        binary (bool): True to mark the rule outputs as binary
   197        test_only (bool): If true the exported file can only be used by test targets.
   198      """
   199      filegroup(
   200          name = name,
   201          srcs = [src],
   202          visibility = visibility,
   203          binary = binary,
   204          test_only = test_only,
   205      )
   206  
   207  
   208  def filegroup(name:str, tag:str='', srcs:list=None, deps:list=None, exported_deps:list=None,
   209                visibility:list=None, labels:list&features&tags=None, binary:bool=False, output_is_complete:bool=True,
   210                requires:list=None, provides:dict=None, hashes:list=None, test_only:bool&testonly=False):
   211      """Defines a collection of files which other rules can depend on.
   212  
   213      Sources can be omitted entirely in which case it acts simply as a rule to collect other rules,
   214      which is often more handy than you might think.
   215  
   216      Args:
   217        name (str): Name of the rule
   218        tag (str): Tag applied to name; generates a private rule, for example name='a',tag='b' gives
   219                   _a#b. Typically used for "private" rules.
   220        srcs (list): Source files for the rule.
   221        deps (list): Dependencies of the rule.
   222        exported_deps (list): Dependencies that will become visible to any rules that depend on this rule.
   223        visibility (list): Visibility declaration
   224        labels (list): Labels to apply to this rule
   225        binary (bool): True to mark the rule outputs as binary
   226        output_is_complete (bool): If this is true then the rule blocks downwards searches of transitive
   227                                   dependencies by other rules.
   228        requires (list): Kinds of output from other rules that this one requires.
   229        provides (dict): Kinds of output that this provides for other rules (see genrule() for a more
   230                         in-depth discussion of this).
   231        hashes (list): List of acceptable output hashes for this rule.
   232        test_only (bool): If true the exported file can only be used by test targets.
   233      """
   234      return build_rule(
   235          name=name,
   236          tag=tag,
   237          cmd='',
   238          srcs=srcs,
   239          deps=deps,
   240          exported_deps=exported_deps,
   241          visibility=visibility,
   242          building_description='Copying...',
   243          output_is_complete=output_is_complete,
   244          requires=requires,
   245          provides=provides,
   246          test_only=test_only,
   247          labels=labels,
   248          binary=binary,
   249          hashes=hashes,
   250          _filegroup=True,
   251      )
   252  
   253  
   254  def hash_filegroup(name:str, srcs:list=None, deps:list=None, exported_deps:list=None, visibility:list=None,
   255                     labels:list&features&tags=None, test_only:bool&testonly=False, requires:list=None):
   256      """Copies a set of files to output names which are uniquely hashed based on their contents.
   257  
   258      For example, srcs = ["test.txt"] might output "test-b250cnf30f3h.txt".
   259  
   260      Args:
   261        name (str): Name of the rule.
   262        srcs (list): Source files for the rule.
   263        deps (list): Dependencies of the rule.
   264        exported_deps (list): Dependencies that will become visible to any rules that depend on this rule.
   265        visibility (list): Visibility declaration
   266        labels (list): Labels to apply to this rule
   267        test_only (bool): If true the exported file can only be used by test targets.
   268        requires (list): Kinds of output from other rules that this one requires.
   269      """
   270      return build_rule(
   271          name=name,
   272          srcs=srcs,
   273          cmd='',
   274          deps=deps,
   275          exported_deps=exported_deps,
   276          visibility=visibility,
   277          building_description='Copying...',
   278          output_is_complete=True,
   279          test_only=test_only,
   280          labels=labels,
   281          requires=requires,
   282          _hash_filegroup=True,
   283      )
   284  
   285  
   286  def system_library(name:str, srcs:list, deps:list=None, hashes:list=None,
   287                     visibility:list=None, test_only:bool&testonly=False):
   288      """Defines a rule to collect some dependencies from outside the build tree.
   289  
   290      This is essentially the same as a filegroup; it will simply copy files from the system
   291      into the build tree, you must add additional rules if compilation is necessary.
   292  
   293      Args:
   294        name (str): Name of the rule.
   295        srcs (list): System-level sources. Should all be absolute paths.
   296        deps (list): Dependencies of the rule.
   297        hashes (list): List of hashes; the output must match at least one of these. This is not required
   298                       but could be used to assert that the system lib is of some known version.
   299        visibility (list): Visibility declaration of the rule.
   300        test_only (bool): If true the rule is only visible to test targets.
   301      """
   302      build_rule(
   303          name = name,
   304          system_srcs = srcs,
   305          outs = [basename(src) for src in srcs],
   306          deps = deps,
   307          cmd = 'cp $SRCS .',
   308          hashes = hashes,
   309          visibility = visibility,
   310          test_only = test_only,
   311          sandbox = False,
   312      )
   313  
   314  
   315  def remote_file(name:str, url:str|list, hashes:list=None, out:str=None, binary:bool=False,
   316                  visibility:list=None, licences:list=None, test_only:bool&testonly=False,
   317                  labels:list=[], deps:list=None, exported_deps:list=None, _tag=''):
   318      """Defines a rule to fetch a file over HTTP(S).
   319  
   320      Args:
   321        name (str): Name of the rule
   322        url (str | list): URL or URLs to fetch. If multiple are passed then they will be tried
   323                          in sequence until one succeeds.
   324        hashes (list): List of hashes; the output must match at least one of these.
   325        out (str): Output name of the file. Chosen automatically if not given.
   326        binary (bool): True to mark the output as binary and runnable.
   327        visibility (list): Visibility declaration of the rule.
   328        licences (list): List of licences that apply to this rule.
   329        test_only (bool): If true the rule is only visible to test targets.
   330        labels (list): Labels to apply to this rule.
   331        deps (list): List of extra dependencies for this rule.
   332        exported_deps (list): Dependencies that will become visible to any rules that depend on this rule.
   333      """
   334      urls = [url] if isinstance(url, str) else url
   335      url = urls[0]
   336      return build_rule(
   337          name = name,
   338          tag = _tag,
   339          cmd = '',
   340          _urls = urls,
   341          outs = [out or url[url.rfind('/') + 1:]],
   342          binary = binary,
   343          visibility = visibility,
   344          hashes = hashes,
   345          licences = licences,
   346          building_description = 'Fetching...',
   347          deps = deps,
   348          exported_deps = exported_deps,
   349          test_only = test_only,
   350          labels = labels,
   351          sandbox = False,
   352      )
   353  
   354  
   355  def tarball(name:str, srcs:list, out:str=None, deps:list=None, subdir:str=None, gzip:bool=True,
   356              xzip:bool=False, test_only:bool=False, visibility:list=None, labels:list&features&tags=[]):
   357      """Defines a rule to create a tarball containing outputs of other rules.
   358  
   359      File mode and ownership are preserved. However, the atime and mtime of all
   360      files will be set to 1 Jan 2000 00:00:00.
   361  
   362      Args:
   363        name (str): Rule name
   364        srcs (list): Source files to include in the tarball
   365        out (str): Name of output tarball (defaults to `name`.tar.gz, but see below re compression)
   366        subdir (str): Subdirectory to create in. All files will be flattened into this directory.
   367        gzip (bool): If True, the output will be gzipped. If False, it will just be a tarball.
   368        xzip (bool): If True, the output will be xzipped. Overrides gzip if passed.
   369        deps (list): Dependencies
   370        visibility (list): Visibility specification.
   371        labels (list): Labels associated with this rule.
   372      """
   373      tar_out = out or (name + ('.tar.gz' if gzip else '.tar'))
   374      cmd = '$TOOL tar '
   375      tar_name = name
   376      if xzip:
   377          tar_out = name + '.tar'
   378          tar_name = f'_{name}#tar'
   379      elif gzip:
   380          cmd += ' -z'
   381      if subdir:
   382          cmd += ' --prefix ' + subdir
   383  
   384      tar_rule = build_rule(
   385          name = tar_name,
   386          cmd = cmd,
   387          srcs = srcs,
   388          tools = [CONFIG.JARCAT_TOOL],
   389          outs = [tar_out],
   390          deps = deps,
   391          visibility = visibility,
   392          labels = labels + ['tar'],
   393          output_is_complete = True,
   394          test_only = test_only,
   395      )
   396      if not xzip:
   397          return tar_rule
   398      # The golang xzip package produces significantly (20%) larger output than filtering
   399      # through xz. At some point we will ditch that when they become closer, but for now
   400      # that's a bit hard to swallow so we'll use the command-line version.
   401      return build_rule(
   402          name = name,
   403          srcs = [tar_rule],
   404          outs = [out or (name + '.tar.xz')],
   405          cmd = 'xz -zc $SRCS > $OUT',
   406          visibility = visibility,
   407          labels = labels + ['tar'],
   408          output_is_complete = True,
   409          test_only = test_only,
   410      )
   411  
   412  
   413  def decompose(label:str):
   414      """Decomposes a build label into the package and name parts.
   415  
   416      Consider carefully when you should use this - command replacements may be more appropriate.
   417      Most rules should be able to accept labels without having to know anything about their structure.
   418  
   419      Args:
   420        label (str): A build label in either relative or absolute form.
   421      Raises:
   422        ValueError: if the given label is not a correctly formatted build label.
   423      """
   424      if label.startswith(':'):
   425          return get_base_path(), label.lstrip(':')
   426      elif label.startswith('//'):
   427          package, _, name = label.partition(':')
   428          return package.lstrip('/'), name
   429      else:
   430          raise ValueError(f"{label} doesn't look like a build label")
   431  
   432  
   433  def check_config(key:str, section:str='buildconfig', rule:str='', example:str='...'):
   434      """Checks the configuration for the given item and raises an exception if it's not set.
   435  
   436      Args:
   437        key (str): Config key that must be set, e.g. ANDROID_HOME
   438        section (str): Section of the configuration that it must be set in.
   439                       Defaults to buildconfig since most external rules will have to use that.
   440        rule (str): Kind of rule that will be using this, e.g. "Android". Affects the output message.
   441        example (str): Example that they might consider setting.
   442      """
   443      val = CONFIG.get(key)
   444      if not val:
   445          key = key.lower().replace('_', '-')
   446          rule_msg = f' to use {rule} rules' if rule else ''
   447          msg = f'You must set {section}.{key} in your .plzconfig{rule_msg}'
   448          msg = f'{msg}, e.g.\n[{section}]\n{key} = {example}\n'
   449          raise ConfigError(msg)
   450      return val
   451  
   452  
   453  def _test_size_and_timeout(size, timeout, labels:list=[]):
   454      """Resolves size and timeout arguments for a test."""
   455      if size:
   456          labels += [size]
   457          if not timeout:
   458              timeout = _SIZE_TIMEOUTS.get(size, 0)
   459      if isinstance(timeout, str):
   460          timeout = _TIMEOUT_NAMES[timeout]
   461      return timeout, labels
   462  
   463  
   464  _SIZE_TIMEOUTS = {
   465      'enormous': 3600,
   466      'large': 900,
   467      'medium': 300,
   468      'small': 60,
   469  }
   470  
   471  _TIMEOUT_NAMES = {
   472      'eternal': 0,  # means unlimited
   473      'long': 900,
   474      'moderate': 300,
   475      'short': 60,
   476  }
   477  
   478  
   479  if CONFIG.BAZEL_COMPATIBILITY:
   480      def bind(name, actual=None):
   481          """Mimics the Bazel bind() function which binds some target or sub-target into our repo.
   482  
   483          TODO(peterebden): Remove, it seems to be deprecated in Bazel so may not be much value having it here.
   484          """
   485          if not actual:
   486              return
   487          if actual.startswith('@') and actual.endswith('//jar'):
   488              actual = ':' + actual[:-len('//jar')].lstrip('@')
   489          filegroup(
   490              name = name,
   491              srcs = [actual],
   492              visibility = ['PUBLIC'],
   493          )
   494  
   495      def exports_files(srcs, name='exported_files', visibility=['PUBLIC'], licenses=None):
   496          """Makes a bunch of files available as a filegroup.
   497  
   498          Note that the semantics are not the same as Bazel; it lifts the restrictions on those
   499          files being available to other packages, whereas we're just creating a rule. It's not
   500          impossible to extend Please to support that but right now we're not bothering.
   501          """
   502          filegroup(
   503              name = name,
   504              srcs = srcs,
   505              visibility = visibility,
   506          )