github.phpd.cn/thought-machine/please@v12.2.0+incompatible/src/parse/rules/python_rules.build_defs

""" Rules to build Python code.

The output artifacts for Python rules are .pex files (see https://github.com/pantsbuild/pex).
Pex is a rather nice system for combining Python code and all needed dependencies
(excluding the actual interpreter and possibly some system level bits) into a single file.

The process of compiling pex files can be a little slow when including many large files, as
often happens when one's binary includes large compiled dependencies (eg. numpy...). Hence
we have a fairly elaborate optimisation whereby each python_library rule builds a little
zipfile containing just its sources, and all of those are combined at the end to produce
the final .pex. This builds at roughly the same pace for a clean build of a single target,
but is drastically faster for building many targets with similar dependencies or rebuilding
a target which has only had small changes.
"""


def python_library(name:str, srcs:list=[], resources:list=[], deps:list=[], visibility:list=None,
                   test_only:bool&testonly=False, zip_safe:bool=True, labels:list&features&tags=[], interpreter:str=None,
                   strip:bool=False):
    """Generates a Python library target, which collects Python files for use by dependent rules.

    Note that each python_library performs some pre-zipping of its inputs before they're combined
    in a python_binary or python_test. Hence while it's of course not required that all dependencies
    of those rules are python_library rules, it's often a good idea to wrap any large dependencies
    in one to improve incrementality (not necessary for pip_library, of course).

    Args:
      name (str): Name of the rule.
      srcs (list): Python source files for this rule.
      resources (list): Non-Python files that this rule collects which will be included in the final .pex.
                        The distinction between this and srcs is fairly arbitrary and historical, but
                        semantically quite nice and parallels python_test.
      deps (list): Dependencies of this rule.
      visibility (list): Visibility specification.
      test_only (bool): If True, can only be depended on by tests.
      zip_safe (bool): Should be set to False if this library can't be safely run inside a .pex
                       (the most obvious reason not is when it contains .so modules).
                       See python_binary for more information.
      labels (list): Labels to apply to this rule.
      interpreter (str): The Python interpreter to use. Defaults to the config setting
                         which is normally just 'python', but could be 'python3' or
                         'pypy' or whatever.
      strip (bool): If True, the original sources are stripped and only bytecode is output.
    """
    if not zip_safe:
        labels.append('py:zip-unsafe')
    if srcs or resources:
        cmd = '$TOOLS_JARCAT z -d -o ${OUTS} -i .'
        interpreter = interpreter or CONFIG.DEFAULT_PYTHON_INTERPRETER
        if srcs:
            # This is a bit of a hack, but rather annoying. We want to put bytecode in its 'legacy' location
            # in python3 because zipimport doesn't look in __pycache__. Unfortunately the flag doesn't exist
            # in python2 so we have to guess whether we should apply it or not.
            bytecode_flag = '-b' if 'python3' in interpreter or 'pypy3' in interpreter else ''
            compile_cmd = '$TOOLS_INT -S -m compileall %s -f $SRCS_SRCS' % bytecode_flag
            if strip:
                cmd = ' && '.join([compile_cmd, 'rm -f $SRCS_SRCS', cmd])
            else:
                cmd = ' && '.join([compile_cmd, cmd])
        # Pre-zip the files for later collection by python_binary.
        zip_rule = build_rule(
            name=name,
            tag='zip',
            srcs={
                'SRCS': srcs,
                'RES': resources,
            },
            outs=['.%s.pex.zip' % name],
            cmd=cmd,
            building_description='Compressing...',
            requires=['py'],
            test_only=test_only,
            output_is_complete=True,
            tools={
                'int': [interpreter],
                'jarcat': [CONFIG.JARCAT_TOOL],
            },
        )
        deps.append(zip_rule)
    elif strip:
        raise ParseError("Can't pass strip=True to a python_library with no srcs")

    return filegroup(
        name=name,
        srcs=resources if strip else (srcs + resources),
        deps=deps,
        visibility=visibility,
        output_is_complete=False,
        requires=['py'],
        test_only=test_only,
        labels=labels,
    )


def python_binary(name:str, main:str, resources:list=[], out:str=None, deps:list=[],
                  visibility:list=None, zip_safe:bool=None, interpreter:str=None, shebang:str='',
                  labels:list&features&tags=[]):
    """Generates a Python binary target.

    This compiles all source files together into a single .pex file which can
    be easily copied or deployed. The construction of the .pex is done in parts
    by the dependent python_library rules, and this rule simply builds the
    metadata for it and concatenates them all together.

    Args:
      name (str): Name of the rule.
      main (str): Python file which is the entry point and __main__ module.
      resources (list): List of static resources to include in the .pex.
      out (str): Name of the output file. Default to name + .pex
      deps (list): Dependencies of this rule.
      visibility (list): Visibility specification.
      zip_safe (bool): Allows overriding whether the output is marked zip safe or not.
                       If set to explicitly True or False, the output will be marked
                       appropriately; by default it will be safe unless any of the
                       transitive dependencies are themselves marked as not zip-safe.
      interpreter (str): The Python interpreter to use. Defaults to the config setting
                         which is normally just 'python', but could be 'python3' or
                         'pypy' or whatever.
      shebang (str): Exact shebang to apply to the generated file. By default we will
                     determine something appropriate for the given interpreter.
      labels (list): Labels to apply to this rule.
    """
    shebang = shebang or interpreter or CONFIG.DEFAULT_PYTHON_INTERPRETER
    cmd = '$TOOLS_PEX -s "%s" -m "%s" --zip_safe' % (shebang, CONFIG.PYTHON_MODULE_DIR)
    pre_build, cmd = _handle_zip_safe(cmd, zip_safe)

    lib_rule = python_library(
        name='_%s#lib' % name,
        srcs=[main],
        resources=resources,
        interpreter=interpreter,
        deps=deps,
        visibility=visibility,
    )

    # Use the pex tool to compress the entry point & add all the bootstrap helpers etc.
    pex_rule = build_rule(
        name = name,
        tag = 'pex',
        srcs=[main],
        outs=['.%s_main.pex.zip' % name],  # just call it .zip so everything has the same extension
        cmd=cmd,
        requires=['py', 'pex'],
        pre_build=pre_build,
        deps=deps,
        needs_transitive_deps=True,  # Needed so we can find anything with zip_safe=False on it.
        output_is_complete=True,
        tools={
            'interpreter': [interpreter or CONFIG.DEFAULT_PYTHON_INTERPRETER],
            'pex': [CONFIG.PEX_TOOL],
        },
    )
    # This rule concatenates the .pex with all the other precompiled zip files from dependent rules.
    build_rule(
        name=name,
        srcs=[pex_rule],
        deps=[lib_rule],
        outs=[out or (name + '.pex')],
        cmd=_PYTHON_BINARY_CMDS,
        needs_transitive_deps=True,
        binary=True,
        output_is_complete=True,
        building_description="Creating pex...",
        visibility=visibility,
        requires=['py', interpreter or CONFIG.DEFAULT_PYTHON_INTERPRETER],
        tools=[CONFIG.JARCAT_TOOL],
        # This makes the python_library rule the dependency for other python_library or
        # python_test rules that try to import it. Does mean that they cannot collect a .pex
        # by depending directly on the rule, they'll just get the Python files instead.
        # This is not a common case anyway; more usually you'd treat that as a runtime data
        # file rather than trying to pack into a pex. Can be worked around with an
        # intermediary filegroup rule if really needed.
        provides={'py': lib_rule},
        labels=labels,
    )


def python_test(name:str, srcs:list, data:list=[], resources:list=[], deps:list=[],
                labels:list&features&tags=None, size:str=None, flags:str='', visibility:list=None,
                container:bool|dict=False, sandbox:bool=None, timeout:int=0, flaky:bool|int=0,
                test_outputs:list=None, zip_safe:bool=None, interpreter:str=None):
    """Generates a Python test target.

    This works very similarly to python_binary; it is also a single .pex file
    which is run to execute the tests. The tests are run via either unittest or pytest, depending
    on which is set for the test runner, which can be configured either via the python_test_runner
    package property or python.testrunner in the config.

    Args:
      name (str): Name of the rule.
      srcs (list): Source files for this test.
      data (list): Runtime data files for the test.
      resources (list): Non-Python files to be included in the pex. Note that the distinction
                        vs. srcs is important here; srcs are passed to unittest for it to run
                        and it may or may not be happy if given non-Python files.
      deps (list): Dependencies of this rule.
      labels (list): Labels for this rule.
      size (str): Test size (enormous, large, medium or small).
      flags (str): Flags to apply to the test command.
      visibility (list): Visibility specification.
      container (bool | dict): If True, the test will be run in a container (eg. Docker).
      sandbox (bool): Sandbox the test on Linux to restrict access to namespaces such as network.
      timeout (int): Maximum time this test is allowed to run for, in seconds.
      flaky (int | bool): True to mark this test as flaky, or an integer for a number of reruns.
      test_outputs (list): Extra test output files to generate from this test.
      zip_safe (bool): Allows overriding whether the output is marked zip safe or not.
                       If set to explicitly True or False, the output will be marked
                       appropriately; by default it will be safe unless any of the
                       transitive dependencies are themselves marked as not zip-safe.
      interpreter (str): The Python interpreter to use. Defaults to the config setting
                         which is normally just 'python', but could be 'python3' or
                        'pypy' or whatever.
    """
    timeout, labels = _test_size_and_timeout(size, timeout, labels)
    interpreter = interpreter or CONFIG.DEFAULT_PYTHON_INTERPRETER
    cmd = '$TOOLS_PEX -t -s "%s" -m "%s" -r "%s" --zip_safe' % (interpreter, CONFIG.PYTHON_MODULE_DIR, CONFIG.PYTHON_TEST_RUNNER)
    pre_build, cmd = _handle_zip_safe(cmd, zip_safe)

    # Use the pex tool to compress the entry point & add all the bootstrap helpers etc.
    pex_rule = build_rule(
        name = name,
        tag = 'pex',
        srcs=srcs,
        outs=['.%s_main.pex.zip' % name],  # just call it .zip so everything has the same extension
        cmd=cmd,
        requires=['py'],
        test_only=True,
        needs_transitive_deps=True,  # needed for zip-safe detection
        building_description="Creating pex info...",
        pre_build=pre_build,
        deps=deps,
        tools={
            'interpreter': [interpreter or CONFIG.DEFAULT_PYTHON_INTERPRETER],
            'pex': [CONFIG.PEX_TOOL],
        },
    )

    # If there are resources specified, they have to get built into the pex.
    deps = [pex_rule]
    if resources:
        # Test library itself.
        lib_rule = python_library(
            name='_%s#lib' % name,
            resources=resources,
            interpreter=interpreter,
            deps=deps,
            test_only=True,
        )
        deps = [pex_rule, lib_rule]

    # This rule concatenates the .pex with all the other precompiled zip files from dependent rules.
    build_rule(
        name=name,
        srcs=[pex_rule],
        deps=deps,
        # N.B. the actual test sources are passed as data files as well. This is needed for pytest but
        #      is faster for unittest as well (because we don't need to rebuild the pex if they change).
        data=data + srcs,
        outs=['%s.pex' % name],
        labels=labels,
        cmd=_PYTHON_BINARY_CMDS,
        test_cmd = '$TEST ' + flags,
        needs_transitive_deps=True,
        output_is_complete=True,
        binary=True,
        test=True,
        container=container,
        test_sandbox=sandbox,
        building_description="Building pex...",
        visibility=visibility,
        test_timeout=timeout,
        flaky=flaky,
        test_outputs=test_outputs,
        requires=['py', interpreter or CONFIG.DEFAULT_PYTHON_INTERPRETER],
        tools=[CONFIG.JARCAT_TOOL],
    )


def pip_library(name:str, version:str, hashes:list=None, package_name:str=None, outs:list=None,
                test_only:bool&testonly=False, env:dict=None, deps:list=[], post_install_commands:list=None,
                install_subdirectory:bool=False, repo:str=None, use_pypi:bool=None, patch:str|list=None,
                visibility:list=None, zip_safe:bool=True, licences:list=None, pip_flags:str=None):
    """Provides a build rule for third-party dependencies to be installed by pip.

    Args:
      name (str): Name of the build rule.
      version (str): Specific version of the package to install.
      hashes (list): List of acceptable hashes for this target.
      package_name (str): Name of the pip package to install. Defaults to the same as 'name'.
      outs (list): List of output files / directories. Defaults to [name].
      test_only (bool): If True, can only be used by test rules or other test_only libraries.
      env (dict): Deprecated, has no effect.
      deps (list): List of rules this library depends on.
      post_install_commands (list): Commands run after pip install has completed.
      install_subdirectory (bool): Forces the package to install into a subdirectory with this name.
      repo (str): Allows specifying a custom repo to fetch from.
      use_pypi (bool): If True, will check PyPI as well for packages.
      patch (str | list): A patch file or files to be applied after install.
      visibility (list): Visibility declaration for this rule.
      zip_safe (bool): Flag to indicate whether a pex including this rule will be zip-safe.
      licences (list): Licences this rule is subject to. Default attempts to detect from package metadata.
      pip_flags (str): List of additional flags to pass to pip.
    """
    package_name = '%s==%s' % (package_name or name, version)
    outs = outs or [name]
    post_install_commands = post_install_commands or []
    post_build = None
    use_pypi = CONFIG.USE_PYPI if use_pypi is None else use_pypi
    index_flag = '' if use_pypi else '--no-index'
    pip_flags = pip_flags or CONFIG.PIP_FLAGS

    repo_flag = ''
    repo = repo or CONFIG.PYTHON_DEFAULT_PIP_REPO
    if repo:
        if repo.startswith('//') or repo.startswith(':'):  # Looks like a build label, not a URL.
            repo_flag = '-f %(location %s)' % repo
            deps.append(repo)
        else:
            repo_flag = '-f ' + repo

    target = outs[0] if install_subdirectory else '.'

    cmd = '$TOOLS_PIP install --no-deps --no-compile --no-cache-dir --default-timeout=60 --target=' + target
    if CONFIG.OS == 'linux':
        # Fix for https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=830892
        # tl;dr: Debian has broken --target with a custom patch, the only way to fix is to pass --system
        # which is itself Debian-specific, so we need to find if we're running on Debian. AAAAARGGGHHHH...
        cmd = '[ -f /etc/debian_version ] && [ $TOOLS_PIP == "/usr/bin/pip3" ] && SYS_FLAG="--system" || SYS_FLAG=""; ' + cmd + ' $SYS_FLAG'
    elif CONFIG.OS == 'darwin':
        # Fix for Homebrew which fails with a superficially similar issue.
        # https://github.com/Homebrew/brew/blob/master/docs/Homebrew-and-Python.md suggests fixing with --install-option
        # but that prevents downloading binary wheels. This is more fiddly but seems to work.
        # Unfortunately it does *not* work similarly on the Debian problem :(
        cmd = 'echo "[install]\nprefix=" > setup.cfg; ' + cmd
    cmd += ' -q -b build %s %s %s %s' % (repo_flag, index_flag, pip_flags, package_name)
    cmd += ' && find . -name "*.pyc" -or -name "tests" | xargs rm -rf'

    if not licences:
        cmd += ' && find . -name METADATA -or -name PKG-INFO | grep -v "^./build/" | xargs grep -E "License ?:" | grep -v UNKNOWN | cat'

    if install_subdirectory:
        cmd += ' && touch %s/__init__.py && rm -rf %s/*.egg-info %s/*.dist-info' % (target, target, target)

    if patch:
        patches = [patch] if isinstance(patch, str) else patch
        if CONFIG.OS == 'freebsd':
            # --no-backup-if-mismatch is not supported, but we need to get rid of the .orig
            # files for hashes to match correctly.
            cmd += ' && ' + ' && '.join(['patch -p0 < $(location %s)' % patch for patch in patches])
            cmd += ' && find . -name "*.orig" | xargs rm'
        else:
            cmd += ' && ' + ' && '.join(['patch -p0 --no-backup-if-mismatch < $(location %s)' % patch for patch in patches])

    if post_install_commands:
        cmd = ' && '.join([cmd] + post_install_commands)

    # TODO(peterebden): --prefix preserves the old behaviour here, but I'm not sure how intuitive that is; leaving it
    #                   out puts everything at the top level of the pex, which is more normal for Python really.
    cmd += ' && $TOOLS_JARCAT z -d --prefix $PKG_DIR -i ' + ' -i '.join(outs)

    wheel_rule = build_rule(
        name = name,
        tag = 'wheel',
        cmd = cmd,
        outs = [name + '.whl'],
        srcs = patches if patch else [],
        deps = deps,
        building_description = 'Fetching...',
        hashes = hashes,
        requires = ['py'],
        test_only = test_only,
        licences = licences,
        tools = {
            'pip': [CONFIG.PIP_TOOL],
            'jarcat': [CONFIG.JARCAT_TOOL],
        },
        post_build = None if licences else _add_licences,
        sandbox = False,
        labels = ['py:zip-unsafe'] if not zip_safe else None,
    )
    return build_rule(
        name = name,
        srcs = [wheel_rule],
        outs = outs,
        cmd = '$TOOL x $SRCS -s $PKG_DIR',
        tools = [CONFIG.JARCAT_TOOL],
        labels = ['py', 'pip:' + package_name],
        provides = {'py': wheel_rule},
        visibility = visibility,
        test_only = test_only,
        deps = deps,
    )


def python_wheel(name:str, version:str, hashes:list=None, package_name:str=None, outs:list=None,
                 post_install_commands:list=None, patch:str|list=None, licences:list=None,
                 test_only:bool&testonly=False, repo:str=None, zip_safe:bool=True, visibility:list=None,
                 deps:list=[], name_scheme:str=None):
    """Downloads a Python wheel and extracts it.

    This is a lightweight pip-free alternative to pip_library which supports cross-compiling.
    Rather than leaning on pip which is difficult to achieve reproducible builds with and
    support on different platforms, this rule is a simple wrapper around curl and unzip.
    Unless otherwise specified, the wheels are expected to adhere to common naming schemes,
    such as:
      <package_name>-<version>[-<os>-<arch>].whl
      <package_name>-<version>[-<os>_<arch>].whl
      <package_name>-<version>.whl

    Args:
      name (str): Name of the rule. Also doubles as the name of the package if package_name
            is not set.
      version (str): Version of the package to install.
      hashes (list): List of hashes to verify the package against.
      package_name (str): If given, overrides `name` for the name of the package to look for.
      outs (list): List of output files. Defaults to a directory named the same as `name`.
      post_install_commands (list): Commands to run after 'install'.
      patch (str | list): Patch file to apply after install to fix any upstream code that has
                          done bad things.
      licences (list): Licences that this rule is subject to.
      test_only (bool): If True, this library can only be used by tests.
      repo (str): Repository to download wheels from.
      zip_safe (bool): Flag to indicate whether a pex including this rule will be zip-safe.
      visibility (list): Visibility declaration.
      deps (list): Dependencies of this rule.
      name_scheme (str): The templatized wheel naming scheme (available template variables
                         are `url_base`, `package_name`, and `version`).
    """
    package_name = package_name or name.replace('-', '_')
    url_base = repo or CONFIG.PYTHON_WHEEL_REPO
    if not url_base:
        raise ParseError('python.wheel_repo is not set in the config, must pass repo explicitly '
                         'to python_wheel')
    urls = []
    if name_scheme:
        urls.append(name_scheme.format(url_base=url_base,
                                       package_name=package_name,
                                       version=version))
    elif CONFIG.PYTHON_WHEEL_NAME_SCHEME:
        urls.append(CONFIG.PYTHON_WHEEL_NAME_SCHEME.format(url_base=url_base,
                                                           package_name=package_name,
                                                           version=version))
    else:
        # Populate urls using a reasonable set of possible wheel naming schemes.
        # Look for an arch-specific wheel first; in some cases there can be both (e.g. protobuf
        # has optional arch-specific bits) and we prefer the one with the cool stuff.
        urls.append('{url_base}/{package_name}-{version}-${{OS}}-${{ARCH}}.whl'.format(url_base=url_base,
                                                                                       package_name=package_name,
                                                                                       version=version))
        urls.append('{url_base}/{package_name}-{version}-${{OS}}_${{ARCH}}.whl'.format(url_base=url_base,
                                                                                       package_name=package_name,
                                                                                       version=version))
        urls.append('{url_base}/{package_name}-{version}.whl'.format(url_base=url_base,
                                                                     package_name=package_name,
                                                                     version=version))

    curl_commands = ' || '.join(['curl -fsSO {url}'.format(url=url) for url in urls])
    cmd = [
        curl_commands,
        '$TOOL x *.whl',
        # Strip any bytecode, it might lead to nondeterminism. Similarly some wheels annoyingly
        # contain test code that we don't want.
        'find . -name "*.pyc" -or -name "tests" | xargs rm -rf',
    ]
    if not licences:
        cmd.append('find . -name METADATA -or -name PKG-INFO | grep -v "^./build/" | '
                   'xargs grep -E "License ?:" | grep -v UNKNOWN | cat')
    if patch:
        patches = [patch] if isinstance(patch, str) else patch
        cmd += ['patch -p0 --no-backup-if-mismatch < $(location %s)' % p for p in patches]
    if post_install_commands:
        cmd += post_install_commands
    cmd += ['$TOOL z -d --prefix $PKG -i ' + ' -i '.join(outs or [name])]

    wheel_rule = build_rule(
        name = name,
        tag = 'wheel',
        cmd = ' && '.join(cmd),
        outs = [name + '.whl'],
        srcs = patches if patch else None,
        building_description = 'Downloading...',
        requires = ['py'],
        deps = deps,
        test_only = test_only,
        licences = licences,
        tools = [CONFIG.JARCAT_TOOL],
        post_build = None if licences else _add_licences,
        sandbox = False,
        labels = ['py:zip-unsafe'] if not zip_safe else None,
    )
    cmd = '$TOOL x $SRCS -s $PKG_DIR'
    if outs:
        # Hacky solution to handle things being in subdirectories in awkward ways.
        before, _, after = outs[0].partition('/')
        if after:
            cmd = 'rm -rf %s && %s' % (before, cmd)
    return build_rule(
        name = name,
        srcs = [wheel_rule],
        hashes = hashes,  # TODO(peterebden): Move this onto wheel_rule when we're willing to break hash compatibility.
        outs = outs or [name],
        tools = [CONFIG.JARCAT_TOOL],
        cmd = cmd,
        deps = deps,
        visibility = visibility,
        test_only = test_only,
        labels = ['whl:%s==%s' % (package_name, version)],
        provides = {'py': wheel_rule},
    )


def _handle_zip_safe(cmd, zip_safe):
    """Handles the zip safe flag. Returns a tuple of (pre-build function, new command)."""
    if zip_safe is None:
        return lambda name: (set_command(name, cmd.replace('--zip_safe', ' --nozip_safe'))
                             if has_label(name, 'py:zip-unsafe') else None), cmd
    elif zip_safe:
        return None, cmd
    else:
        return None, cmd.replace('--zip_safe', ' --nozip_safe')


def _add_licences(name, output):
    """Annotates a pip_library rule with detected licences after download."""
    for line in output:
        if line.startswith('License: '):
            for licence in line[9:].split(' or '):  # Some are defined this way (eg. "PSF or ZPL")
                add_licence(name, licence)
            return
        elif line.startswith('Classifier: License'):
            # Oddly quite a few packages seem to have UNKNOWN for the licence but this Classifier
            # section still seems to know what they are licenced as.
            add_licence(name, line.split(' :: ')[-1])
            return
    log.warning('No licence found for %s, should add licences = [...] to the rule',
                name.lstrip('_').split('#')[0])


# The commands that we use for python_binary and python_test rules.
_PYTHON_BINARY_CMDS = {
    'opt': '$TOOL z -i . -o $OUTS -s .pex.zip -s .whl --preamble_from="$SRC" --include_other --add_init_py --strict',
    'stripped': '$TOOL z -i . -o $OUTS -s .pex.zip -s .whl --preamble_from="$SRC" --include_other --add_init_py --strict -e .py -x "*.py"',
}


if CONFIG.BAZEL_COMPATIBILITY:
    py_library = python_library
    py_binary = python_binary
    py_test = python_test