github.com/apache/beam/sdks/v2@v2.48.2/python/container/run_generate_requirements.sh (about)

     1  #!/bin/bash
     2  #
     3  #    Licensed to the Apache Software Foundation (ASF) under one or more
     4  #    contributor license agreements.  See the NOTICE file distributed with
     5  #    this work for additional information regarding copyright ownership.
     6  #    The ASF licenses this file to You under the Apache License, Version 2.0
     7  #    (the "License"); you may not use this file except in compliance with
     8  #    the License.  You may obtain a copy of the License at
     9  #
    10  #       http://www.apache.org/licenses/LICENSE-2.0
    11  #
    12  #    Unless required by applicable law or agreed to in writing, software
    13  #    distributed under the License is distributed on an "AS IS" BASIS,
    14  #    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    15  #    See the License for the specific language governing permissions and
    16  #    limitations under the License.
    17  #
    18  
    19  # Generates requirements files, which list PyPI dependencies to install in
    20  # Apache Beam Python SDK container images. To generate the list,
    21  # we use two sources of information:
    22  # 1) Requirements of Apache Beam itself, as defined by setup.py.
    23  # 2) A list of dependencies from base_image_requirements_manual.txt, which we
    24  # maintain manually.
    25  
    26  # It is recommended to run this script via gradle commands such as:
    27  # ./gradlew :sdks:python:container:generatePythonRequirementsAll
    28  # ./gradlew :sdks:python:container:py38:generatePythonRequirements
    29  
    30  # You will need Python interpreters for all versions supported by Beam, see:
    31  # https://s.apache.org/beam-python-dev-wiki
    32  
    33  if [[ $# -lt 2 ]]; then
    34    printf "Example usage: \n$> ./sdks/python/container/run_generate_requirements.sh 3.8 <sdk_tarball>"
    35    printf "\n\where 3.8 is the Python major.minor version."
    36    exit 1
    37  fi
    38  
    39  PY_VERSION=$1
    40  SDK_TARBALL=$2
    41  # Use the PIP_EXTRA_OPTIONS environment variable to pass additional flags to the pip install command.
    42  # For example, you can include the --pre flag in $PIP_EXTRA_OPTIONS to download pre-release versions of packages.
    43  # Note that you can modify the behavior of the pip install command in this script by passing in your own $PIP_EXTRA_OPTIONS.
    44  PIP_EXTRA_OPTIONS=$3
    45  
    46  if ! python"$PY_VERSION" --version > /dev/null 2>&1 ; then
    47    echo "Please install a python${PY_VERSION} interpreter. See s.apache.org/beam-python-dev-wiki for Python installation tips."
    48    exit 1
    49  fi
    50  
    51  if ! python"$PY_VERSION" -m venv --help > /dev/null 2>&1 ; then
    52    echo "Your python${PY_VERSION} installation does not have a required venv module. See s.apache.org/beam-python-dev-wiki for Python installation tips."
    53    exit 1
    54  fi
    55  
    56  set -ex
    57  
    58  ENV_PATH="$PWD/build/python${PY_VERSION/./}_requirements_gen"
    59  rm -rf "$ENV_PATH" 2>/dev/null || true
    60  python"${PY_VERSION}" -m venv "$ENV_PATH"
    61  source "$ENV_PATH"/bin/activate
    62  pip install --upgrade pip setuptools wheel
    63  
    64  # Install gcp extra deps since these deps are commonly used with Apache Beam.
    65  # Install dataframe deps to add have Dataframe support in released images.
    66  # Install test deps since some integration tests need dependencies,
    67  # such as pytest, installed in the runner environment.
    68  pip install ${PIP_EXTRA_OPTIONS:+"$PIP_EXTRA_OPTIONS"}  --no-cache-dir "$SDK_TARBALL"[gcp,dataframe,test]
    69  pip install ${PIP_EXTRA_OPTIONS:+"$PIP_EXTRA_OPTIONS"}  --no-cache-dir -r "$PWD"/sdks/python/container/base_image_requirements_manual.txt
    70  
    71  pip uninstall -y apache-beam
    72  echo "Checking for broken dependencies:"
    73  pip check
    74  echo "Installed dependencies:"
    75  pip freeze
    76  
    77  PY_IMAGE="py${PY_VERSION//.}"
    78  REQUIREMENTS_FILE=$PWD/sdks/python/container/$PY_IMAGE/base_image_requirements.txt
    79  cat <<EOT > "$REQUIREMENTS_FILE"
    80  #    Licensed to the Apache Software Foundation (ASF) under one or more
    81  #    contributor license agreements.  See the NOTICE file distributed with
    82  #    this work for additional information regarding copyright ownership.
    83  #    The ASF licenses this file to You under the Apache License, Version 2.0
    84  #    (the "License"); you may not use this file except in compliance with
    85  #    the License.  You may obtain a copy of the License at
    86  #
    87  #       http://www.apache.org/licenses/LICENSE-2.0
    88  #
    89  #    Unless required by applicable law or agreed to in writing, software
    90  #    distributed under the License is distributed on an "AS IS" BASIS,
    91  #    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    92  #    See the License for the specific language governing permissions and
    93  #    limitations under the License.
    94  
    95  # Autogenerated requirements file for Apache Beam $PY_IMAGE container image.
    96  # Run ./gradlew :sdks:python:container:generatePythonRequirementsAll to update.
    97  # Do not edit manually, adjust ../base_image_requirements_manual.txt or
    98  # Apache Beam's setup.py instead, and regenerate the list.
    99  # You will need Python interpreters for all versions supported by Beam, see:
   100  # https://s.apache.org/beam-python-dev-wiki
   101  # Reach out to a committer if you need help.
   102  
   103  EOT
   104  # Remove pkg_resources to guard against
   105  # https://stackoverflow.com/questions/39577984/what-is-pkg-resources-0-0-0-in-output-of-pip-freeze-command
   106  pip freeze | grep -v pkg_resources >> "$REQUIREMENTS_FILE"
   107  rm -rf "$ENV_PATH"