github.com/apache/beam/sdks/v2@v2.48.2/python/container/run_validatescontainer.sh (about)

     1  #!/bin/bash
     2  #
     3  #    Licensed to the Apache Software Foundation (ASF) under one or more
     4  #    contributor license agreements.  See the NOTICE file distributed with
     5  #    this work for additional information regarding copyright ownership.
     6  #    The ASF licenses this file to You under the Apache License, Version 2.0
     7  #    (the "License"); you may not use this file except in compliance with
     8  #    the License.  You may obtain a copy of the License at
     9  #
    10  #       http://www.apache.org/licenses/LICENSE-2.0
    11  #
    12  #    Unless required by applicable law or agreed to in writing, software
    13  #    distributed under the License is distributed on an "AS IS" BASIS,
    14  #    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    15  #    See the License for the specific language governing permissions and
    16  #    limitations under the License.
    17  #
    18  
    19  # This script will be run by Jenkins as a post commit test. In order to run
    20  # locally make the following changes:
    21  #
    22  # GCS_LOCATION -> Temporary location to use for service tests.
    23  # PROJECT      -> Project name to use for dataflow and docker images.
    24  # REGION       -> Region name to use for Dataflow
    25  #
    26  # Execute from the root of the repository:
    27  #     test Python3.7 container:
    28  #         ./gradlew :sdks:python:test-suites:dataflow:py37:validatesContainer
    29  #     test Python3.8 container:
    30  #         ./gradlew :sdks:python:test-suites:dataflow:py38:validatesContainer
    31  #     or test all supported python versions together:
    32  #         ./gradlew :sdks:python:test-suites:dataflow:validatesContainer
    33  
    34  echo "This script must be executed in the root of beam project. Please set GCS_LOCATION, PROJECT and REGION as desired."
    35  
    36  if [[ $# != 2 ]]; then
    37    printf "Usage: \n$> ./sdks/python/container/run_validatescontainer.sh <python_version> <sdk_location>"
    38    printf "\n\tpython_version: [required] Python version used for container build and run tests."
    39    printf " Sample value: 3.9"
    40    exit 1
    41  fi
    42  
    43  set -e
    44  set -v
    45  
    46  # Where to store integration test outputs.
    47  GCS_LOCATION=${GCS_LOCATION:-gs://temp-storage-for-end-to-end-tests}
    48  
    49  # Project for the container and integration test
    50  PROJECT=${PROJECT:-apache-beam-testing}
    51  REGION=${REGION:-us-central1}
    52  IMAGE_PREFIX="$(grep 'docker_image_default_repo_prefix' gradle.properties | cut -d'=' -f2)"
    53  SDK_VERSION="$(grep 'sdk_version' gradle.properties | cut -d'=' -f2)"
    54  PY_VERSION=$1
    55  IMAGE_NAME="${IMAGE_PREFIX}python${PY_VERSION}_sdk"
    56  CONTAINER_PROJECT="sdks:python:container:py${PY_VERSION//.}"  # Note: we substitute away the dot in the version.
    57  PY_INTERPRETER="python${PY_VERSION}"
    58  
    59  XUNIT_FILE="pytest-$IMAGE_NAME.xml"
    60  
    61  # Verify in the root of the repository
    62  test -d sdks/python/container
    63  
    64  # Verify docker and gcloud commands exist
    65  command -v docker
    66  command -v gcloud
    67  docker -v
    68  gcloud -v
    69  
    70  # Verify docker image has been built.
    71  docker images | grep "apache/$IMAGE_NAME" | grep "$SDK_VERSION"
    72  
    73  TAG=$(date +%Y%m%d-%H%M%S%N)
    74  CONTAINER=us.gcr.io/$PROJECT/$USER/$IMAGE_NAME
    75  PREBUILD_SDK_CONTAINER_REGISTRY_PATH=us.gcr.io/$PROJECT/$USER/prebuild_python${PY_VERSION//.}_sdk
    76  echo "Using container $CONTAINER"
    77  
    78  # Tag the docker container.
    79  docker tag "apache/$IMAGE_NAME:$SDK_VERSION" "$CONTAINER:$TAG"
    80  
    81  # Push the container.
    82  gcloud docker -- push $CONTAINER:$TAG
    83  
    84  function cleanup_container {
    85    # Delete the container locally and remotely
    86    docker rmi $CONTAINER:$TAG || echo "Failed to remove container image"
    87    for image in $(docker images --format '{{.Repository}}:{{.Tag}}' | grep $PREBUILD_SDK_CONTAINER_REGISTRY_PATH)
    88      do docker rmi $image || echo "Failed to remove prebuilt sdk container image"
    89    done
    90    gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to delete container"
    91    for digest in $(gcloud container images list-tags $PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk  --format="get(digest)")
    92      do gcloud container images delete $PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk@$digest --force-delete-tags --quiet || echo "Failed to remove prebuilt sdk container image"
    93    done
    94  
    95    echo "Removed the container"
    96  }
    97  trap cleanup_container EXIT
    98  
    99  echo ">>> Successfully built and push container $CONTAINER"
   100  
   101  cd sdks/python
   102  SDK_LOCATION=$2
   103  
   104  # Run ValidatesRunner tests on Google Cloud Dataflow service
   105  echo ">>> RUNNING DATAFLOW RUNNER VALIDATESCONTAINER TEST"
   106  pytest -o junit_suite_name=$IMAGE_NAME \
   107    -m="it_validatescontainer" \
   108    --show-capture=no \
   109    --numprocesses=1 \
   110    --timeout=1800 \
   111    --junitxml=$XUNIT_FILE \
   112    --ignore-glob '.*py3\d?\.py$' \
   113    --log-cli-level=INFO \
   114    --test-pipeline-options=" \
   115      --runner=TestDataflowRunner \
   116      --project=$PROJECT \
   117      --region=$REGION \
   118      --sdk_container_image=$CONTAINER:$TAG \
   119      --staging_location=$GCS_LOCATION/staging-validatesrunner-test \
   120      --temp_location=$GCS_LOCATION/temp-validatesrunner-test \
   121      --output=$GCS_LOCATION/output \
   122      --sdk_location=$SDK_LOCATION \
   123      --num_workers=1 \
   124      --docker_registry_push_url=$PREBUILD_SDK_CONTAINER_REGISTRY_PATH"
   125  
   126  echo ">>> SUCCESS DATAFLOW RUNNER VALIDATESCONTAINER TEST"