github.com/apache/beam/sdks/v2@v2.48.2/python/scripts/run_pylint.sh (about)

     1  #!/bin/bash
     2  #
     3  #    Licensed to the Apache Software Foundation (ASF) under one or more
     4  #    contributor license agreements.  See the NOTICE file distributed with
     5  #    this work for additional information regarding copyright ownership.
     6  #    The ASF licenses this file to You under the Apache License, Version 2.0
     7  #    (the "License"); you may not use this file except in compliance with
     8  #    the License.  You may obtain a copy of the License at
     9  #
    10  #       http://www.apache.org/licenses/LICENSE-2.0
    11  #
    12  #    Unless required by applicable law or agreed to in writing, software
    13  #    distributed under the License is distributed on an "AS IS" BASIS,
    14  #    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    15  #    See the License for the specific language governing permissions and
    16  #    limitations under the License.
    17  #
    18  
    19  # This script will run pylint and pep8 on all module files.
    20  #
    21  # Use "pylint apache_beam" to run pylint all files.
    22  # Use "pep8 apache_beam" to run pep8 all files.
    23  #
    24  # The exit-code of the script indicates success or a failure.
    25  
    26  # Check that the script is running in a known directory.
    27  if [[ $PWD != *sdks/python* ]]; then
    28    echo 'Unable to locate Apache Beam Python SDK root directory'
    29    exit 1
    30  fi
    31  
    32  # Go to the Apache Beam Python SDK root
    33  if [[ $PWD != *sdks/python ]]; then
    34    cd $(pwd | sed 's/sdks\/python.*/sdks\/python/')
    35  fi
    36  
    37  set -o errexit
    38  set -o pipefail
    39  
    40  MODULE=$(ls -d -C apache_beam *.py)
    41  
    42  usage(){ echo "Usage: $0 [MODULE|--help]  # The default MODULE is $MODULE"; }
    43  
    44  if test $# -gt 0; then
    45    case "$@" in
    46      --help) usage; exit 1;;
    47  	 *)      MODULE="$*";;
    48    esac
    49  fi
    50  
    51  # Following generated files are excluded from lint checks.
    52  EXCLUDED_GENERATED_FILES=(
    53  "apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_client.py"
    54  "apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_messages.py"
    55  "apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py"
    56  "apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py"
    57  "apache_beam/io/gcp/internal/clients/storage/storage_v1_client.py"
    58  "apache_beam/io/gcp/internal/clients/storage/storage_v1_messages.py"
    59  "apache_beam/coders/proto2_coder_test_messages_pb2.py"
    60  )
    61  
    62  # more portable than shopt -s globstar
    63  while IFS= read -d $'\0' -r file ; do
    64      EXCLUDED_GENERATED_FILES+=("$file")
    65  done < <(find apache_beam/portability/api -type f -name "*pb2*.py" -print0)
    66  
    67  FILES_TO_IGNORE=""
    68  for file in "${EXCLUDED_GENERATED_FILES[@]}"; do
    69    if test -z "$FILES_TO_IGNORE"
    70      then FILES_TO_IGNORE="$(basename $file)"
    71      else FILES_TO_IGNORE="$FILES_TO_IGNORE, $(basename $file)"
    72    fi
    73  done
    74  
    75  echo -e "Skipping lint for files:\n${FILES_TO_IGNORE}"
    76  echo -e "Linting modules:\n${MODULE}"
    77  
    78  echo "Running pylint..."
    79  pylint -j8 ${MODULE} --ignore-patterns="$FILES_TO_IGNORE"
    80  echo "Running flake8..."
    81  flake8 ${MODULE} --count --select=E9,F821,F822,F823 --show-source --statistics \
    82    --exclude="${FILES_TO_IGNORE}"
    83  
    84  echo "Running isort..."
    85  # Skip files where isort is behaving weirdly
    86  ISORT_EXCLUDED=(
    87    "apiclient.py"
    88    "avroio_test.py"
    89    "datastore_wordcount.py"
    90    "datastoreio_test.py"
    91    "hadoopfilesystem.py"
    92    "iobase_test.py"
    93    "fast_coders_test.py"
    94    "slow_coders_test.py"
    95    "tfdv_analyze_and_validate.py"
    96    "preprocess.py"
    97    "model.py"
    98    "taxi.py"
    99    "process_tfma.py"
   100    "doctests_test.py"
   101    "render_test.py"
   102  )
   103  SKIP_PARAM=""
   104  for file in "${ISORT_EXCLUDED[@]}"; do
   105    SKIP_PARAM="$SKIP_PARAM --skip $file"
   106  done
   107  for file in "${EXCLUDED_GENERATED_FILES[@]}"; do
   108    SKIP_PARAM="$SKIP_PARAM --skip $(basename $file)"
   109  done
   110  isort ${MODULE} -p apache_beam --line-width 120 --check-only --order-by-type \
   111      --combine-star --force-single-line-imports --diff --recursive ${SKIP_PARAM}
   112  
   113  echo "Checking unittest.main..."
   114  TESTS_MISSING_MAIN=$(
   115      find ${MODULE} \
   116      | grep '\.py$' \
   117      | xargs grep -l '^import unittest$' \
   118      | xargs grep -L unittest.main \
   119      || true)
   120  if [ -n "${TESTS_MISSING_MAIN}" ]; then
   121    echo -e "\nThe following files are missing a call to unittest.main():"
   122    for FILE in ${TESTS_MISSING_MAIN}; do
   123      echo "  ${FILE}"
   124    done
   125    echo
   126    exit 1
   127  fi