vitess.io/vitess@v0.16.2/tools/e2e_test_runner.sh (about)

     1  #!/bin/bash
     2  
     3  # Copyright 2019 The Vitess Authors.
     4  #
     5  # Licensed under the Apache License, Version 2.0 (the "License");
     6  # you may not use this file except in compliance with the License.
     7  # You may obtain a copy of the License at
     8  #
     9  #     http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14  # See the License for the specific language governing permissions and
    15  # limitations under the License.
    16  
    17  # Custom Go endtoend test runner which runs all endtoend tests in parallel
    18  # except for known flaky tests.
    19  # Flaky unit tests are run sequentially in the second phase and retried up to
    20  # three times.
    21  
    22  # Why are there flaky unit tests?
    23  #
    24  # Some of the Go unit tests are inherently flaky e.g. because they use the
    25  # real timer implementation and might fail when things take longer as usual.
    26  # In particular, this happens when the system is under load and threads do not
    27  # get scheduled as fast as usual. Then, the expected timings do not match.
    28  
    29  # Set VT_GO_PARALLEL variable in the same way as the Makefile does.
    30  # We repeat this here because this script is called directly by test.go
    31  # and not via the Makefile.
    32  
    33  source build.env
    34  
    35  if [[ -z $VT_GO_PARALLEL && -n $VT_GO_PARALLEL_VALUE ]]; then
    36    VT_GO_PARALLEL="-p $VT_GO_PARALLEL_VALUE"
    37  fi
    38  
    39  # All Go packages with test files.
    40  # Output per line: <full Go package name> <all _test.go files in the package>*
    41  packages_with_tests=$(go list -f '{{if len .TestGoFiles}}{{.ImportPath}} {{join .TestGoFiles " "}}{{end}}{{if len .XTestGoFiles}}{{.ImportPath}} {{join .XTestGoFiles " "}}{{end}}' ./go/.../endtoend/... | sort)
    42  
    43  # Flaky tests have the suffix "_flaky_test.go".
    44  all_except_flaky_and_cluster_tests=$(echo "$packages_with_tests" | grep -vE ".+ .+_flaky_test\.go" |  grep -vE "go/test/endtoend" | cut -d" " -f1)
    45  flaky_tests=$(echo "$packages_with_tests" | grep -E ".+ .+_flaky_test\.go" | grep -vE "go/test/endtoend" | cut -d" " -f1)
    46  
    47  # Run non-flaky tests.
    48  echo "$all_except_flaky_and_cluster_tests" | xargs go test -count=1 $VT_GO_PARALLEL
    49  if [ $? -ne 0 ]; then
    50    echo "ERROR: Go unit tests failed. See above for errors."
    51    echo
    52    echo "This should NOT happen. Did you introduce a flaky unit test?"
    53    echo "If so, please rename it to the suffix _flaky_test.go."
    54    exit 1
    55  fi
    56  
    57  # Run flaky tests sequentially. Retry when necessary.
    58  for pkg in $flaky_tests; do
    59    max_attempts=3
    60    attempt=1
    61    # Set a timeout because some tests may deadlock when they flake.
    62    until go test -timeout 30s $VT_GO_PARALLEL $pkg; do
    63      echo "FAILED (try $attempt/$max_attempts) in $pkg (return code $?). See above for errors."
    64      if [ $((++attempt)) -gt $max_attempts ]; then
    65        echo "ERROR: Flaky Go unit tests in package $pkg failed too often (after $max_attempts retries). Please reduce the flakiness."
    66        exit 1
    67      fi
    68    done
    69  done