istio.io/istio@v0.0.0-20240520182934-d79c90f27776/bin/retry.sh (about)

     1  #!/bin/bash
     2  
     3  # Copyright Istio Authors
     4  #
     5  # Licensed under the Apache License, Version 2.0 (the "License");
     6  # you may not use this file except in compliance with the License.
     7  # You may obtain a copy of the License at
     8  #
     9  #    http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14  # See the License for the specific language governing permissions and
    15  # limitations under the License.
    16  
    17  # retry.sh retries a command until it succeeds. It accepts a regex pattern to match failures on to
    18  # determine if a retry should be attempted.
    19  # Example: retry.sh "connection timed out" ./my-flaky-script.sh some args
    20  # This will run "my-flaky-script.sh", retrying any failed runs that output "connection timed out" up
    21  # to 5 times.
    22  
    23  function fail {
    24    echo "${1}" >&2
    25    exit 1
    26  }
    27  
    28  function isatty() {
    29   if [ -t 1 ] ; then
    30     return 0
    31    else
    32     return 1
    33    fi
    34  }
    35  
    36  function retry {
    37    local tmpFile
    38    tmpFile=$(mktemp)
    39    trap 'rm -f "${tmpFile}"' EXIT
    40  
    41    local failureRegex="$1"
    42    shift
    43    local n=1
    44    local max=5
    45    while true; do
    46      unset SHELL # Don't let environment control which shell to use
    47      if isatty; then
    48        if [ "$(uname)" == "Darwin" ]; then
    49          script -q -r "${tmpFile}" "${*}"
    50        else
    51          script --flush --quiet --return "${tmpFile}" --command "${*}"
    52        fi
    53      else
    54        # if we aren't a TTY, run directly as script will always run with a tty, which may output content that
    55        # we cannot display
    56        set -o pipefail; "$@" 2>&1 | tee "${tmpFile}"
    57      fi
    58      # shellcheck disable=SC2181
    59      if [[ $? == 0 ]]; then
    60        break
    61      fi
    62      if ! grep -Eq "${failureRegex}" "${tmpFile}"; then
    63        fail "Unexpected failure"
    64      fi
    65      if [[ $n -lt $max ]]; then
    66        ((n++))
    67        echo "Command failed. Attempt $n/$max:"
    68      else
    69        fail "The command has failed after $n attempts."
    70      fi
    71    done
    72  }
    73  
    74  retry "$@"