github.com/m3db/m3@v1.5.0/scripts/docker-integration-tests/common.sh (about)

     1  #!/usr/bin/env bash
     2  
     3  # Retries a command a configurable number of times with backoff.
     4  #
     5  # The retry count is given by ATTEMPTS (default 3), the initial backoff
     6  # timeout is given by TIMEOUT in seconds (default 1.)
     7  #
     8  # Successive backoffs double the timeout.
     9  # adapted from: https://stackoverflow.com/questions/8350942/how-to-re-run-the-curl-command-automatically-when-the-error-occurs/8351489#8351489
    10  function retry_with_backoff {
    11    local max_attempts=${ATTEMPTS-5}
    12    local timeout=${TIMEOUT-1}
    13    local max_timeout=${MAX_TIMEOUT}
    14    local attempt=1
    15    local exitCode=0
    16  
    17    while (( $attempt < $max_attempts ))
    18    do
    19      set +e
    20      eval "$@"
    21      exitCode=$?
    22      set -e
    23      if [[ "$exitCode" == 0 ]]; then
    24        return 0
    25      fi
    26  
    27      echo "Failure! Retrying in $timeout.." 1>&2
    28      sleep $timeout
    29      attempt=$(( attempt + 1 ))
    30      timeout=$(( timeout * 2 ))
    31      if [[ $max_timeout != "" ]]; then
    32        if [[ $timeout -gt $max_timeout ]]; then
    33          timeout=$max_timeout
    34        fi
    35      fi
    36    done
    37  
    38    if [[ $exitCode != 0 ]]
    39    then
    40      echo "You've failed me for the last time! ($@)" 1>&2
    41    fi
    42  
    43    return $exitCode
    44  }
    45  
    46  function setup_single_m3db_node_long_namespaces {
    47    local dbnode_host=${DBNODE_HOST:-dbnode01}
    48    local dbnode_port=${DBNODE_PORT:-9000}
    49    local dbnode_health_port=${DBNODE_HEALTH_PORT:-9002}
    50    local dbnode_id=${DBNODE_ID:-m3db_local}
    51    local coordinator_port=${COORDINATOR_PORT:-7201}
    52    local zone=${ZONE:-embedded}
    53  
    54    echo "Wait for API to be available"
    55    ATTEMPTS=100 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
    56      '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace | jq ".namespaces | length")" == "0" ]'
    57  
    58    echo "Adding placement and agg namespace"
    59    curl -vvvsSf -X POST 0.0.0.0:${coordinator_port}/api/v1/database/create -d '{
    60      "type": "cluster",
    61      "namespaceName": "agg",
    62      "retentionTime": "24h",
    63      "num_shards": 4,
    64      "replicationFactor": 1,
    65      "hosts": [
    66        {
    67            "id": "'${dbnode_id}'",
    68            "isolation_group": "rack-a",
    69            "zone": "'${zone}'",
    70            "weight": 1024,
    71            "address": "'"${dbnode_host}"'",
    72            "port": '"${dbnode_port}"'
    73        }
    74      ]
    75    }'
    76  
    77    echo "Updating aggregation options for agg namespace"
    78    curl -vvvsSf -X PUT 0.0.0.0:${coordinator_port}/api/v1/services/m3db/namespace -d '{
    79      "name": "agg",
    80      "options": {
    81        "aggregationOptions": {
    82          "aggregations": [
    83             {
    84              "aggregated": true,
    85              "attributes": {
    86                "resolutionDuration": "30s",
    87                "downsampleOptions": { "all": false }
    88              }
    89            }
    90          ]
    91        }
    92      }
    93    }'
    94  
    95    echo "Wait until placement is init'd"
    96    ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
    97      '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/placement | jq .placement.instances.'${dbnode_id}'.id)" == \"'${dbnode_id}'\" ]'
    98  
    99    echo "Wait until agg namespace is ready"
   100    ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
   101      '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace/ready -d "{ \"name\": \"agg\"}" | grep -c true)" -eq 1 ]'
   102  
   103    wait_for_namespaces
   104  
   105    echo "Adding agg2d namespace"
   106    curl -vvvsSf -X POST 0.0.0.0:${coordinator_port}/api/v1/database/namespace/create -d '{
   107      "namespaceName": "agg2d",
   108      "retentionTime": "48h"
   109    }'
   110  
   111    echo "Updating aggregation options for agg namespace"
   112    curl -vvvsSf -X PUT 0.0.0.0:${coordinator_port}/api/v1/services/m3db/namespace -d '{
   113      "name": "agg2d",
   114      "options": {
   115        "aggregationOptions": {
   116          "aggregations": [
   117            {
   118              "aggregated": true,
   119              "attributes": {
   120                "resolutionDuration": "1m",
   121                "downsampleOptions": { "all": false }
   122              }
   123            }
   124          ]
   125        }
   126      }
   127    }'
   128  
   129    echo "Wait until agg2d namespace is init'd"
   130    ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
   131      '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace | jq .registry.namespaces.agg2d.indexOptions.enabled)" == true ]'
   132  
   133    echo "Wait until agg2d namespace is ready"
   134    ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
   135      '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace/ready -d "{ \"name\": \"agg2d\"}" | grep -c true)" -eq 1 ]'
   136  
   137    echo "Wait until bootstrapped"
   138    ATTEMPTS=100 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
   139      '[ "$(curl -sSf 0.0.0.0:'"${dbnode_health_port}"'/health | jq .bootstrapped)" == true ]'
   140  }
   141  
   142  function setup_single_m3db_node {
   143    local dbnode_host=${DBNODE_HOST:-dbnode01}
   144    local dbnode_port=${DBNODE_PORT:-9000}
   145    local dbnode_health_port=${DBNODE_HEALTH_PORT:-9002}
   146    local dbnode_id=${DBNODE_ID:-m3db_local}
   147    local coordinator_port=${COORDINATOR_PORT:-7201}
   148    local zone=${ZONE:-embedded}
   149    local agg_resolution=${AGG_RESOLUTION:-15s}
   150    local agg_retention=${AGG_RETENTION:-10h}
   151  
   152    echo "Wait for API to be available"
   153    ATTEMPTS=100 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
   154      '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace | jq ".namespaces | length")" == "0" ]'
   155  
   156    echo "Adding placement and agg namespace"
   157    curl -vvvsSf -X POST 0.0.0.0:${coordinator_port}/api/v1/database/create -d '{
   158      "type": "cluster",
   159      "namespaceName": "agg",
   160      "retentionTime": "'${agg_retention}'",
   161      "num_shards": 4,
   162      "replicationFactor": 1,
   163      "hosts": [
   164        {
   165            "id": "'${dbnode_id}'",
   166            "isolation_group": "rack-a",
   167            "zone": "'${zone}'",
   168            "weight": 1024,
   169            "address": "'"${dbnode_host}"'",
   170            "port": '"${dbnode_port}"'
   171        }
   172      ]
   173    }'
   174  
   175    echo "Updating aggregation options for agg namespace"
   176    curl -vvvsSf -X PUT 0.0.0.0:${coordinator_port}/api/v1/services/m3db/namespace -d '{
   177      "name": "agg",
   178      "options": {
   179        "aggregationOptions": {
   180          "aggregations": [
   181            {
   182              "aggregated": true,
   183              "attributes": {
   184                "resolutionDuration": "'${agg_resolution}'"
   185              }
   186            }
   187          ]
   188        }
   189      }
   190    }'
   191  
   192    echo "Wait until placement is init'd"
   193    ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
   194      '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/placement | jq .placement.instances.'${dbnode_id}'.id)" == \"'${dbnode_id}'\" ]'
   195  
   196    echo "Wait until agg namespace is ready"
   197    ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
   198      '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace/ready -d "{ \"name\": \"agg\"}" | grep -c true)" -eq 1 ]'
   199  
   200    wait_for_namespaces
   201  
   202    echo "Wait until bootstrapped"
   203    ATTEMPTS=100 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
   204      '[ "$(curl -sSf 0.0.0.0:'"${dbnode_health_port}"'/health | jq .bootstrapped)" == true ]'
   205  }
   206  
   207  function setup_two_m3db_nodes {
   208    local dbnode_id_1=${DBNODE_ID_01:-m3db_local_1}
   209    local dbnode_id_2=${DBNODE_ID_02:-m3db_local_2}
   210    local dbnode_host_1=${DBNODE_HOST_01:-dbnode01}
   211    local dbnode_host_2=${DBNODE_HOST_02:-dbnode02}
   212    local dbnode_port=${DBNODE_PORT:-9000}
   213    local dbnode_host_1_health_port=${DBNODE_HEALTH_PORT_01:-9012}
   214    local dbnode_host_2_health_port=${DBNODE_HEALTH_PORT_02:-9022}
   215    local coordinator_port=${COORDINATOR_PORT:-7201}
   216    local agg_resolution=${AGG_RESOLUTION:-15s}
   217    local agg_retention=${AGG_RETENTION:-10h}
   218  
   219    echo "Wait for API to be available"
   220    ATTEMPTS=100 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
   221      '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace | jq ".namespaces | length")" == "0" ]'
   222  
   223    echo "Adding placement and agg namespace"
   224    curl -vvvsSf -X POST 0.0.0.0:${coordinator_port}/api/v1/database/create -d '{
   225      "type": "cluster",
   226      "namespaceName": "agg",
   227      "retentionTime": "'${agg_retention}'",
   228      "num_shards": 2,
   229      "replicationFactor": 2,
   230      "hosts": [
   231        {
   232            "id": "'"${dbnode_id_1}"'",
   233            "isolation_group": "rack-a",
   234            "zone": "embedded",
   235            "weight": 1024,
   236            "address": "'"${dbnode_host_1}"'",
   237            "port": '"${dbnode_port}"'
   238        },
   239        {
   240            "id": "'"${dbnode_id_2}"'",
   241            "isolation_group": "rack-b",
   242            "zone": "embedded",
   243            "weight": 1024,
   244            "address": "'"${dbnode_host_2}"'",
   245            "port": '"${dbnode_port}"'
   246        }
   247      ]
   248    }'
   249  
   250    echo "Updating aggregation options for agg namespace"
   251    curl -vvvsSf -X PUT 0.0.0.0:${coordinator_port}/api/v1/services/m3db/namespace -d '{
   252      "name": "agg",
   253      "options": {
   254        "aggregationOptions": {
   255          "aggregations": [
   256            {
   257              "aggregated": true,
   258              "attributes": {
   259                "resolutionDuration": "'${agg_resolution}'"
   260              }
   261            }
   262          ]
   263        }
   264      }
   265    }'
   266  
   267    echo "Wait until placement is init'd"
   268    ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
   269      '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/placement | jq .placement.instances.'"${dbnode_id_1}"'.id)" == \"'"${dbnode_id_1}"'\" ]'
   270  
   271    echo "Wait until agg namespace is ready"
   272    ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
   273      '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace/ready -d "{ \"name\": \"agg\"}" | grep -c true)" -eq 1 ]'
   274  
   275    wait_for_namespaces
   276  
   277    echo "Wait until bootstrapped"
   278    ATTEMPTS=100 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
   279      '[ "$(curl -sSf 0.0.0.0:'"${dbnode_host_1_health_port}"'/health | jq .bootstrapped)" == true ]'
   280    ATTEMPTS=100 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
   281      '[ "$(curl -sSf 0.0.0.0:'"${dbnode_host_2_health_port}"'/health | jq .bootstrapped)" == true ]'
   282  }
   283  
   284  function wait_for_namespaces {
   285    local coordinator_port=${COORDINATOR_PORT:-7201}
   286    local unagg_retention=${UNAGG_RETENTION:-10h}
   287  
   288    echo "Wait until agg namespace is init'd"
   289    ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
   290      '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace | jq .registry.namespaces.agg.indexOptions.enabled)" == true ]'
   291  
   292    echo "Adding unagg namespace"
   293    curl -vvvsSf -X POST 0.0.0.0:${coordinator_port}/api/v1/database/namespace/create -d '{
   294      "namespaceName": "unagg",
   295      "retentionTime": "'${unagg_retention}'"
   296    }'
   297  
   298    echo "Wait until unagg namespace is init'd"
   299    ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
   300      '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace | jq .registry.namespaces.unagg.indexOptions.enabled)" == true ]'
   301  
   302    echo "Wait until unagg namespace is ready"
   303    ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
   304      '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace/ready -d "{ \"name\": \"unagg\"}" | grep -c true)" -eq 1 ]'
   305  
   306    echo "Adding coldWritesRepairAndNoIndex namespace"
   307    curl -vvvsSf -X POST 0.0.0.0:${coordinator_port}/api/v1/services/m3db/namespace -d '{
   308      "name": "coldWritesRepairAndNoIndex",
   309      "options": {
   310        "bootstrapEnabled": true,
   311        "flushEnabled": true,
   312        "writesToCommitLog": true,
   313        "cleanupEnabled": true,
   314        "snapshotEnabled": true,
   315        "repairEnabled": true,
   316        "coldWritesEnabled": true,
   317        "retentionOptions": {
   318          "retentionPeriodDuration": "4h",
   319          "blockSizeDuration": "1h",
   320          "bufferFutureDuration": "10m",
   321          "bufferPastDuration": "10m",
   322          "blockDataExpiry": true,
   323          "blockDataExpiryAfterNotAccessPeriodDuration": "5m"
   324        },
   325        "aggregationOptions": {
   326          "aggregations": [
   327            { "aggregated": false }
   328          ]
   329        }
   330      }
   331    }'
   332  
   333    echo "Wait until coldWritesRepairAndNoIndex namespace is init'd"
   334    ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
   335      '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace | jq .registry.namespaces.coldWritesRepairAndNoIndex.coldWritesEnabled)" == true ]'
   336  }
   337