github.com/m3db/m3@v1.5.0/scripts/docker-integration-tests/aggregator_legacy/test.sh (about)

     1  #!/usr/bin/env bash
     2  
     3  set -xe
     4  
     5  source "$M3_PATH"/scripts/docker-integration-tests/common.sh
     6  REVISION=$(git rev-parse HEAD)
     7  COMPOSE_FILE="$M3_PATH"/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml
     8  export REVISION
     9  
    10  echo "Run m3dbnode"
    11  docker-compose -f ${COMPOSE_FILE} up -d dbnode01
    12  
    13  # Stop containers on exit
    14  METRIC_EMIT_PID="-1"
    15  function defer {
    16    docker-compose -f ${COMPOSE_FILE} down || echo "unable to shutdown containers" # CI fails to stop all containers sometimes
    17    if [ "$METRIC_EMIT_PID" != "-1" ]; then
    18      echo "Kill metric emit process"
    19      kill $METRIC_EMIT_PID
    20    fi
    21  }
    22  trap defer EXIT
    23  
    24  echo "Setup DB node"
    25  AGG_RESOLUTION=10s AGG_RETENTION=6h setup_single_m3db_node
    26  
    27  echo "Initializing aggregator topology"
    28  curl -vvvsSf -X POST -H "Cluster-Environment-Name: override_test_env" localhost:7201/api/v1/services/m3aggregator/placement/init -d '{
    29      "num_shards": 64,
    30      "replication_factor": 2,
    31      "instances": [
    32          {
    33              "id": "m3aggregator01",
    34              "isolation_group": "availability-zone-a",
    35              "zone": "embedded",
    36              "weight": 100,
    37              "endpoint": "m3aggregator01:6000",
    38              "hostname": "m3aggregator01",
    39              "port": 6000
    40          },
    41          {
    42              "id": "m3aggregator02",
    43              "isolation_group": "availability-zone-b",
    44              "zone": "embedded",
    45              "weight": 100,
    46              "endpoint": "m3aggregator02:6000",
    47              "hostname": "m3aggregator02",
    48              "port": 6000
    49          }
    50      ]
    51  }'
    52  
    53  echo "Initializing m3msg topic for m3coordinator ingestion from m3aggregators"
    54  curl -vvvsSf -X POST -H "Cluster-Environment-Name: override_test_env" localhost:7201/api/v1/topic/init -d '{
    55      "numberOfShards": 64
    56  }'
    57  
    58  echo "Initializing m3coordinator topology"
    59  curl -vvvsSf -X POST localhost:7201/api/v1/services/m3coordinator/placement/init -d '{
    60      "instances": [
    61          {
    62              "id": "m3coordinator01",
    63              "zone": "embedded",
    64              "endpoint": "m3coordinator01:7507",
    65              "hostname": "m3coordinator01",
    66              "port": 7507
    67          }
    68      ]
    69  }'
    70  echo "Done initializing m3coordinator topology"
    71  
    72  echo "Validating m3coordinator topology"
    73  [ "$(curl -sSf localhost:7201/api/v1/services/m3coordinator/placement | jq .placement.instances.m3coordinator01.id)" == '"m3coordinator01"' ]
    74  echo "Done validating topology"
    75  
    76  # Do this after placement for m3coordinator is created.
    77  echo "Adding m3coordinator as a consumer to the aggregator topic"
    78  curl -vvvsSf -X POST -H "Cluster-Environment-Name: override_test_env" localhost:7201/api/v1/topic -d '{
    79    "consumerService": {
    80      "serviceId": {
    81        "name": "m3coordinator",
    82        "environment": "default_env",
    83        "zone": "embedded"
    84      },
    85      "consumptionType": "SHARED",
    86      "messageTtlNanos": "600000000000"
    87    }
    88  }' # msgs will be discarded after 600000000000ns = 10mins
    89  
    90  echo "Running m3coordinator container"
    91  echo "> port 7202 is coordinator API"
    92  echo "> port 7203 is coordinator metrics"
    93  echo "> port 7204 is coordinator graphite ingest"
    94  echo "> port 7507 is coordinator m3msg ingest from aggregator ingest"
    95  docker-compose -f ${COMPOSE_FILE} up -d m3coordinator01
    96  COORDINATOR_API="localhost:7202"
    97  
    98  echo "Running m3aggregator containers"
    99  docker-compose -f ${COMPOSE_FILE} up -d m3aggregator01
   100  docker-compose -f ${COMPOSE_FILE} up -d m3aggregator02
   101  
   102  echo "Verifying aggregation with remote aggregators"
   103  
   104  function read_carbon {
   105    target=$1
   106    expected_val=$2
   107    end=$(date +%s)
   108    start=$(($end-1000))
   109    RESPONSE=$(curl -sSfg "http://${COORDINATOR_API}/api/v1/graphite/render?target=$target&from=$start&until=$end")
   110    test "$(echo "$RESPONSE" | jq ".[0].datapoints | .[][0] | select(. != null)" | tail -n 1)" = "$expected_val"
   111    return $?
   112  }
   113  
   114  # Send metric values 40 and 44 every second
   115  echo "Sending unaggregated carbon metrics to m3coordinator"
   116  bash -c 'while true; do t=$(date +%s); echo "foo.bar.baz 40 $t" | nc 0.0.0.0 7204; echo "foo.bar.baz 44 $t" | nc 0.0.0.0 7204; sleep 1; done' &
   117  
   118  # Track PID to kill on exit
   119  METRIC_EMIT_PID="$!"
   120  
   121  # Read back the averaged averaged metric, we configured graphite
   122  # aggregation policy to average each tile and we are emitting
   123  # values 40 and 44 to get an average of 42 each tile
   124  echo "Read back aggregated averaged metric"
   125  ATTEMPTS=10 TIMEOUT=1 retry_with_backoff read_carbon foo.bar.* 42