github.com/m3db/m3@v1.5.1-0.20231129193456-75a402aa583b/scripts/development/m3_stack/start_m3.sh (about)

     1  #!/usr/bin/env bash
     2  
     3  set -xe
     4  
     5  source "$(pwd)/../../docker-integration-tests/common.sh"
     6  
     7  # Locally don't care if we hot loop faster
     8  export MAX_TIMEOUT=4
     9  
    10  RELATIVE="./../../.."
    11  prepare_build_cmd() {
    12      build_cmd="cd $RELATIVE && make clean-build docker-dev-prep && cp -r ./docker ./bin/ && $1"
    13  }
    14  DOCKER_ARGS="-d --renew-anon-volumes"
    15  
    16  echo "Bringing up nodes in the background with docker compose, remember to run ./stop.sh when done"
    17  
    18  # need to start Jaeger before m3db or else m3db will not be able to talk to the Jaeger agent.
    19  if [[ "$USE_JAEGER" = true ]] ; then
    20      docker-compose -f docker-compose.yml up $DOCKER_ARGS jaeger
    21      sleep 3
    22      # rely on 204 status code until https://github.com/jaegertracing/jaeger/issues/1450 is resolved.
    23      JAEGER_STATUS=$(curl -s -o /dev/null -w '%{http_code}' localhost:14269)
    24      if [ $JAEGER_STATUS -ne 204 ]; then
    25          echo "Jaeger could not start"
    26          return 1
    27      fi
    28  fi
    29  
    30  M3DBNODE_DEV_IMG=$(docker images m3dbnode:dev | fgrep -iv repository | wc -l | xargs)
    31  M3COORDINATOR_DEV_IMG=$(docker images m3coordinator:dev | fgrep -iv repository | wc -l | xargs)
    32  M3AGGREGATOR_DEV_IMG=$(docker images m3aggregator:dev | fgrep -iv repository | wc -l | xargs)
    33  
    34  if [[ "$M3DBNODE_DEV_IMG" == "0" ]] || [[ "$FORCE_BUILD" == true ]] || [[ "$BUILD_M3DBNODE" == true ]]; then
    35      prepare_build_cmd "make m3dbnode-linux-amd64"
    36      echo "Building m3dbnode binary first"
    37      bash -c "$build_cmd"
    38  
    39      docker-compose -f docker-compose.yml up --build $DOCKER_ARGS m3db_seed
    40  else
    41      docker-compose -f docker-compose.yml up $DOCKER_ARGS m3db_seed
    42  fi
    43  
    44  # Bring up any other replicas
    45  if [[ "$USE_MULTI_DB_NODES" = true ]] ; then
    46      echo "Running multi node"
    47      docker-compose -f docker-compose.yml up $DOCKER_ARGS m3db_data01
    48      docker-compose -f docker-compose.yml up $DOCKER_ARGS m3db_data02
    49  else
    50      echo "Running single node"
    51  fi
    52  
    53  # Use standard coordinator config when bringing up coordinator first time
    54  # Note: Use ".tmp" suffix to be git ignored.
    55  cp ./m3coordinator-standard.yml ./m3coordinator.yml.tmp
    56  if [[ "$USE_MULTIPROCESS_COORDINATOR" = true ]]; then
    57      cat ./m3coordinator-snippet-multiprocess.yml >> ./m3coordinator.yml.tmp
    58  fi
    59  
    60  if [[ "$M3COORDINATOR_DEV_IMG" == "0" ]] || [[ "$FORCE_BUILD" == true ]] || [[ "$BUILD_M3COORDINATOR" == true ]]; then
    61      prepare_build_cmd "make m3coordinator-linux-amd64"
    62      echo "Building m3coordinator binary first"
    63      bash -c "$build_cmd"
    64  
    65      docker-compose -f docker-compose.yml up --build $DOCKER_ARGS m3coordinator01
    66  else
    67      docker-compose -f docker-compose.yml up $DOCKER_ARGS m3coordinator01
    68  fi
    69  
    70  echo "Wait for coordinator API to be up"
    71  ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
    72    'curl -vvvsSf localhost:7201/health'
    73  
    74  if [[ "$USE_AGGREGATOR" = true ]]; then
    75      echo "Running aggregator pipeline"
    76      if [[ "$USE_AGGREGATOR_HA" != true ]]; then
    77          # Use single replica.
    78          curl -vvvsSf -X POST localhost:7201/api/v1/services/m3aggregator/placement/init -d '{
    79              "num_shards": 64,
    80              "replication_factor": 1,
    81              "instances": [
    82                  {
    83                      "id": "m3aggregator01",
    84                      "isolation_group": "rack-a",
    85                      "zone": "embedded",
    86                      "weight": 1024,
    87                      "endpoint": "m3aggregator01:6000",
    88                      "hostname": "m3aggregator01",
    89                      "port": 6000
    90                  }
    91              ]
    92          }'
    93      else
    94          # Use two replicas.
    95          curl -vvvsSf -X POST localhost:7201/api/v1/services/m3aggregator/placement/init -d '{
    96              "num_shards": 64,
    97              "replication_factor": 2,
    98              "instances": [
    99                  {
   100                      "id": "m3aggregator01",
   101                      "isolation_group": "rack-a",
   102                      "zone": "embedded",
   103                      "weight": 1024,
   104                      "endpoint": "m3aggregator01:6000",
   105                      "hostname": "m3aggregator01",
   106                      "port": 6000
   107                  },
   108                  {
   109                      "id": "m3aggregator02",
   110                      "isolation_group": "rack-b",
   111                      "zone": "embedded",
   112                      "weight": 1024,
   113                      "endpoint": "m3aggregator02:6000",
   114                      "hostname": "m3aggregator02",
   115                      "port": 6000
   116                  }
   117              ]
   118          }'
   119      fi
   120  
   121      echo "Initializing m3msg inbound topic for m3aggregator ingestion from m3coordinators"
   122      curl -vvvsSf -X POST -H "Topic-Name: aggregator_ingest" -H "Cluster-Environment-Name: default_env" localhost:7201/api/v1/topic/init -d '{
   123          "numberOfShards": 64
   124      }'
   125  
   126      echo "Adding m3aggregator as a consumer to the aggregator ingest topic"
   127      curl -vvvsSf -X POST -H "Topic-Name: aggregator_ingest" -H "Cluster-Environment-Name: default_env" localhost:7201/api/v1/topic -d '{
   128      "consumerService": {
   129          "serviceId": {
   130          "name": "m3aggregator",
   131          "environment": "default_env",
   132          "zone": "embedded"
   133          },
   134          "consumptionType": "REPLICATED",
   135          "messageTtlNanos": "600000000000"
   136      }
   137      }' # msgs will be discarded after 600000000000ns = 10mins
   138  
   139      # Create outbound m3msg topic for m3 aggregators to coordinators
   140      echo "Initializing m3msg outbound topic for m3 aggregators to coordinators"
   141      curl -vvvsSf -X POST -H "Topic-Name: aggregated_metrics" -H "Cluster-Environment-Name: default_env" localhost:7201/api/v1/topic/init -d '{
   142          "numberOfShards": 64
   143      }'
   144  
   145      if [[ "$M3AGGREGATOR_DEV_IMG" == "0" ]] || [[ "$FORCE_BUILD" == true ]] || [[ "$BUILD_M3AGGREGATOR" == true ]]; then
   146          prepare_build_cmd "make m3aggregator-linux-amd64"
   147          echo "Building m3aggregator binary first"
   148          bash -c "$build_cmd"
   149  
   150          docker-compose -f docker-compose.yml up --build $DOCKER_ARGS m3aggregator01
   151      else
   152          docker-compose -f docker-compose.yml up $DOCKER_ARGS m3aggregator01
   153      fi
   154  
   155      if [[ "$USE_AGGREGATOR_HA" == true ]]; then
   156          # Bring up the second replica
   157          docker-compose -f docker-compose.yml up $DOCKER_ARGS m3aggregator02
   158      fi
   159  else
   160      echo "Not running aggregator pipeline"
   161  fi
   162  
   163  echo "Initializing namespaces"
   164  curl -vvvsSf -X POST localhost:7201/api/v1/services/m3db/namespace -d '{
   165    "name": "metrics_0_30m",
   166    "options": {
   167      "bootstrapEnabled": true,
   168      "flushEnabled": true,
   169      "writesToCommitLog": true,
   170      "cleanupEnabled": true,
   171      "snapshotEnabled": true,
   172      "repairEnabled": false,
   173      "retentionOptions": {
   174        "retentionPeriodDuration": "30m",
   175        "blockSizeDuration": "10m",
   176        "bufferFutureDuration": "5m",
   177        "bufferPastDuration": "5m",
   178        "blockDataExpiry": true,
   179        "blockDataExpiryAfterNotAccessPeriodDuration": "5m"
   180      },
   181      "indexOptions": {
   182        "enabled": true,
   183        "blockSizeDuration": "10m"
   184      },
   185      "aggregationOptions": {
   186        "aggregations": [
   187          {
   188            "aggregated": false
   189          }
   190        ]
   191      },
   192      "stagingState": {
   193        "status": "INITIALIZING"
   194      }
   195    }
   196  }'
   197  curl -vvvsSf -X POST localhost:7201/api/v1/services/m3db/namespace -d '{
   198    "name": "metrics_30s_24h",
   199    "options": {
   200      "bootstrapEnabled": true,
   201      "flushEnabled": true,
   202      "writesToCommitLog": true,
   203      "cleanupEnabled": true,
   204      "snapshotEnabled": true,
   205      "repairEnabled": false,
   206      "retentionOptions": {
   207        "retentionPeriodDuration": "24h",
   208        "blockSizeDuration": "2h",
   209        "bufferFutureDuration": "10m",
   210        "bufferPastDuration": "10m",
   211        "blockDataExpiry": true,
   212        "blockDataExpiryAfterNotAccessPeriodDuration": "5m"
   213      },
   214      "indexOptions": {
   215        "enabled": true,
   216        "blockSizeDuration": "2h"
   217      },
   218      "aggregationOptions": {
   219        "aggregations": [
   220          {
   221            "aggregated": true,
   222            "attributes": {
   223              "resolutionDuration": "30s"
   224            }
   225          }
   226        ]
   227      },
   228      "stagingState": {
   229        "status": "INITIALIZING"
   230      }
   231    }
   232  }'
   233  echo "Done initializing namespaces"
   234  
   235  echo "Validating namespace"
   236  [ "$(curl -sSf localhost:7201/api/v1/services/m3db/namespace | jq .registry.namespaces.metrics_0_30m.indexOptions.enabled)" == true ]
   237  [ "$(curl -sSf localhost:7201/api/v1/services/m3db/namespace | jq .registry.namespaces.metrics_30s_24h.indexOptions.enabled)" == true ]
   238  echo "Done validating namespace"
   239  
   240  echo "Waiting for namespaces to be ready"
   241  [ $(curl -sSf -X POST localhost:7201/api/v1/services/m3db/namespace/ready -d "{ \"name\": \"metrics_0_30m\", \"force\": true }" | grep -c true) -eq 1 ]
   242  [ $(curl -sSf -X POST localhost:7201/api/v1/services/m3db/namespace/ready -d "{ \"name\": \"metrics_30s_24h\", \"force\": true }" | grep -c true) -eq 1 ]
   243  echo "Done waiting for namespaces to be ready"
   244  
   245  echo "Initializing topology"
   246  if [[ "$USE_MULTI_DB_NODES" = true ]] ; then
   247      curl -vvvsSf -X POST localhost:7201/api/v1/services/m3db/placement/init -d '{
   248          "num_shards": 64,
   249          "replication_factor": 3,
   250          "instances": [
   251              {
   252                  "id": "m3db_seed",
   253                  "isolation_group": "rack-a",
   254                  "zone": "embedded",
   255                  "weight": 1024,
   256                  "endpoint": "m3db_seed:9000",
   257                  "hostname": "m3db_seed",
   258                  "port": 9000
   259              },
   260              {
   261                  "id": "m3db_data01",
   262                  "isolation_group": "rack-b",
   263                  "zone": "embedded",
   264                  "weight": 1024,
   265                  "endpoint": "m3db_data01:9000",
   266                  "hostname": "m3db_data01",
   267                  "port": 9000
   268              },
   269              {
   270                  "id": "m3db_data02",
   271                  "isolation_group": "rack-c",
   272                  "zone": "embedded",
   273                  "weight": 1024,
   274                  "endpoint": "m3db_data02:9000",
   275                  "hostname": "m3db_data02",
   276                  "port": 9000
   277              }
   278          ]
   279      }'
   280  else
   281      curl -vvvsSf -X POST localhost:7201/api/v1/services/m3db/placement/init -d '{
   282          "num_shards": 64,
   283          "replication_factor": 1,
   284          "instances": [
   285              {
   286                  "id": "m3db_seed",
   287                  "isolation_group": "rack-a",
   288                  "zone": "embedded",
   289                  "weight": 1024,
   290                  "endpoint": "m3db_seed:9000",
   291                  "hostname": "m3db_seed",
   292                  "port": 9000
   293              }
   294          ]
   295      }'
   296  fi
   297  
   298  echo "Validating topology"
   299  [ "$(curl -sSf localhost:7201/api/v1/services/m3db/placement | jq .placement.instances.m3db_seed.id)" == '"m3db_seed"' ]
   300  echo "Done validating topology"
   301  
   302  echo "Waiting until shards are marked as available"
   303  ATTEMPTS=100 TIMEOUT=2 retry_with_backoff  \
   304    '[ "$(curl -sSf 0.0.0.0:7201/api/v1/services/m3db/placement | grep -c INITIALIZING)" -eq 0 ]'
   305  
   306  if [[ "$USE_AGGREGATOR" = true ]]; then
   307      echo "Initializing M3Coordinator topology"
   308      curl -vvvsSf -X POST localhost:7201/api/v1/services/m3coordinator/placement/init -d '{
   309          "instances": [
   310              {
   311                  "id": "m3coordinator01",
   312                  "zone": "embedded",
   313                  "endpoint": "m3coordinator01:7507",
   314                  "hostname": "m3coordinator01",
   315                  "port": 7507
   316              }
   317          ]
   318      }'
   319      echo "Done initializing M3Coordinator topology"
   320  
   321      echo "Validating M3Coordinator topology"
   322      [ "$(curl -sSf localhost:7201/api/v1/services/m3coordinator/placement | jq .placement.instances.m3coordinator01.id)" == '"m3coordinator01"' ]
   323      echo "Done validating topology"
   324  
   325      # Do this after placement for m3coordinator is created.
   326      echo "Adding coordinator as a consumer to the aggregator outbound topic"
   327      curl -vvvsSf -X POST -H "Topic-Name: aggregated_metrics" -H "Cluster-Environment-Name: default_env" localhost:7201/api/v1/topic -d '{
   328          "consumerService": {
   329                  "serviceId": {
   330                  "name": "m3coordinator",
   331                  "environment": "default_env",
   332                  "zone": "embedded"
   333              },
   334              "consumptionType": "SHARED",
   335              "messageTtlNanos": "600000000000"
   336          }
   337      }' # msgs will be discarded after 600000000000ns = 10mins
   338  
   339      # Restart with aggregator coordinator config
   340      docker-compose -f docker-compose.yml stop m3coordinator01
   341  
   342      # Note: Use ".tmp" suffix to be git ignored.
   343      cp ./m3coordinator-aggregator.yml ./m3coordinator.yml.tmp
   344      if [[ "$USE_MULTIPROCESS_COORDINATOR" = true ]]; then
   345          cat ./m3coordinator-snippet-multiprocess.yml >> ./m3coordinator.yml.tmp
   346      fi
   347  
   348      docker-compose -f docker-compose.yml up $DOCKER_ARGS m3coordinator01
   349  fi
   350  
   351  echo "Starting Prometheus"
   352  if [[ "$FORCE_BUILD" == true ]] || [[ "$BUILD_PROMETHEUS" == true ]]; then
   353      docker-compose -f docker-compose.yml up --build $DOCKER_ARGS prometheus01
   354  else
   355      docker-compose -f docker-compose.yml up $DOCKER_ARGS prometheus01
   356  fi
   357  
   358  if [[ "$USE_PROMETHEUS_HA" = true ]] ; then
   359      echo "Starting Prometheus HA replica"
   360      docker-compose -f docker-compose.yml up $DOCKER_ARGS prometheus02
   361  fi
   362  
   363  echo "Starting Grafana"
   364  if [[ "$FORCE_BUILD" == true ]] || [[ "$BUILD_GRAFANA" == true ]]; then
   365      docker-compose -f docker-compose.yml up --build $DOCKER_ARGS grafana
   366  else
   367      docker-compose -f docker-compose.yml up $DOCKER_ARGS grafana
   368  fi
   369  
   370  if [[ "$USE_JAEGER" = true ]] ; then
   371      echo "Jaeger UI available at localhost:16686"
   372  fi
   373  echo "Prometheus available at localhost:9090"
   374  if [[ "$USE_PROMETHEUS_HA" = true ]] ; then
   375      echo "Prometheus HA replica available at localhost:9091"
   376  fi
   377  echo "Grafana available at localhost:3000"
   378  echo "Run ./stop.sh to shutdown nodes when done"