github.com/m3db/m3@v1.5.0/scripts/docker-integration-tests/carbon/test.sh (about) 1 #!/usr/bin/env bash 2 3 set -xe 4 5 source "$M3_PATH"/scripts/docker-integration-tests/common.sh 6 REVISION=$(git rev-parse HEAD) 7 SCRIPT_PATH="$M3_PATH"/scripts/docker-integration-tests/carbon 8 COMPOSE_FILE=$SCRIPT_PATH/docker-compose.yml 9 EXPECTED_PATH=$SCRIPT_PATH/expected 10 export REVISION 11 12 echo "Run m3dbnode and m3coordinator containers" 13 docker-compose -f ${COMPOSE_FILE} up -d dbnode01 14 docker-compose -f ${COMPOSE_FILE} up -d coordinator01 15 16 # Think of this as a defer func() in golang 17 METRIC_EMIT_PID="-1" 18 function defer { 19 docker-compose -f ${COMPOSE_FILE} down || echo "unable to shutdown containers" # CI fails to stop all containers sometimes 20 if [ "$METRIC_EMIT_PID" != "-1" ]; then 21 echo "Kill metric emit process" 22 kill $METRIC_EMIT_PID 23 fi 24 } 25 trap defer EXIT 26 27 AGG_RESOLUTION=5s setup_single_m3db_node 28 29 function read_carbon { 30 target=$1 31 expected_val=$2 32 end=$(date +%s) 33 start=$(($end-1000)) 34 if [ "$3" != "" ]; then 35 start=$3 36 fi 37 if [ "$4" != "" ]; then 38 end=$4 39 fi 40 params=$5 41 if [ "$params" != "" ]; then 42 params="&${params}" 43 fi 44 45 RESPONSE=$(curl -sSfg "http://localhost:7201/api/v1/graphite/render?target=${target}&from=${start}&until=${end}${params}") 46 47 expr="last_non_null" 48 if [ "$6" != "" ]; then 49 expr="$6" 50 fi 51 52 if [ "$expr" = "last_non_null" ]; then 53 test "$(echo "$RESPONSE" | jq ".[0].datapoints | .[][0] | select(. != null)" | jq -s last)" = "$expected_val" 54 return $? 55 fi 56 57 if [ "$expr" = "max_non_null" ]; then 58 test "$(echo "$RESPONSE" | jq "[ .[0].datapoints | .[] | select(.[0] != null) | .[0] ] | max")" = "$expected_val" 59 return $? 60 fi 61 62 return 1 63 } 64 65 function wait_carbon_values_accumulated { 66 target=$1 67 expected_val=$2 68 end=$(date +%s) 69 start=$(($end-1000)) 70 RESPONSE=$(curl -sSfg "http://localhost:7201/api/v1/graphite/render?target=${target}&from=${start}&until=${end}") 71 test "$(echo "$RESPONSE" | jq "[ .[0].datapoints | .[][0] | select(. != null) ] | length")" -gt "$expected_val" 72 return $? 73 } 74 75 function find_carbon { 76 query=$1 77 expected_file=$2 78 RESPONSE=$(curl -sSg "http://localhost:7201/api/v1/graphite/metrics/find?query=${query}") 79 ACTUAL=$(echo $RESPONSE | jq '. | sort') 80 EXPECTED=$(cat $EXPECTED_PATH/$expected_file | jq '. | sort') 81 if [ "$ACTUAL" == "$EXPECTED" ] 82 then 83 return 0 84 fi 85 return 1 86 } 87 88 echo "Writing out a carbon metric that should use a min aggregation" 89 t=$(date +%s) 90 # 41 should win out here because min(42,41) == 41. Note that technically this test could 91 # behave incorrectly if the values end up in separate flushes due to the bufferPast 92 # configuration of the downsampler, but we set reasonable values for bufferPast so that 93 # should not happen. 94 echo "foo.min.aggregate.baz 41 $t" | nc 0.0.0.0 7204 95 echo "foo.min.aggregate.baz 42 $t" | nc 0.0.0.0 7204 96 echo "Attempting to read min aggregated carbon metric" 97 ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'foo.min.aggregate.baz' 41" 98 99 echo "Writing out a carbon metric that should not be aggregated" 100 t=$(date +%s) 101 # 43 should win out here because of M3DB's upsert semantics. While M3DB's upsert 102 # semantics are not always guaranteed, it is guaranteed for a minimum time window 103 # that is as large as bufferPast/bufferFuture which should be much more than enough 104 # time for these two commands to complete. 105 echo "foo.min.already-aggregated.baz 42 $t" | nc 0.0.0.0 7204 106 echo "foo.min.already-aggregated.baz 43 $t" | nc 0.0.0.0 7204 107 echo "Attempting to read unaggregated carbon metric" 108 ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'foo.min.already-aggregated.baz' 43" 109 110 echo "Writing out a carbon metric that should should use the default mean aggregation" 111 t=$(date +%s) 112 # Mean of 10 and 20 is 15. Same comment as the min aggregation test above. 113 echo "foo.min.catch-all.baz 10 $t" | nc 0.0.0.0 7204 114 echo "foo.min.catch-all.baz 20 $t" | nc 0.0.0.0 7204 115 echo "Attempting to read mean aggregated carbon metric" 116 ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'foo.min.catch-all.baz' 15" 117 118 # Test writing and reading IDs with colons in them. 119 t=$(date +%s) 120 echo "foo.bar:baz.qux 42 $t" | nc 0.0.0.0 7204 121 ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'foo.bar:*.*' 42" 122 123 # Test writing and reading IDs with a single element. 124 t=$(date +%s) 125 echo "quail 42 $t" | nc 0.0.0.0 7204 126 ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'quail' 42" 127 128 # Test using "**" in queries 129 t=$(date +%s) 130 echo "qux.pos1-a.pos2-0 1 $t" | nc 0.0.0.0 7204 131 echo "qux.pos1-a.pos2-1 1 $t" | nc 0.0.0.0 7204 132 echo "qux.pos1-b.pos2-0 1 $t" | nc 0.0.0.0 7204 133 echo "qux.pos1-b.pos2-1 1 $t" | nc 0.0.0.0 7204 134 echo "qux.pos1-c.pos2-0 1 $t" | nc 0.0.0.0 7204 135 echo "qux.pos1-c.pos2-1 1 $t" | nc 0.0.0.0 7204 136 ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'sum(qux**)' 6" 137 ATTEMPTS=2 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'sum(qux.pos1-a**)' 2" 138 ATTEMPTS=2 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'sum(**pos1-a**)' 2" 139 ATTEMPTS=2 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'sum(**pos2-1**)' 3" 140 ATTEMPTS=2 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'sum(**pos2-1)' 3" 141 142 # Test consolidateBy function correctly changes behavior of downsampling 143 # Send metric values 42 every 5 seconds 144 echo "Sending unaggregated carbon metrics to m3coordinator" 145 bash -c 'while true; do t=$(date +%s); echo "stat.already-aggregated.foo 42 $t" | nc 0.0.0.0 7204; sleep 5; done' & 146 147 # Track PID to kill on exit 148 METRIC_EMIT_PID="$!" 149 150 # Wait until there's at least four values accumulated 151 ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "wait_carbon_values_accumulated 'stat.already-aggregated.foo' 4" 152 153 # Now test the max datapoints behavior using max of four datapoints (4x 5s resolution = 20s) 154 end=$(date +%s) 155 start=$(($end-20)) 156 # 1. no max datapoints set, should not adjust number of datapoints coming back 157 ATTEMPTS=2 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'stat.already-aggregated.foo' 42 $start $end" 158 # 2. max datapoints with LTTB, should be an existing value (i.e. 42) 159 ATTEMPTS=2 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'stat.already-aggregated.foo' 42 $start $end 'maxDataPoints=2' 'max_non_null'" 160 # 3. max datapoints with consolidateBy(.., aggFunction), should be resized according to function 161 ATTEMPTS=2 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'consolidateBy(stat.already-aggregated.foo,\"sum\")' 84 $start $end 'maxDataPoints=2' 'max_non_null'" 162 163 # Test basic find cases 164 t=$(date +%s) 165 echo "a 0 $t" | nc 0.0.0.0 7204 166 echo "a.bar 0 $t" | nc 0.0.0.0 7204 167 echo "a.biz 0 $t" | nc 0.0.0.0 7204 168 echo "a.biz.cake 0 $t" | nc 0.0.0.0 7204 169 echo "a.bar.caw.daz 0 $t" | nc 0.0.0.0 7204 170 echo "a.bag 0 $t" | nc 0.0.0.0 7204 171 echo "c:bar.c:baz 0 $t" | nc 0.0.0.0 7204 172 173 # Test rewrite multiple dots 174 echo "d..bar.baz 0 $t" | nc 0.0.0.0 7204 175 echo "e.bar...baz 0 $t" | nc 0.0.0.0 7204 176 177 # Test rewrite leading or trailing dots 178 echo "..f.bar.baz 0 $t" | nc 0.0.0.0 7204 179 echo "g.bar.baz.. 0 $t" | nc 0.0.0.0 7204 180 181 # Test rewrite bad chars 182 echo "h.bar@@baz 0 $t" | nc 0.0.0.0 7204 183 echo "i.bar!!baz 0 $t" | nc 0.0.0.0 7204 184 185 ATTEMPTS=10 TIMEOUT=1 retry_with_backoff "find_carbon 'a*' a.json" 186 ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'a.b*' ab.json" 187 ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'a.ba[rg]' aba.json" 188 ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'a.b*.c*' abc.json" 189 ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'a.b*.caw.*' abcd.json" 190 ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'x' none.json" 191 ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'a.d' none.json" 192 ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon '*.*.*.*.*' none.json" 193 ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'c:*' cbar.json" 194 ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'c:bar.*' cbaz.json" 195 ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'd.bar.*' dbaz.json" 196 ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'e.bar.*' ebaz.json" 197 ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'f.bar.*' fbaz.json" 198 ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'g.bar.*' gbaz.json" 199 ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'h.bar*' hbarbaz.json" 200 ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'i.bar*' ibarbaz.json" 201 ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'a**.*' a_starstar_dot_star.json" 202 ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'a.**.*' a_dot_starstar_dot_star.json" 203 ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'qux**.*' qux_starstar_dot_star.json" 204 ATTEMPTS=2 TIMEOUT=1 retry_with_backoff "find_carbon 'qux.**.*' qux_dot_starstar_dot_star.json" 205 206 # Test find limits from config of matching max docs of 200 with: 207 # carbon: 208 # limitsFind: 209 # perQuery: 210 # maxFetchedDocs: 100 211 # requireExhaustive: false 212 t=$(date +%s) 213 for i in $(seq 1 200); do 214 echo "find.limits.perquery.maxdocs.series_${i} 42 $t" | nc 0.0.0.0 7204 215 done 216 217 # Check between 90 and 10 (won't be exact match since we're limiting by docs 218 # not by max fetched results). 219 # Note: First check that there's 200 series there by using count(). 220 ATTEMPTS=20 TIMEOUT=2 MAX_TIMEOUT=4 retry_with_backoff "read_carbon 'countSeries(find.limits.perquery.maxdocs.*)' 200" 221 ATTEMPTS=2 TIMEOUT=2 MAX_TIMEOUT=4 retry_with_backoff \ 222 '[[ $(curl -s localhost:7201/api/v1/graphite/metrics/find?query=find.limits.perquery.maxdocs.* | jq -r ". | length") -ge 90 ]]' 223 ATTEMPTS=2 TIMEOUT=2 MAX_TIMEOUT=4 retry_with_backoff \ 224 '[[ $(curl -s localhost:7201/api/v1/graphite/metrics/find?query=find.limits.perquery.maxdocs.* | jq -r ". | length") -le 110 ]]'