github.com/muhammadn/cortex@v1.9.1-0.20220510110439-46bb7000d03d/tools/querytee/response_comparator_test.go (about) 1 package querytee 2 3 import ( 4 "encoding/json" 5 "errors" 6 "testing" 7 8 "github.com/stretchr/testify/require" 9 ) 10 11 func TestCompareMatrix(t *testing.T) { 12 for _, tc := range []struct { 13 name string 14 expected json.RawMessage 15 actual json.RawMessage 16 err error 17 }{ 18 { 19 name: "no metrics", 20 expected: json.RawMessage(`[]`), 21 actual: json.RawMessage(`[]`), 22 }, 23 { 24 name: "no metrics in actual response", 25 expected: json.RawMessage(`[ 26 {"metric":{"foo":"bar"},"values":[[1,"1"]]} 27 ]`), 28 actual: json.RawMessage(`[]`), 29 err: errors.New("expected 1 metrics but got 0"), 30 }, 31 { 32 name: "extra metric in actual response", 33 expected: json.RawMessage(`[ 34 {"metric":{"foo":"bar"},"values":[[1,"1"]]} 35 ]`), 36 actual: json.RawMessage(`[ 37 {"metric":{"foo":"bar"},"values":[[1,"1"]]}, 38 {"metric":{"foo1":"bar1"},"values":[[1,"1"]]} 39 ]`), 40 err: errors.New("expected 1 metrics but got 2"), 41 }, 42 { 43 name: "same number of metrics but with different labels", 44 expected: json.RawMessage(`[ 45 {"metric":{"foo":"bar"},"values":[[1,"1"]]} 46 ]`), 47 actual: json.RawMessage(`[ 48 {"metric":{"foo1":"bar1"},"values":[[1,"1"]]} 49 ]`), 50 err: errors.New("expected metric {foo=\"bar\"} missing from actual response"), 51 }, 52 { 53 name: "difference in number of samples", 54 expected: json.RawMessage(`[ 55 {"metric":{"foo":"bar"},"values":[[1,"1"],[2,"2"]]} 56 ]`), 57 actual: json.RawMessage(`[ 58 {"metric":{"foo":"bar"},"values":[[1,"1"]]} 59 ]`), 60 err: errors.New("expected 2 samples for metric {foo=\"bar\"} but got 1"), 61 }, 62 { 63 name: "difference in sample timestamp", 64 expected: json.RawMessage(`[ 65 {"metric":{"foo":"bar"},"values":[[1,"1"],[2,"2"]]} 66 ]`), 67 actual: json.RawMessage(`[ 68 {"metric":{"foo":"bar"},"values":[[1,"1"],[3,"2"]]} 69 ]`), 70 // timestamps are parsed from seconds to ms which are then added to errors as is so adding 3 0s to expected error. 71 err: errors.New("sample pair not matching for metric {foo=\"bar\"}: expected timestamp 2 but got 3"), 72 }, 73 { 74 name: "difference in sample value", 75 expected: json.RawMessage(`[ 76 {"metric":{"foo":"bar"},"values":[[1,"1"],[2,"2"]]} 77 ]`), 78 actual: json.RawMessage(`[ 79 {"metric":{"foo":"bar"},"values":[[1,"1"],[2,"3"]]} 80 ]`), 81 err: errors.New("sample pair not matching for metric {foo=\"bar\"}: expected value 2 for timestamp 2 but got 3"), 82 }, 83 { 84 name: "correct samples", 85 expected: json.RawMessage(`[ 86 {"metric":{"foo":"bar"},"values":[[1,"1"],[2,"2"]]} 87 ]`), 88 actual: json.RawMessage(`[ 89 {"metric":{"foo":"bar"},"values":[[1,"1"],[2,"2"]]} 90 ]`), 91 }, 92 } { 93 t.Run(tc.name, func(t *testing.T) { 94 err := compareMatrix(tc.expected, tc.actual, 0) 95 if tc.err == nil { 96 require.NoError(t, err) 97 return 98 } 99 require.Error(t, err) 100 require.Equal(t, tc.err.Error(), err.Error()) 101 }) 102 } 103 } 104 105 func TestCompareVector(t *testing.T) { 106 for _, tc := range []struct { 107 name string 108 expected json.RawMessage 109 actual json.RawMessage 110 err error 111 }{ 112 { 113 name: "no metrics", 114 expected: json.RawMessage(`[]`), 115 actual: json.RawMessage(`[]`), 116 }, 117 { 118 name: "no metrics in actual response", 119 expected: json.RawMessage(`[ 120 {"metric":{"foo":"bar"},"value":[1,"1"]} 121 ]`), 122 actual: json.RawMessage(`[]`), 123 err: errors.New("expected 1 metrics but got 0"), 124 }, 125 { 126 name: "extra metric in actual response", 127 expected: json.RawMessage(`[ 128 {"metric":{"foo":"bar"},"value":[1,"1"]} 129 ]`), 130 actual: json.RawMessage(`[ 131 {"metric":{"foo":"bar"},"value":[1,"1"]}, 132 {"metric":{"foo1":"bar1"},"value":[1,"1"]} 133 ]`), 134 err: errors.New("expected 1 metrics but got 2"), 135 }, 136 { 137 name: "same number of metrics but with different labels", 138 expected: json.RawMessage(`[ 139 {"metric":{"foo":"bar"},"value":[1,"1"]} 140 ]`), 141 actual: json.RawMessage(`[ 142 {"metric":{"foo1":"bar1"},"value":[1,"1"]} 143 ]`), 144 err: errors.New("expected metric {foo=\"bar\"} missing from actual response"), 145 }, 146 { 147 name: "difference in sample timestamp", 148 expected: json.RawMessage(`[ 149 {"metric":{"foo":"bar"},"value":[1,"1"]} 150 ]`), 151 actual: json.RawMessage(`[ 152 {"metric":{"foo":"bar"},"value":[2,"1"]} 153 ]`), 154 err: errors.New("sample pair not matching for metric {foo=\"bar\"}: expected timestamp 1 but got 2"), 155 }, 156 { 157 name: "difference in sample value", 158 expected: json.RawMessage(`[ 159 {"metric":{"foo":"bar"},"value":[1,"1"]} 160 ]`), 161 actual: json.RawMessage(`[ 162 {"metric":{"foo":"bar"},"value":[1,"2"]} 163 ]`), 164 err: errors.New("sample pair not matching for metric {foo=\"bar\"}: expected value 1 for timestamp 1 but got 2"), 165 }, 166 { 167 name: "correct samples", 168 expected: json.RawMessage(`[ 169 {"metric":{"foo":"bar"},"value":[1,"1"]} 170 ]`), 171 actual: json.RawMessage(`[ 172 {"metric":{"foo":"bar"},"value":[1,"1"]} 173 ]`), 174 }, 175 } { 176 t.Run(tc.name, func(t *testing.T) { 177 err := compareVector(tc.expected, tc.actual, 0) 178 if tc.err == nil { 179 require.NoError(t, err) 180 return 181 } 182 require.Error(t, err) 183 require.Equal(t, tc.err.Error(), err.Error()) 184 }) 185 } 186 } 187 188 func TestCompareScalar(t *testing.T) { 189 for _, tc := range []struct { 190 name string 191 expected json.RawMessage 192 actual json.RawMessage 193 err error 194 }{ 195 { 196 name: "difference in timestamp", 197 expected: json.RawMessage(`[1,"1"]`), 198 actual: json.RawMessage(`[2,"1"]`), 199 err: errors.New("expected timestamp 1 but got 2"), 200 }, 201 { 202 name: "difference in value", 203 expected: json.RawMessage(`[1,"1"]`), 204 actual: json.RawMessage(`[1,"2"]`), 205 err: errors.New("expected value 1 for timestamp 1 but got 2"), 206 }, 207 { 208 name: "correct values", 209 expected: json.RawMessage(`[1,"1"]`), 210 actual: json.RawMessage(`[1,"1"]`), 211 }, 212 } { 213 t.Run(tc.name, func(t *testing.T) { 214 err := compareScalar(tc.expected, tc.actual, 0) 215 if tc.err == nil { 216 require.NoError(t, err) 217 return 218 } 219 require.Error(t, err) 220 require.Equal(t, tc.err.Error(), err.Error()) 221 }) 222 } 223 } 224 225 func TestCompareSamplesResponse(t *testing.T) { 226 for _, tc := range []struct { 227 name string 228 tolerance float64 229 expected json.RawMessage 230 actual json.RawMessage 231 err error 232 }{ 233 { 234 name: "difference in response status", 235 expected: json.RawMessage(`{ 236 "status": "success", 237 "data": {"resultType":"scalar","result":[1,"1"]} 238 }`), 239 actual: json.RawMessage(`{ 240 "status": "fail" 241 }`), 242 err: errors.New("expected status success but got fail"), 243 }, 244 { 245 name: "difference in resultType", 246 expected: json.RawMessage(`{ 247 "status": "success", 248 "data": {"resultType":"scalar","result":[1,"1"]} 249 }`), 250 actual: json.RawMessage(`{ 251 "status": "success", 252 "data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[1,"1"]}]} 253 }`), 254 err: errors.New("expected resultType scalar but got vector"), 255 }, 256 { 257 name: "unregistered resultType", 258 expected: json.RawMessage(`{ 259 "status": "success", 260 "data": {"resultType":"new-scalar","result":[1,"1"]} 261 }`), 262 actual: json.RawMessage(`{ 263 "status": "success", 264 "data": {"resultType":"new-scalar","result":[1,"1"]} 265 }`), 266 err: errors.New("resultType new-scalar not registered for comparison"), 267 }, 268 { 269 name: "valid scalar response", 270 expected: json.RawMessage(`{ 271 "status": "success", 272 "data": {"resultType":"scalar","result":[1,"1"]} 273 }`), 274 actual: json.RawMessage(`{ 275 "status": "success", 276 "data": {"resultType":"scalar","result":[1,"1"]} 277 }`), 278 }, 279 { 280 name: "should pass if values are slightly different but within the tolerance", 281 tolerance: 0.000001, 282 expected: json.RawMessage(`{ 283 "status": "success", 284 "data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[1,"773054.5916666666"]}]} 285 }`), 286 actual: json.RawMessage(`{ 287 "status": "success", 288 "data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[1,"773054.59166667"]}]} 289 }`), 290 }, 291 { 292 name: "should correctly compare NaN values with tolerance is disabled", 293 tolerance: 0, 294 expected: json.RawMessage(`{ 295 "status": "success", 296 "data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[1,"NaN"]}]} 297 }`), 298 actual: json.RawMessage(`{ 299 "status": "success", 300 "data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[1,"NaN"]}]} 301 }`), 302 }, 303 { 304 name: "should correctly compare NaN values with tolerance is enabled", 305 tolerance: 0.000001, 306 expected: json.RawMessage(`{ 307 "status": "success", 308 "data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[1,"NaN"]}]} 309 }`), 310 actual: json.RawMessage(`{ 311 "status": "success", 312 "data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[1,"NaN"]}]} 313 }`), 314 }, 315 { 316 name: "should fail if values are significantly different, over the tolerance", 317 tolerance: 0.000001, 318 expected: json.RawMessage(`{ 319 "status": "success", 320 "data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[1,"773054.5916666666"]}]} 321 }`), 322 actual: json.RawMessage(`{ 323 "status": "success", 324 "data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[1,"773054.789"]}]} 325 }`), 326 err: errors.New(`sample pair not matching for metric {foo="bar"}: expected value 773054.5916666666 for timestamp 1 but got 773054.789`), 327 }, 328 } { 329 t.Run(tc.name, func(t *testing.T) { 330 samplesComparator := NewSamplesComparator(tc.tolerance) 331 err := samplesComparator.Compare(tc.expected, tc.actual) 332 if tc.err == nil { 333 require.NoError(t, err) 334 return 335 } 336 require.Error(t, err) 337 require.Equal(t, tc.err.Error(), err.Error()) 338 }) 339 } 340 }