vitess.io/vitess@v0.16.2/go/vt/vtgate/vindexes/cfc_test.go (about) 1 /* 2 Copyright 2021 The Vitess Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package vindexes 18 19 import ( 20 "context" 21 "testing" 22 23 "github.com/stretchr/testify/assert" 24 "github.com/stretchr/testify/require" 25 26 "vitess.io/vitess/go/sqltypes" 27 "vitess.io/vitess/go/vt/key" 28 topodatapb "vitess.io/vitess/go/vt/proto/topodata" 29 "vitess.io/vitess/go/vt/proto/vtrpc" 30 "vitess.io/vitess/go/vt/vterrors" 31 ) 32 33 func assertEqualVtError(t *testing.T, expected, actual error) { 34 // vterrors.Errorf returns a struct containing a stacktrace, which fails 35 // assert.EqualError since the stacktrace would be guaranteed to be different. 36 // so just check the error message 37 if expected == nil { 38 assert.NoError(t, actual) 39 } else { 40 assert.EqualError(t, actual, expected.Error()) 41 } 42 } 43 44 func TestCFCBuildCFC(t *testing.T) { 45 cases := []struct { 46 testName string 47 params map[string]string 48 err error 49 offsets []int 50 }{ 51 { 52 testName: "no params", 53 }, 54 { 55 testName: "no hash", 56 params: map[string]string{}, 57 }, 58 { 59 testName: "no hash", 60 params: map[string]string{"offsets": "[1,2]"}, 61 }, 62 { 63 testName: "no offsets", 64 params: map[string]string{"hash": "md5"}, 65 err: vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "CFC vindex requires offsets when hash is defined"), 66 }, 67 { 68 testName: "invalid offset", 69 params: map[string]string{"hash": "md5", "offsets": "10,12"}, 70 err: vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "invalid offsets 10,12 to CFC vindex cfc. expected sorted positive ints in brackets"), 71 }, 72 { 73 testName: "invalid offset", 74 params: map[string]string{"hash": "md5", "offsets": "xxx"}, 75 err: vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "invalid offsets xxx to CFC vindex cfc. expected sorted positive ints in brackets"), 76 }, 77 { 78 testName: "empty offsets", 79 params: map[string]string{"hash": "md5", "offsets": "[]"}, 80 err: vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "invalid offsets [] to CFC vindex cfc. expected sorted positive ints in brackets"), 81 }, 82 { 83 testName: "unsorted offsets", 84 params: map[string]string{"hash": "md5", "offsets": "[10,3]"}, 85 err: vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "invalid offsets [10,3] to CFC vindex cfc. expected sorted positive ints in brackets"), 86 }, 87 { 88 testName: "negative offsets", 89 params: map[string]string{"hash": "md5", "offsets": "[-1,3]"}, 90 err: vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "invalid offsets [-1,3] to CFC vindex cfc. expected sorted positive ints in brackets"), 91 }, 92 { 93 testName: "normal", 94 params: map[string]string{"hash": "md5", "offsets": "[3, 7]"}, 95 offsets: []int{3, 7}, 96 }, 97 { 98 testName: "duplicated offsets", 99 params: map[string]string{"hash": "md5", "offsets": "[4,4,6]"}, 100 err: vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "invalid offsets [4,4,6] to CFC vindex cfc. expected sorted positive ints in brackets"), 101 }, 102 } 103 104 for _, tc := range cases { 105 t.Run(tc.testName, func(t *testing.T) { 106 cfc, err := NewCFC("cfc", tc.params) 107 assertEqualVtError(t, tc.err, err) 108 if cfc != nil { 109 assert.EqualValues(t, tc.offsets, cfc.(*CFC).offsets) 110 assert.Equal(t, "cfc", cfc.String()) 111 assert.Equal(t, 1, cfc.Cost()) 112 assert.Equal(t, true, cfc.IsUnique()) 113 assert.Equal(t, false, cfc.NeedsVCursor()) 114 } 115 }) 116 } 117 } 118 119 func makeCFC(t *testing.T, params map[string]string) *CFC { 120 vind, err := NewCFC("cfc", params) 121 require.NoError(t, err) 122 cfc, ok := vind.(*CFC) 123 require.True(t, ok) 124 return cfc 125 } 126 127 func expectedHash(id [][]byte) (res []byte) { 128 for _, c := range id { 129 res = append(res, md5hash(c)...) 130 } 131 return res 132 } 133 134 func flattenKey(id [][]byte) (res []byte) { 135 for _, c := range id { 136 res = append(res, c...) 137 } 138 return res 139 } 140 141 func TestCFCComputeKsidNoHash(t *testing.T) { 142 cfc := makeCFC(t, nil) 143 id := []byte{3, 6, 20, 7, 60, 1} 144 ksid, err := cfc.computeKsid(id, true) 145 assert.NoError(t, err) 146 assert.EqualValues(t, id, ksid) 147 } 148 149 func TestCFCComputeKsid(t *testing.T) { 150 cfc := makeCFC(t, map[string]string{"hash": "md5", "offsets": "[3,5]"}) 151 152 cases := []struct { 153 testName string 154 id [][]byte 155 prefix bool 156 expected []byte 157 err error 158 }{ 159 { 160 testName: "nonprefix too short", 161 id: [][]byte{{3, 4, 5}}, 162 prefix: false, 163 expected: nil, 164 err: vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "insufficient size for cfc vindex cfc. need 5, got 3"), 165 }, 166 { 167 testName: "prefix ok", 168 id: [][]byte{{3, 4, 5}}, 169 prefix: true, 170 expected: expectedHash([][]byte{{3, 4, 5}}), 171 err: nil, 172 }, 173 { 174 testName: "misaligned prefix", 175 id: [][]byte{{3, 4, 5}, {1}}, 176 prefix: true, 177 // use the first component that's availabe 178 expected: expectedHash([][]byte{{3, 4, 5}}), 179 err: nil, 180 }, 181 { 182 testName: "misaligned prefix", 183 id: [][]byte{{3, 4}}, 184 prefix: true, 185 // use the first component that's availabe 186 expected: nil, 187 err: nil, 188 }, 189 { 190 testName: "normal", 191 id: [][]byte{{12, 234, 2}, {7, 1}, {6}}, 192 prefix: false, 193 expected: expectedHash([][]byte{{12, 234, 2}, {7, 1}, {6}}), 194 err: nil, 195 }, 196 { 197 testName: "normal", 198 id: [][]byte{{5, 21, 124}, {75, 5}}, 199 prefix: true, 200 expected: expectedHash([][]byte{{5, 21, 124}, {75, 5}}), 201 err: nil, 202 }, 203 { 204 testName: "long key", 205 id: [][]byte{{5, 21, 124}, {75, 5}, {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18}}, 206 prefix: true, 207 expected: expectedHash([][]byte{{5, 21, 124}, {75, 5}, {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18}}), 208 err: nil, 209 }, 210 { 211 testName: "empty key", 212 id: [][]byte{}, 213 prefix: true, 214 expected: nil, 215 err: nil, 216 }, 217 } 218 for _, tc := range cases { 219 t.Run(tc.testName, func(t *testing.T) { 220 fid := flattenKey(tc.id) 221 ksid, err := cfc.computeKsid(fid, tc.prefix) 222 assertEqualVtError(t, tc.err, err) 223 if err == nil { 224 assert.EqualValues(t, tc.expected, ksid) 225 } 226 }) 227 } 228 229 } 230 231 func TestCFCComputeKsidXxhash(t *testing.T) { 232 cfc := makeCFC(t, map[string]string{"hash": "xxhash64", "offsets": "[3,5]"}) 233 234 expectedHashXX := func(ids [][]byte) (res []byte) { 235 for _, c := range ids { 236 res = append(res, xxhash64(c)...) 237 } 238 return res 239 } 240 cases := []struct { 241 testName string 242 id [][]byte 243 prefix bool 244 expected []byte 245 err error 246 }{ 247 { 248 testName: "nonprefix too short", 249 id: [][]byte{{3, 4, 5}}, 250 prefix: false, 251 expected: nil, 252 err: vterrors.Errorf(vtrpc.Code_INVALID_ARGUMENT, "insufficient size for cfc vindex cfc. need 5, got 3"), 253 }, 254 { 255 testName: "prefix ok", 256 id: [][]byte{{3, 4, 5}}, 257 prefix: true, 258 expected: expectedHashXX([][]byte{{3, 4, 5}}), 259 err: nil, 260 }, 261 { 262 testName: "misaligned prefix", 263 id: [][]byte{{3, 4, 5}, {1}}, 264 prefix: true, 265 // use the first component that's availabe 266 expected: expectedHashXX([][]byte{{3, 4, 5}}), 267 err: nil, 268 }, 269 { 270 testName: "misaligned prefix", 271 id: [][]byte{{3, 4}}, 272 prefix: true, 273 // use the first component that's availabe 274 expected: nil, 275 err: nil, 276 }, 277 { 278 testName: "normal", 279 id: [][]byte{{12, 234, 2}, {7, 1}, {6}}, 280 prefix: false, 281 expected: expectedHashXX([][]byte{{12, 234, 2}, {7, 1}, {6}}), 282 err: nil, 283 }, 284 { 285 testName: "long key", 286 id: [][]byte{{5, 21, 124}, {75, 5}, {1, 2, 3, 4, 5, 6}}, 287 prefix: true, 288 expected: expectedHashXX([][]byte{{5, 21, 124}, {75, 5}, {1, 2, 3, 4, 5, 6}}), 289 err: nil, 290 }, 291 { 292 testName: "long key", 293 id: [][]byte{}, 294 prefix: true, 295 expected: nil, 296 err: nil, 297 }, 298 } 299 for _, tc := range cases { 300 t.Run(tc.testName, func(t *testing.T) { 301 fid := flattenKey(tc.id) 302 ksid, err := cfc.computeKsid(fid, tc.prefix) 303 assertEqualVtError(t, tc.err, err) 304 if err == nil { 305 assert.EqualValues(t, tc.expected, ksid) 306 } 307 }) 308 } 309 310 } 311 312 func TestCFCVerifyNoHash(t *testing.T) { 313 cfc := makeCFC(t, nil) 314 id := []byte{3, 10, 7, 200} 315 out, err := cfc.Verify(context.Background(), nil, []sqltypes.Value{sqltypes.NewVarBinary(string(id))}, [][]byte{id}) 316 assert.NoError(t, err) 317 assert.EqualValues(t, []bool{true}, out) 318 319 out, err = cfc.Verify(context.Background(), nil, []sqltypes.Value{sqltypes.NewVarBinary("foobar")}, [][]byte{id}) 320 assert.NoError(t, err) 321 assert.EqualValues(t, []bool{false}, out) 322 pcfc := cfc.PrefixVindex() 323 out, err = pcfc.Verify(context.Background(), nil, []sqltypes.Value{sqltypes.NewVarBinary("foobar")}, [][]byte{id}) 324 assert.NoError(t, err) 325 assert.EqualValues(t, []bool{false}, out) 326 } 327 328 func TestCFCVerifyWithHash(t *testing.T) { 329 cfc := makeCFC(t, map[string]string{"hash": "md5", "offsets": "[3,5]"}) 330 id := [][]byte{ 331 {1, 234, 3}, {12, 32}, {7, 9}, 332 } 333 out, err := cfc.Verify(context.Background(), nil, []sqltypes.Value{sqltypes.NewVarBinary(string(flattenKey(id)))}, [][]byte{expectedHash(id)}) 334 assert.NoError(t, err) 335 assert.EqualValues(t, []bool{true}, out) 336 337 _, err = cfc.Verify(context.Background(), nil, []sqltypes.Value{sqltypes.NewVarBinary("foo")}, [][]byte{expectedHash(id)}) 338 assert.Error(t, err) 339 340 pcfc := cfc.PrefixVindex() 341 out, err = pcfc.Verify(context.Background(), nil, []sqltypes.Value{sqltypes.NewVarBinary(string(flattenKey(id)))}, [][]byte{expectedHash(id)}) 342 assert.NoError(t, err) 343 assert.EqualValues(t, []bool{true}, out) 344 345 } 346 347 func TestCFCMap(t *testing.T) { 348 cfc := makeCFC(t, map[string]string{"hash": "md5", "offsets": "[3,5]"}) 349 _, err := cfc.Map(context.Background(), nil, []sqltypes.Value{sqltypes.NewVarBinary("abc")}) 350 assert.EqualError(t, err, "insufficient size for cfc vindex cfc. need 5, got 3") 351 352 dests, err := cfc.Map(context.Background(), nil, []sqltypes.Value{sqltypes.NewVarBinary("12345567")}) 353 require.NoError(t, err) 354 ksid, ok := dests[0].(key.DestinationKeyspaceID) 355 require.True(t, ok) 356 out, err := cfc.Verify(context.Background(), nil, []sqltypes.Value{sqltypes.NewVarBinary("12345567")}, [][]byte{ksid}) 357 assert.NoError(t, err) 358 assert.EqualValues(t, []bool{true}, out) 359 } 360 361 func TestCFCBuildPrefix(t *testing.T) { 362 cfc := makeCFC(t, nil) 363 prefixcfc := cfc.PrefixVindex() 364 assert.Equal(t, "cfc", prefixcfc.String()) 365 assert.False(t, prefixcfc.IsUnique()) 366 assert.False(t, prefixcfc.NeedsVCursor()) 367 assert.Equal(t, 2, prefixcfc.Cost()) 368 369 cfc = makeCFC(t, map[string]string{"offsets": "[1,2,3]", "hash": "xxhash64"}) 370 prefixcfc = cfc.PrefixVindex() 371 assert.Equal(t, "cfc", prefixcfc.String()) 372 assert.False(t, prefixcfc.IsUnique()) 373 assert.False(t, prefixcfc.NeedsVCursor()) 374 assert.Equal(t, 3, prefixcfc.Cost()) 375 } 376 377 func TestCFCPrefixMap(t *testing.T) { 378 cfc := makeCFC(t, map[string]string{"hash": "md5", "offsets": "[3,5]"}) 379 prefixcfc := cfc.PrefixVindex() 380 381 cases := []struct { 382 testName string 383 id string 384 dest key.Destination 385 }{ 386 { 387 testName: "literal regular", 388 id: "abcdef", 389 dest: NewKeyRangeFromPrefix(expectedHash([][]byte{{'a', 'b', 'c'}, {'d', 'e'}, {'f'}})), 390 }, 391 { 392 testName: "literal use first component", 393 id: "abcd", 394 dest: NewKeyRangeFromPrefix(expectedHash([][]byte{{'a', 'b', 'c'}})), 395 }, 396 { 397 testName: "literal prefix too short", 398 id: "ab", 399 dest: key.DestinationAllShards{}, 400 }, 401 } 402 for _, tc := range cases { 403 t.Run(tc.testName, func(t *testing.T) { 404 dests, err := prefixcfc.Map(context.Background(), nil, []sqltypes.Value{sqltypes.NewVarBinary(tc.id)}) 405 require.NoError(t, err) 406 assert.EqualValues(t, tc.dest, dests[0]) 407 }) 408 } 409 410 } 411 412 func TestCFCPrefixQueryMapNoHash(t *testing.T) { 413 cfc := makeCFC(t, nil) 414 prefixcfc := cfc.PrefixVindex() 415 416 expected := []struct { 417 start, end []byte 418 }{ 419 {[]byte{3, 123, 255}, []byte{3, 124, 0}}, 420 {[]byte{3, 123, 255, 6, 7}, []byte{3, 123, 255, 6, 8}}, 421 {[]byte{255, 255, 255}, nil}, 422 } 423 var ids []sqltypes.Value 424 for _, exp := range expected { 425 ids = append(ids, sqltypes.NewVarBinary(string(exp.start)+"%")) 426 } 427 dests, err := prefixcfc.Map(context.Background(), nil, ids) 428 require.NoError(t, err) 429 430 for i, dest := range dests { 431 kr, ok := dest.(key.DestinationKeyRange) 432 require.True(t, ok) 433 assert.EqualValues(t, expected[i].start, kr.KeyRange.Start) 434 assert.EqualValues(t, expected[i].end, kr.KeyRange.End) 435 } 436 } 437 438 func TestCFCFindPrefixEscape(t *testing.T) { 439 cases := []struct { 440 str, prefix string 441 }{ 442 { 443 str: "ab%", 444 prefix: "ab", 445 }, 446 { 447 str: "abc", 448 prefix: "abc", 449 }, 450 { 451 str: "a%%", 452 prefix: "a", 453 }, 454 { 455 str: "%ab", 456 prefix: "", 457 }, 458 { 459 str: `\%a%`, 460 prefix: `%a`, 461 }, 462 { 463 str: `\%%`, 464 prefix: `%`, 465 }, 466 { 467 str: `\%`, 468 prefix: `%`, 469 }, 470 { 471 str: `\\%a`, 472 prefix: `\`, 473 }, 474 { 475 str: `a\\%a`, 476 prefix: `a\`, 477 }, 478 { 479 str: `\\\%`, 480 prefix: `\%`, 481 }, 482 { 483 str: `\\\%a%`, 484 prefix: `\%a`, 485 }, 486 { 487 str: `\_\\\%a_`, 488 prefix: `_\%a`, 489 }, 490 { 491 str: `_%a%`, 492 prefix: ``, 493 }, 494 { 495 str: `\i\j\k`, 496 prefix: `ijk`, 497 }, 498 { 499 str: `\0\'\"\b\n\r\t\Z\\`, 500 prefix: "\x00'\"\b\n\r\t\x1A\\", 501 }, 502 { 503 str: `\\0\\'\\"\\b\\n\\r\\t\\Z`, 504 prefix: `\0\'\"\b\n\r\t\Z`, 505 }, 506 { 507 str: `\`, 508 prefix: `\`, 509 }, 510 } 511 512 for _, tc := range cases { 513 assert.EqualValues(t, tc.prefix, string(findPrefix([]byte(tc.str)))) 514 } 515 516 } 517 518 func TestDestinationKeyRangeFromPrefix(t *testing.T) { 519 testCases := []struct { 520 start []byte 521 dest key.Destination 522 }{ 523 { 524 start: []byte{3, 123, 255}, 525 dest: key.DestinationKeyRange{ 526 KeyRange: &topodatapb.KeyRange{ 527 Start: []byte{3, 123, 255}, 528 End: []byte{3, 124, 0}, 529 }, 530 }, 531 }, 532 { 533 start: []byte{3, 123, 255, 6, 7}, 534 dest: key.DestinationKeyRange{ 535 KeyRange: &topodatapb.KeyRange{ 536 Start: []byte{3, 123, 255, 6, 7}, 537 End: []byte{3, 123, 255, 6, 8}, 538 }, 539 }, 540 }, 541 { 542 start: []byte{255, 255, 255}, 543 dest: key.DestinationKeyRange{ 544 KeyRange: &topodatapb.KeyRange{ 545 Start: []byte{255, 255, 255}, 546 End: nil, 547 }, 548 }, 549 }, 550 { 551 start: nil, 552 dest: key.DestinationAllShards{}, 553 }, 554 { 555 start: []byte{}, 556 dest: key.DestinationAllShards{}, 557 }, 558 } 559 560 for _, tc := range testCases { 561 dest := NewKeyRangeFromPrefix(tc.start) 562 assert.EqualValues(t, tc.dest, dest) 563 } 564 565 t.Run("add one to bytes", func(t *testing.T) { 566 cases := []struct { 567 testName string 568 input, expected []byte 569 }{ 570 { 571 testName: "regular one byte", 572 input: []byte{213}, 573 expected: []byte{214}, 574 }, 575 { 576 testName: "regular two byte", 577 input: []byte{213, 7}, 578 expected: []byte{213, 8}, 579 }, 580 { 581 testName: "carry one", 582 input: []byte{213, 255}, 583 expected: []byte{214, 0}, 584 }, 585 { 586 testName: "carry multi", 587 input: []byte{1, 255, 255}, 588 expected: []byte{2, 0, 0}, 589 }, 590 { 591 testName: "overflow one byte", 592 input: []byte{255}, 593 expected: nil, 594 }, 595 { 596 testName: "overflow multi", 597 input: []byte{255, 255}, 598 expected: nil, 599 }, 600 } 601 602 for _, tc := range cases { 603 t.Run(tc.testName, func(t *testing.T) { 604 assert.EqualValues(t, tc.expected, addOne(tc.input)) 605 }) 606 } 607 }) 608 } 609 610 func TestCFCHashFunction(t *testing.T) { 611 cases := []struct { 612 src string 613 outMD5, outXXHash int 614 }{ 615 {"asdf", 4, 4}, 616 {"abcdefgh", 8, 8}, 617 {"abcdefghijkl", 12, 8}, 618 {"abcdefghijklmnop", 16, 8}, 619 {"abcdefghijklmnopqrst", 16, 8}, 620 } 621 for _, c := range cases { 622 assert.Equal(t, c.outMD5, len(md5hash([]byte(c.src)))) 623 assert.Equal(t, c.outXXHash, len(xxhash64([]byte(c.src)))) 624 } 625 } 626 627 // TestCFCCache tests for CFC vindex, cache size can be calculated. 628 func TestCFCCache(t *testing.T) { 629 cfc := makeCFC(t, map[string]string{"hash": "md5", "offsets": "[3,5]"}) 630 // should be able to return. 631 _ = cfc.CachedSize(true) 632 }