github.com/matrixorigin/matrixone@v0.7.0/pkg/txn/storage/memorystorage/storage_test.go (about) 1 // Copyright 2022 Matrix Origin 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package memorystorage 16 17 import ( 18 "bytes" 19 "context" 20 "encoding/gob" 21 "github.com/matrixorigin/matrixone/pkg/pb/plan" 22 "testing" 23 "time" 24 25 "github.com/matrixorigin/matrixone/pkg/common/moerr" 26 "github.com/matrixorigin/matrixone/pkg/common/mpool" 27 "github.com/matrixorigin/matrixone/pkg/container/batch" 28 "github.com/matrixorigin/matrixone/pkg/container/types" 29 "github.com/matrixorigin/matrixone/pkg/container/vector" 30 "github.com/matrixorigin/matrixone/pkg/pb/timestamp" 31 "github.com/matrixorigin/matrixone/pkg/pb/txn" 32 "github.com/matrixorigin/matrixone/pkg/testutil" 33 "github.com/matrixorigin/matrixone/pkg/vm/engine" 34 "github.com/matrixorigin/matrixone/pkg/vm/engine/memoryengine" 35 "github.com/stretchr/testify/assert" 36 ) 37 38 func testDatabase( 39 t *testing.T, 40 newStorage func() (*Storage, error), 41 ) { 42 mp := mpool.MustNewZero() 43 ctx, cancel := context.WithTimeout(context.Background(), time.Hour) 44 defer cancel() 45 46 // new 47 s, err := newStorage() 48 assert.Nil(t, err) 49 defer s.Close(ctx) 50 51 // txn 52 txnMeta := txn.TxnMeta{ 53 ID: []byte("1"), 54 Status: txn.TxnStatus_Active, 55 SnapshotTS: timestamp.Timestamp{ 56 PhysicalTime: 1, 57 LogicalTime: 1, 58 }, 59 } 60 defer func() { 61 err := s.Commit(ctx, txnMeta) 62 assert.Nil(t, err) 63 }() 64 65 // open database 66 { 67 _, err := testRead[memoryengine.OpenDatabaseResp]( 68 ctx, t, s, txnMeta, 69 memoryengine.OpOpenDatabase, 70 memoryengine.OpenDatabaseReq{ 71 Name: "foo", 72 }, 73 ) 74 assert.True(t, moerr.IsMoErrCode(err, moerr.ErrNoDB)) 75 } 76 77 // create database 78 { 79 resp, err := testWrite[memoryengine.CreateDatabaseResp]( 80 ctx, t, s, txnMeta, 81 memoryengine.OpCreateDatabase, 82 memoryengine.CreateDatabaseReq{ 83 Name: "foo", 84 }, 85 ) 86 assert.Nil(t, err) 87 assert.NotEmpty(t, resp.ID) 88 } 89 90 // get databases 91 { 92 resp, err := testRead[memoryengine.GetDatabasesResp]( 93 ctx, t, s, txnMeta, 94 memoryengine.OpGetDatabases, 95 memoryengine.GetDatabasesReq{}, 96 ) 97 assert.Nil(t, err) 98 assert.Equal(t, 1, len(resp.Names)) 99 assert.Equal(t, "foo", resp.Names[0]) 100 } 101 102 // open database 103 var dbID ID 104 { 105 resp, err := testRead[memoryengine.OpenDatabaseResp]( 106 ctx, t, s, txnMeta, 107 memoryengine.OpOpenDatabase, 108 memoryengine.OpenDatabaseReq{ 109 Name: "foo", 110 }, 111 ) 112 assert.Nil(t, err) 113 assert.NotNil(t, resp.ID) 114 dbID = resp.ID 115 116 // delete database 117 defer func() { 118 { 119 resp, err := testWrite[memoryengine.DeleteDatabaseResp]( 120 ctx, t, s, txnMeta, 121 memoryengine.OpDeleteDatabase, 122 memoryengine.DeleteDatabaseReq{ 123 Name: "foo", 124 }, 125 ) 126 assert.Nil(t, err) 127 assert.NotEmpty(t, resp.ID) 128 } 129 { 130 resp, err := testRead[memoryengine.GetDatabasesResp]( 131 ctx, t, s, txnMeta, 132 memoryengine.OpGetDatabases, 133 memoryengine.GetDatabasesReq{}, 134 ) 135 assert.Nil(t, err) 136 for _, name := range resp.Names { 137 if name == "foo" { 138 t.Fatal() 139 } 140 } 141 } 142 }() 143 } 144 145 // open relation 146 { 147 _, err := testRead[memoryengine.OpenRelationResp]( 148 ctx, t, s, txnMeta, 149 memoryengine.OpOpenRelation, 150 memoryengine.OpenRelationReq{ 151 DatabaseID: dbID, 152 Name: "table", 153 }, 154 ) 155 assert.True(t, moerr.IsMoErrCode(err, moerr.ErrNoSuchTable)) 156 } 157 158 // create relation 159 { 160 resp, err := testWrite[memoryengine.CreateRelationResp]( 161 ctx, t, s, txnMeta, 162 memoryengine.OpCreateRelation, 163 memoryengine.CreateRelationReq{ 164 DatabaseID: dbID, 165 Name: "table", 166 Type: memoryengine.RelationTable, 167 Defs: []engine.TableDef{ 168 &engine.AttributeDef{ 169 Attr: engine.Attribute{ 170 Name: "a", 171 Type: types.T_int64.ToType(), 172 Primary: true, 173 }, 174 }, 175 &engine.AttributeDef{ 176 Attr: engine.Attribute{ 177 Name: "b", 178 Type: types.T_int64.ToType(), 179 Primary: false, 180 }, 181 }, 182 &engine.ConstraintDef{ 183 Cts: []engine.Constraint{ 184 &engine.PrimaryKeyDef{ 185 Pkey: &plan.PrimaryKeyDef{ 186 PkeyColName: "a", 187 Names: []string{"a"}, 188 }, 189 }, 190 }, 191 }, 192 }, 193 }, 194 ) 195 assert.Nil(t, err) 196 assert.NotEmpty(t, resp.ID) 197 } 198 199 // get relations 200 { 201 resp, err := testRead[memoryengine.GetRelationsResp]( 202 ctx, t, s, txnMeta, 203 memoryengine.OpGetRelations, 204 memoryengine.GetRelationsReq{ 205 DatabaseID: dbID, 206 }, 207 ) 208 assert.Nil(t, err) 209 assert.Equal(t, 1, len(resp.Names)) 210 assert.Equal(t, "table", resp.Names[0]) 211 } 212 213 // open relation 214 var relID ID 215 { 216 resp, err := testRead[memoryengine.OpenRelationResp]( 217 ctx, t, s, txnMeta, 218 memoryengine.OpOpenRelation, 219 memoryengine.OpenRelationReq{ 220 DatabaseID: dbID, 221 Name: "table", 222 }, 223 ) 224 assert.Nil(t, err) 225 assert.NotNil(t, resp.ID) 226 relID = resp.ID 227 assert.Equal(t, memoryengine.RelationTable, resp.Type) 228 } 229 _ = relID 230 231 // get relation defs 232 { 233 resp, err := testRead[memoryengine.GetTableDefsResp]( 234 ctx, t, s, txnMeta, 235 memoryengine.OpGetTableDefs, 236 memoryengine.GetTableDefsReq{ 237 TableID: relID, 238 }, 239 ) 240 assert.Nil(t, err) 241 assert.Equal(t, 3, len(resp.Defs)) 242 } 243 244 // write 245 { 246 colA := testutil.NewVector( 247 5, 248 types.T_int64.ToType(), 249 mp, 250 false, 251 []int64{ 252 1, 2, 3, 4, 5, 253 }, 254 ) 255 colB := testutil.NewVector( 256 5, 257 types.T_int64.ToType(), 258 mp, 259 false, 260 []int64{ 261 6, 7, 8, 9, 10, 262 }, 263 ) 264 bat := batch.New(false, []string{"a", "b"}) 265 bat.Vecs[0] = colA 266 bat.Vecs[1] = colB 267 bat.InitZsOne(5) 268 _, err := testWrite[memoryengine.WriteResp]( 269 ctx, t, s, txnMeta, 270 memoryengine.OpWrite, 271 memoryengine.WriteReq{ 272 TableID: relID, 273 Batch: bat, 274 }, 275 ) 276 assert.Nil(t, err) 277 } 278 279 // read 280 var iterID ID 281 { 282 resp, err := testRead[memoryengine.NewTableIterResp]( 283 ctx, t, s, txnMeta, 284 memoryengine.OpNewTableIter, 285 memoryengine.NewTableIterReq{ 286 TableID: relID, 287 }, 288 ) 289 assert.Nil(t, err) 290 assert.NotEmpty(t, resp.IterID) 291 iterID = resp.IterID 292 } 293 { 294 resp, err := testRead[memoryengine.ReadResp]( 295 ctx, t, s, txnMeta, 296 memoryengine.OpRead, 297 memoryengine.ReadReq{ 298 IterID: iterID, 299 ColNames: []string{"a", "b"}, 300 }, 301 ) 302 assert.Nil(t, err) 303 assert.NotNil(t, resp.Batch) 304 assert.Equal(t, 5, resp.Batch.Length()) 305 } 306 307 // delete by primary key 308 { 309 colA := testutil.NewVector( 310 1, 311 types.T_int64.ToType(), 312 mp, 313 false, 314 []int64{ 315 1, 316 }, 317 ) 318 _, err := testWrite[memoryengine.DeleteResp]( 319 ctx, t, s, txnMeta, 320 memoryengine.OpDelete, 321 memoryengine.DeleteReq{ 322 TableID: relID, 323 ColumnName: "a", 324 Vector: colA, 325 }, 326 ) 327 assert.Nil(t, err) 328 } 329 330 // read after delete 331 { 332 resp, err := testRead[memoryengine.NewTableIterResp]( 333 ctx, t, s, txnMeta, 334 memoryengine.OpNewTableIter, 335 memoryengine.NewTableIterReq{ 336 TableID: relID, 337 }, 338 ) 339 assert.Nil(t, err) 340 assert.NotEmpty(t, resp.IterID) 341 iterID = resp.IterID 342 } 343 { 344 resp, err := testRead[memoryengine.ReadResp]( 345 ctx, t, s, txnMeta, 346 memoryengine.OpRead, 347 memoryengine.ReadReq{ 348 IterID: iterID, 349 ColNames: []string{"a", "b"}, 350 }, 351 ) 352 assert.Nil(t, err) 353 assert.NotNil(t, resp.Batch) 354 assert.Equal(t, 4, resp.Batch.Length()) 355 } 356 357 // delete by non-primary key 358 { 359 colB := testutil.NewVector( 360 1, 361 types.T_int64.ToType(), 362 mp, 363 false, 364 []int64{ 365 8, 366 }, 367 ) 368 _, err := testWrite[memoryengine.DeleteResp]( 369 ctx, t, s, txnMeta, 370 memoryengine.OpDelete, 371 memoryengine.DeleteReq{ 372 TableID: relID, 373 ColumnName: "b", 374 Vector: colB, 375 }, 376 ) 377 assert.Nil(t, err) 378 } 379 380 // read after delete 381 { 382 resp, err := testRead[memoryengine.NewTableIterResp]( 383 ctx, t, s, txnMeta, 384 memoryengine.OpNewTableIter, 385 memoryengine.NewTableIterReq{ 386 TableID: relID, 387 }, 388 ) 389 assert.Nil(t, err) 390 assert.NotEmpty(t, resp.IterID) 391 iterID = resp.IterID 392 } 393 { 394 resp, err := testRead[memoryengine.ReadResp]( 395 ctx, t, s, txnMeta, 396 memoryengine.OpRead, 397 memoryengine.ReadReq{ 398 IterID: iterID, 399 ColNames: []string{"a", "b"}, 400 }, 401 ) 402 assert.Nil(t, err) 403 assert.NotNil(t, resp.Batch) 404 assert.Equal(t, 3, resp.Batch.Length()) 405 } 406 407 // write after delete 408 { 409 colA := testutil.NewVector( 410 1, 411 types.T_int64.ToType(), 412 mp, 413 false, 414 []int64{ 415 1, 416 }, 417 ) 418 colB := testutil.NewVector( 419 1, 420 types.T_int64.ToType(), 421 mp, 422 false, 423 []int64{ 424 6, 425 }, 426 ) 427 bat := batch.New(false, []string{"a", "b"}) 428 bat.Vecs[0] = colA 429 bat.Vecs[1] = colB 430 bat.InitZsOne(1) 431 _, err := testWrite[memoryengine.WriteResp]( 432 ctx, t, s, txnMeta, 433 memoryengine.OpWrite, 434 memoryengine.WriteReq{ 435 TableID: relID, 436 Batch: bat, 437 }, 438 ) 439 assert.Nil(t, err) 440 } 441 442 // delete relation 443 { 444 resp, err := testWrite[memoryengine.DeleteRelationResp]( 445 ctx, t, s, txnMeta, 446 memoryengine.OpDeleteRelation, 447 memoryengine.DeleteRelationReq{ 448 DatabaseID: dbID, 449 Name: "table", 450 }, 451 ) 452 assert.Nil(t, err) 453 assert.NotEmpty(t, resp.ID) 454 } 455 { 456 resp, err := testRead[memoryengine.GetRelationsResp]( 457 ctx, t, s, txnMeta, 458 memoryengine.OpGetRelations, 459 memoryengine.GetRelationsReq{ 460 DatabaseID: dbID, 461 }, 462 ) 463 assert.Nil(t, err) 464 assert.Equal(t, 0, len(resp.Names)) 465 } 466 467 // new relation without primary key 468 { 469 resp, err := testWrite[memoryengine.CreateRelationResp]( 470 ctx, t, s, txnMeta, 471 memoryengine.OpCreateRelation, 472 memoryengine.CreateRelationReq{ 473 DatabaseID: dbID, 474 Name: "table", 475 Type: memoryengine.RelationTable, 476 Defs: []engine.TableDef{ 477 &engine.AttributeDef{ 478 Attr: engine.Attribute{ 479 Name: "a", 480 Type: types.T_int64.ToType(), 481 }, 482 }, 483 &engine.AttributeDef{ 484 Attr: engine.Attribute{ 485 Name: "b", 486 Type: types.T_int64.ToType(), 487 }, 488 }, 489 }, 490 }, 491 ) 492 assert.Nil(t, err) 493 assert.NotEmpty(t, resp.ID) 494 relID = resp.ID 495 } 496 497 // write 498 { 499 colA := testutil.NewVector( 500 5, 501 types.T_int64.ToType(), 502 mp, 503 false, 504 []int64{ 505 1, 2, 3, 4, 5, 506 }, 507 ) 508 colB := testutil.NewVector( 509 5, 510 types.T_int64.ToType(), 511 mp, 512 false, 513 []int64{ 514 6, 7, 8, 9, 10, 515 }, 516 ) 517 bat := batch.New(false, []string{"a", "b"}) 518 bat.Vecs[0] = colA 519 bat.Vecs[1] = colB 520 bat.InitZsOne(5) 521 _, err := testWrite[memoryengine.WriteResp]( 522 ctx, t, s, txnMeta, 523 memoryengine.OpWrite, 524 memoryengine.WriteReq{ 525 TableID: relID, 526 Batch: bat, 527 }, 528 ) 529 assert.Nil(t, err) 530 } 531 532 // delete by primary key 533 { 534 colA := testutil.NewVector( 535 1, 536 types.T_int64.ToType(), 537 mp, 538 false, 539 []int64{ 540 1, 541 }, 542 ) 543 _, err := testWrite[memoryengine.DeleteResp]( 544 ctx, t, s, txnMeta, 545 memoryengine.OpDelete, 546 memoryengine.DeleteReq{ 547 TableID: relID, 548 ColumnName: "a", 549 Vector: colA, 550 }, 551 ) 552 assert.Nil(t, err) 553 } 554 555 // read after delete 556 { 557 resp, err := testRead[memoryengine.NewTableIterResp]( 558 ctx, t, s, txnMeta, 559 memoryengine.OpNewTableIter, 560 memoryengine.NewTableIterReq{ 561 TableID: relID, 562 }, 563 ) 564 assert.Nil(t, err) 565 assert.NotEmpty(t, resp.IterID) 566 iterID = resp.IterID 567 } 568 { 569 resp, err := testRead[memoryengine.ReadResp]( 570 ctx, t, s, txnMeta, 571 memoryengine.OpRead, 572 memoryengine.ReadReq{ 573 IterID: iterID, 574 ColNames: []string{"a", "b"}, 575 }, 576 ) 577 assert.Nil(t, err) 578 assert.NotNil(t, resp.Batch) 579 assert.Equal(t, 4, resp.Batch.Length()) 580 } 581 582 // delete by non-primary key 583 { 584 colB := testutil.NewVector( 585 1, 586 types.T_int64.ToType(), 587 mp, 588 false, 589 []int64{ 590 8, 591 }, 592 ) 593 _, err := testWrite[memoryengine.DeleteResp]( 594 ctx, t, s, txnMeta, 595 memoryengine.OpDelete, 596 memoryengine.DeleteReq{ 597 TableID: relID, 598 ColumnName: "b", 599 Vector: colB, 600 }, 601 ) 602 assert.Nil(t, err) 603 } 604 605 // read after delete 606 { 607 resp, err := testRead[memoryengine.NewTableIterResp]( 608 ctx, t, s, txnMeta, 609 memoryengine.OpNewTableIter, 610 memoryengine.NewTableIterReq{ 611 TableID: relID, 612 }, 613 ) 614 assert.Nil(t, err) 615 assert.NotEmpty(t, resp.IterID) 616 iterID = resp.IterID 617 } 618 var rowIDs *vector.Vector 619 { 620 resp, err := testRead[memoryengine.ReadResp]( 621 ctx, t, s, txnMeta, 622 memoryengine.OpRead, 623 memoryengine.ReadReq{ 624 IterID: iterID, 625 ColNames: []string{"a", "b", rowIDColumnName}, 626 }, 627 ) 628 assert.Nil(t, err) 629 assert.NotNil(t, resp.Batch) 630 assert.Equal(t, 3, resp.Batch.Length()) 631 rowIDs = resp.Batch.Vecs[2] 632 } 633 634 // delete by row id 635 { 636 _, err := testWrite[memoryengine.DeleteResp]( 637 ctx, t, s, txnMeta, 638 memoryengine.OpDelete, 639 memoryengine.DeleteReq{ 640 TableID: relID, 641 ColumnName: rowIDColumnName, 642 Vector: rowIDs, 643 }, 644 ) 645 assert.Nil(t, err) 646 } 647 648 // read after delete 649 { 650 resp, err := testRead[memoryengine.NewTableIterResp]( 651 ctx, t, s, txnMeta, 652 memoryengine.OpNewTableIter, 653 memoryengine.NewTableIterReq{ 654 TableID: relID, 655 }, 656 ) 657 assert.Nil(t, err) 658 assert.NotEmpty(t, resp.IterID) 659 iterID = resp.IterID 660 } 661 { 662 resp, err := testRead[memoryengine.ReadResp]( 663 ctx, t, s, txnMeta, 664 memoryengine.OpRead, 665 memoryengine.ReadReq{ 666 IterID: iterID, 667 ColNames: []string{"a", "b", rowIDColumnName}, 668 }, 669 ) 670 assert.Nil(t, err) 671 assert.Nil(t, resp.Batch) 672 } 673 674 t.Run("duplicated db", func(t *testing.T) { 675 tx1 := txn.TxnMeta{ 676 ID: []byte("1"), 677 Status: txn.TxnStatus_Active, 678 SnapshotTS: timestamp.Timestamp{ 679 PhysicalTime: 1, 680 LogicalTime: 1, 681 }, 682 } 683 tx2 := txn.TxnMeta{ 684 ID: []byte("1"), 685 Status: txn.TxnStatus_Active, 686 SnapshotTS: timestamp.Timestamp{ 687 PhysicalTime: 1, 688 LogicalTime: 1, 689 }, 690 } 691 { 692 resp, err := testWrite[memoryengine.CreateDatabaseResp]( 693 ctx, t, s, tx1, 694 memoryengine.OpCreateDatabase, 695 memoryengine.CreateDatabaseReq{ 696 Name: "bar", 697 }, 698 ) 699 assert.Nil(t, err) 700 assert.NotEmpty(t, resp.ID) 701 } 702 { 703 _, err := testWrite[memoryengine.CreateDatabaseResp]( 704 ctx, t, s, tx2, 705 memoryengine.OpCreateDatabase, 706 memoryengine.CreateDatabaseReq{ 707 Name: "bar", 708 }, 709 ) 710 assert.NotNil(t, err) 711 } 712 }) 713 714 } 715 716 func testRead[ 717 Resp any, 718 Req any, 719 ]( 720 ctx context.Context, 721 t *testing.T, 722 s *Storage, 723 txnMeta txn.TxnMeta, 724 op uint32, 725 req Req, 726 ) ( 727 resp Resp, 728 err error, 729 ) { 730 731 buf := new(bytes.Buffer) 732 err = gob.NewEncoder(buf).Encode(req) 733 assert.Nil(t, err) 734 735 res, err := s.Read(ctx, txnMeta, op, buf.Bytes()) 736 if err != nil { 737 return 738 } 739 data, err := res.Read() 740 assert.Nil(t, err) 741 742 err = gob.NewDecoder(bytes.NewReader(data)).Decode(&resp) 743 assert.Nil(t, err) 744 745 return 746 } 747 748 func testWrite[ 749 Resp any, 750 Req any, 751 ]( 752 ctx context.Context, 753 t *testing.T, 754 s *Storage, 755 txnMeta txn.TxnMeta, 756 op uint32, 757 req Req, 758 ) ( 759 resp Resp, 760 err error, 761 ) { 762 763 buf := new(bytes.Buffer) 764 err = gob.NewEncoder(buf).Encode(req) 765 assert.Nil(t, err) 766 767 data, err := s.Write(ctx, txnMeta, op, buf.Bytes()) 768 if err != nil { 769 return 770 } 771 772 err = gob.NewDecoder(bytes.NewReader(data)).Decode(&resp) 773 assert.Nil(t, err) 774 775 return 776 }