github.com/nutsdb/nutsdb@v1.0.4/db_test.go (about) 1 // Copyright 2019 The nutsdb Author. All rights reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package nutsdb 16 17 import ( 18 "fmt" 19 "io/ioutil" 20 "math" 21 "os" 22 "strconv" 23 "testing" 24 "time" 25 26 "github.com/stretchr/testify/assert" 27 "github.com/stretchr/testify/require" 28 ) 29 30 var ( 31 db *DB 32 opt Options 33 err error 34 ) 35 36 const NutsDBTestDirPath = "/tmp/nutsdb-test" 37 38 func assertErr(t *testing.T, err error, expectErr error) { 39 if expectErr != nil { 40 require.Equal(t, expectErr, err) 41 } else { 42 require.NoError(t, err) 43 } 44 } 45 46 func removeDir(dir string) { 47 if err := os.RemoveAll(dir); err != nil { 48 panic(err) 49 } 50 } 51 52 func runNutsDBTest(t *testing.T, opts *Options, test func(t *testing.T, db *DB)) { 53 if opts == nil { 54 opts = &DefaultOptions 55 } 56 if opts.Dir == "" { 57 opts.Dir = NutsDBTestDirPath 58 } 59 defer removeDir(opts.Dir) 60 db, err := Open(*opts) 61 require.NoError(t, err) 62 63 test(t, db) 64 t.Cleanup(func() { 65 if !db.IsClose() { 66 require.NoError(t, db.Close()) 67 } 68 }) 69 } 70 71 func txPut(t *testing.T, db *DB, bucket string, key, value []byte, ttl uint32, expectErr error, finalExpectErr error) { 72 err := db.Update(func(tx *Tx) error { 73 err = tx.Put(bucket, key, value, ttl) 74 assertErr(t, err, expectErr) 75 return nil 76 }) 77 assertErr(t, err, finalExpectErr) 78 } 79 80 func txGet(t *testing.T, db *DB, bucket string, key []byte, expectVal []byte, expectErr error) { 81 err := db.View(func(tx *Tx) error { 82 value, err := tx.Get(bucket, key) 83 if expectErr != nil { 84 require.Equal(t, expectErr, err) 85 } else { 86 require.NoError(t, err) 87 require.EqualValuesf(t, expectVal, value, "err Tx Get. got %s want %s", string(value), string(expectVal)) 88 } 89 return nil 90 }) 91 require.NoError(t, err) 92 } 93 94 func txGetAll(t *testing.T, db *DB, bucket string, expectKeys [][]byte, expectValues [][]byte, expectErr error) { 95 require.NoError(t, db.View(func(tx *Tx) error { 96 keys, values, err := tx.GetAll(bucket) 97 if expectErr != nil { 98 require.Equal(t, expectErr, err) 99 } else { 100 require.NoError(t, err) 101 n := len(keys) 102 for i := 0; i < n; i++ { 103 require.Equal(t, expectKeys[i], keys[i]) 104 require.Equal(t, expectValues[i], values[i]) 105 } 106 } 107 return nil 108 })) 109 } 110 111 func txDel(t *testing.T, db *DB, bucket string, key []byte, expectErr error) { 112 err := db.Update(func(tx *Tx) error { 113 err := tx.Delete(bucket, key) 114 assertErr(t, err, expectErr) 115 return nil 116 }) 117 require.NoError(t, err) 118 } 119 120 func txGetMaxOrMinKey(t *testing.T, db *DB, bucket string, isMax bool, expectVal []byte, expectErr error) { 121 err := db.View(func(tx *Tx) error { 122 value, err := tx.getMaxOrMinKey(bucket, isMax) 123 if expectErr != nil { 124 require.Equal(t, expectErr, err) 125 } else { 126 require.NoError(t, err) 127 require.EqualValuesf(t, expectVal, value, "err Tx Get. got %s want %s", string(value), string(expectVal)) 128 } 129 return nil 130 }) 131 require.NoError(t, err) 132 } 133 134 func txDeleteBucket(t *testing.T, db *DB, ds uint16, bucket string, expectErr error) { 135 err := db.Update(func(tx *Tx) error { 136 err := tx.DeleteBucket(ds, bucket) 137 assertErr(t, err, expectErr) 138 return nil 139 }) 140 require.NoError(t, err) 141 } 142 143 func txCreateBucket(t *testing.T, db *DB, ds uint16, bucket string, expectErr error) { 144 err := db.Update(func(tx *Tx) error { 145 err := tx.NewBucket(ds, bucket) 146 assertErr(t, err, expectErr) 147 return nil 148 }) 149 require.NoError(t, err) 150 } 151 152 func InitOpt(fileDir string, isRemoveFiles bool) { 153 if fileDir == "" { 154 fileDir = "/tmp/nutsdbtest" 155 } 156 if isRemoveFiles { 157 files, _ := ioutil.ReadDir(fileDir) 158 for _, f := range files { 159 name := f.Name() 160 if name != "" { 161 err := os.RemoveAll(fileDir + "/" + name) 162 if err != nil { 163 panic(err) 164 } 165 } 166 } 167 } 168 169 opt = DefaultOptions 170 opt.Dir = fileDir 171 opt.SegmentSize = 8 * 1024 172 opt.CleanFdsCacheThreshold = 0.5 173 opt.MaxFdNumsInCache = 1024 174 } 175 176 func TestDB_Basic(t *testing.T) { 177 runNutsDBTest(t, nil, func(t *testing.T, db *DB) { 178 bucket := "bucket" 179 txCreateBucket(t, db, DataStructureBTree, bucket, nil) 180 key0 := GetTestBytes(0) 181 val0 := GetRandomBytes(24) 182 183 // put 184 txPut(t, db, bucket, key0, val0, Persistent, nil, nil) 185 txGet(t, db, bucket, key0, val0, nil) 186 187 val1 := GetRandomBytes(24) 188 189 // update 190 txPut(t, db, bucket, key0, val1, Persistent, nil, nil) 191 txGet(t, db, bucket, key0, val1, nil) 192 193 // del 194 txDel(t, db, bucket, key0, nil) 195 txGet(t, db, bucket, key0, val1, ErrKeyNotFound) 196 }) 197 } 198 199 func TestDB_ReopenWithDelete(t *testing.T) { 200 var opts *Options 201 if opts == nil { 202 opts = &DefaultOptions 203 } 204 if opts.Dir == "" { 205 opts.Dir = NutsDBTestDirPath 206 } 207 db, err := Open(*opts) 208 require.NoError(t, err) 209 defer removeDir(opts.Dir) 210 211 bucket := "bucket" 212 txCreateBucket(t, db, DataStructureList, bucket, nil) 213 txPush(t, db, bucket, GetTestBytes(5), GetTestBytes(0), true, nil, nil) 214 txPush(t, db, bucket, GetTestBytes(5), GetTestBytes(1), true, nil, nil) 215 txDeleteBucket(t, db, DataStructureList, bucket, nil) 216 217 if !db.IsClose() { 218 require.NoError(t, db.Close()) 219 } 220 221 db, err = Open(*opts) 222 require.NoError(t, err) 223 txCreateBucket(t, db, DataStructureList, bucket, nil) 224 txDeleteBucket(t, db, DataStructureList, bucket, nil) 225 if !db.IsClose() { 226 require.NoError(t, db.Close()) 227 } 228 } 229 230 func TestDB_Flock(t *testing.T) { 231 runNutsDBTest(t, nil, func(t *testing.T, db *DB) { 232 db2, err := Open(db.opt) 233 require.Nil(t, db2) 234 require.Equal(t, ErrDirLocked, err) 235 236 err = db.Close() 237 require.NoError(t, err) 238 239 db2, err = Open(db.opt) 240 require.NoError(t, err) 241 require.NotNil(t, db2) 242 243 err = db2.flock.Unlock() 244 require.NoError(t, err) 245 require.False(t, db2.flock.Locked()) 246 247 err = db2.Close() 248 require.Error(t, err) 249 require.Equal(t, ErrDirUnlocked, err) 250 }) 251 } 252 253 func TestDB_DeleteANonExistKey(t *testing.T) { 254 runNutsDBTest(t, nil, func(t *testing.T, db *DB) { 255 testBucket := "test_bucket" 256 txCreateBucket(t, db, DataStructureBTree, testBucket, nil) 257 258 txDel(t, db, testBucket, GetTestBytes(0), ErrKeyNotFound) 259 txPut(t, db, testBucket, GetTestBytes(1), GetRandomBytes(24), Persistent, nil, nil) 260 txDel(t, db, testBucket, GetTestBytes(0), ErrKeyNotFound) 261 }) 262 } 263 264 func TestDB_CheckListExpired(t *testing.T) { 265 runNutsDBTest(t, nil, func(t *testing.T, db *DB) { 266 testBucket := "test_bucket" 267 txCreateBucket(t, db, DataStructureBTree, testBucket, nil) 268 269 txPut(t, db, testBucket, GetTestBytes(0), GetTestBytes(1), Persistent, nil, nil) 270 txPut(t, db, testBucket, GetTestBytes(1), GetRandomBytes(24), 1, nil, nil) 271 272 time.Sleep(1100 * time.Millisecond) 273 274 db.checkListExpired() 275 276 // this entry still alive 277 txGet(t, db, testBucket, GetTestBytes(0), GetTestBytes(1), nil) 278 // this entry will be deleted 279 txGet(t, db, testBucket, GetTestBytes(1), nil, ErrKeyNotFound) 280 }) 281 } 282 283 func txLRem(t *testing.T, db *DB, bucket string, key []byte, count int, value []byte, expectErr error) { 284 err := db.Update(func(tx *Tx) error { 285 err := tx.LRem(bucket, key, count, value) 286 assertErr(t, err, expectErr) 287 return nil 288 }) 289 require.NoError(t, err) 290 } 291 292 func txLRemByIndex(t *testing.T, db *DB, bucket string, key []byte, expectErr error, indexes ...int) { 293 err := db.Update(func(tx *Tx) error { 294 err := tx.LRemByIndex(bucket, key, indexes...) 295 assertErr(t, err, expectErr) 296 return nil 297 }) 298 require.NoError(t, err) 299 } 300 301 func txSAdd(t *testing.T, db *DB, bucket string, key, value []byte, expectErr error, finalExpectErr error) { 302 err := db.Update(func(tx *Tx) error { 303 err := tx.SAdd(bucket, key, value) 304 assertErr(t, err, expectErr) 305 return nil 306 }) 307 assertErr(t, err, finalExpectErr) 308 } 309 310 func txSKeys(t *testing.T, db *DB, bucket, pattern string, f func(key string) bool, expectVal int, expectErr error) { 311 err := db.View(func(tx *Tx) error { 312 patternMatchNum := 0 313 err := tx.SKeys(bucket, pattern, func(key string) bool { 314 patternMatchNum += 1 315 return f(key) 316 }) 317 if expectErr != nil { 318 assert.ErrorIs(t, expectErr, err) 319 } else { 320 assert.NoError(t, err) 321 assert.Equal(t, expectVal, patternMatchNum) 322 } 323 return nil 324 }) 325 require.NoError(t, err) 326 } 327 328 func txSIsMember(t *testing.T, db *DB, bucket string, key, value []byte, expect bool) { 329 err := db.View(func(tx *Tx) error { 330 ok, _ := tx.SIsMember(bucket, key, value) 331 require.Equal(t, expect, ok) 332 return nil 333 }) 334 require.NoError(t, err) 335 } 336 337 func txSAreMembers(t *testing.T, db *DB, bucket string, key []byte, expect bool, value ...[]byte) { 338 err := db.View(func(tx *Tx) error { 339 ok, _ := tx.SAreMembers(bucket, key, value...) 340 require.Equal(t, expect, ok) 341 return nil 342 }) 343 require.NoError(t, err) 344 } 345 346 func txSHasKey(t *testing.T, db *DB, bucket string, key []byte, expect bool) { 347 err := db.View(func(tx *Tx) error { 348 ok, _ := tx.SHasKey(bucket, key) 349 require.Equal(t, expect, ok) 350 return nil 351 }) 352 require.NoError(t, err) 353 } 354 355 func txSMembers(t *testing.T, db *DB, bucket string, key []byte, expectLength int, expectErr error) { 356 err := db.View(func(tx *Tx) error { 357 members, err := tx.SMembers(bucket, key) 358 if expectErr != nil { 359 assert.ErrorIs(t, expectErr, err) 360 } else { 361 assert.NoError(t, err) 362 assert.Equal(t, expectLength, len(members)) 363 } 364 return nil 365 }) 366 require.NoError(t, err) 367 } 368 369 func txSCard(t *testing.T, db *DB, bucket string, key []byte, expectLength int, expectErr error) { 370 err := db.View(func(tx *Tx) error { 371 length, err := tx.SCard(bucket, key) 372 if expectErr != nil { 373 assert.ErrorIs(t, expectErr, err) 374 } else { 375 assert.NoError(t, err) 376 assert.Equal(t, expectLength, length) 377 } 378 return nil 379 }) 380 require.NoError(t, err) 381 } 382 383 func txSDiffByOneBucket(t *testing.T, db *DB, bucket string, key1, key2 []byte, expectVal [][]byte, expectErr error) { 384 err := db.View(func(tx *Tx) error { 385 diff, err := tx.SDiffByOneBucket(bucket, key1, key2) 386 if expectErr != nil { 387 assert.ErrorIs(t, expectErr, err) 388 } else { 389 assert.NoError(t, err) 390 assert.ElementsMatch(t, expectVal, diff) 391 } 392 return nil 393 }) 394 require.NoError(t, err) 395 } 396 397 func txSDiffByTwoBucket(t *testing.T, db *DB, bucket1 string, key1 []byte, bucket2 string, key2 []byte, expectVal [][]byte, expectErr error) { 398 err := db.View(func(tx *Tx) error { 399 diff, err := tx.SDiffByTwoBuckets(bucket1, key1, bucket2, key2) 400 if expectErr != nil { 401 assert.ErrorIs(t, err, expectErr) 402 } else { 403 assert.NoError(t, err) 404 assert.ElementsMatch(t, expectVal, diff) 405 } 406 return nil 407 }) 408 require.NoError(t, err) 409 } 410 411 func txSPop(t *testing.T, db *DB, bucket string, key []byte, expectErr error) { 412 err := db.Update(func(tx *Tx) error { 413 _, err := tx.SPop(bucket, key) 414 assertErr(t, err, expectErr) 415 return nil 416 }) 417 require.NoError(t, err) 418 } 419 420 func txSMoveByOneBucket(t *testing.T, db *DB, bucket1 string, key1, key2, val []byte, expectVal bool, expectErr error) { 421 err := db.View(func(tx *Tx) error { 422 ok, err := tx.SMoveByOneBucket(bucket1, key1, key2, val) 423 if expectErr != nil { 424 assert.ErrorIs(t, err, expectErr) 425 } else { 426 assert.NoError(t, err) 427 assert.Equal(t, expectVal, ok) 428 } 429 return nil 430 }) 431 require.NoError(t, err) 432 } 433 434 func txSMoveByTwoBuckets(t *testing.T, db *DB, bucket1 string, key1 []byte, bucket2 string, key2 []byte, val []byte, expectVal bool, expectErr error) { 435 err := db.View(func(tx *Tx) error { 436 ok, err := tx.SMoveByTwoBuckets(bucket1, key1, bucket2, key2, val) 437 if expectErr != nil { 438 assert.ErrorIs(t, err, expectErr) 439 } else { 440 assert.NoError(t, err) 441 assert.Equal(t, expectVal, ok) 442 } 443 return nil 444 }) 445 require.NoError(t, err) 446 } 447 448 func txSUnionByOneBucket(t *testing.T, db *DB, bucket1 string, key1, key2 []byte, expectVal [][]byte, expectErr error) { 449 err := db.View(func(tx *Tx) error { 450 union, err := tx.SUnionByOneBucket(bucket1, key1, key2) 451 if expectErr != nil { 452 assert.ErrorIs(t, err, expectErr) 453 } else { 454 assert.NoError(t, err) 455 assert.ElementsMatch(t, expectVal, union) 456 } 457 return nil 458 }) 459 require.NoError(t, err) 460 } 461 462 func txSUnionByTwoBuckets(t *testing.T, db *DB, bucket1 string, key1 []byte, bucket2 string, key2 []byte, expectVal [][]byte, expectErr error) { 463 err := db.View(func(tx *Tx) error { 464 union, err := tx.SUnionByTwoBuckets(bucket1, key1, bucket2, key2) 465 if expectErr != nil { 466 assert.ErrorIs(t, err, expectErr) 467 } else { 468 assert.NoError(t, err) 469 assert.ElementsMatch(t, expectVal, union) 470 } 471 return nil 472 }) 473 require.NoError(t, err) 474 } 475 476 func txSRem(t *testing.T, db *DB, bucket string, key, value []byte, expectErr error) { 477 err := db.Update(func(tx *Tx) error { 478 err := tx.SRem(bucket, key, value) 479 assertErr(t, err, expectErr) 480 return nil 481 }) 482 require.NoError(t, err) 483 } 484 485 func txZAdd(t *testing.T, db *DB, bucket string, key, value []byte, score float64, expectErr error, finalExpectErr error) { 486 err := db.Update(func(tx *Tx) error { 487 err := tx.ZAdd(bucket, key, score, value) 488 assertErr(t, err, expectErr) 489 return nil 490 }) 491 assertErr(t, err, finalExpectErr) 492 } 493 494 func txZRem(t *testing.T, db *DB, bucket string, key, value []byte, expectErr error) { 495 err := db.Update(func(tx *Tx) error { 496 err := tx.ZRem(bucket, key, value) 497 assertErr(t, err, expectErr) 498 return nil 499 }) 500 assert.NoError(t, err) 501 } 502 503 func txZCard(t *testing.T, db *DB, bucket string, key []byte, expectLength int, expectErr error) { 504 err := db.View(func(tx *Tx) error { 505 length, err := tx.ZCard(bucket, key) 506 if expectErr != nil { 507 assert.Equal(t, expectErr, err) 508 } else { 509 assert.Equal(t, expectLength, length) 510 } 511 return nil 512 }) 513 assert.NoError(t, err) 514 } 515 516 func txZScore(t *testing.T, db *DB, bucket string, key, value []byte, expectScore float64, expectErr error) { 517 err := db.View(func(tx *Tx) error { 518 score, err := tx.ZScore(bucket, key, value) 519 if err != nil { 520 assert.Equal(t, expectErr, err) 521 } else { 522 assert.Equal(t, expectScore, score) 523 } 524 return nil 525 }) 526 assert.NoError(t, err) 527 } 528 529 func txZRank(t *testing.T, db *DB, bucket string, key, value []byte, isRev bool, expectRank int, expectErr error) { 530 err := db.View(func(tx *Tx) error { 531 var ( 532 rank int 533 err error 534 ) 535 if isRev { 536 rank, err = tx.ZRevRank(bucket, key, value) 537 } else { 538 rank, err = tx.ZRank(bucket, key, value) 539 } 540 if expectErr != nil { 541 assert.Equal(t, expectErr, err) 542 } else { 543 assert.Equal(t, expectRank, rank) 544 } 545 return nil 546 }) 547 assert.NoError(t, err) 548 } 549 550 func txZPop(t *testing.T, db *DB, bucket string, key []byte, isMax bool, expectVal []byte, expectScore float64, expectErr error) { 551 err := db.Update(func(tx *Tx) error { 552 var ( 553 member *SortedSetMember 554 err error 555 ) 556 if isMax { 557 member, err = tx.ZPopMax(bucket, key) 558 } else { 559 member, err = tx.ZPopMin(bucket, key) 560 } 561 562 if expectErr != nil { 563 assert.Equal(t, expectErr, err) 564 } else { 565 assert.Equal(t, expectVal, member.Value) 566 assert.Equal(t, expectScore, member.Score) 567 } 568 return nil 569 }) 570 assert.NoError(t, err) 571 } 572 573 func txZPeekMin(t *testing.T, db *DB, bucket string, key, expectVal []byte, expectScore float64, expectErr, finalExpectErr error) { 574 err := db.View(func(tx *Tx) error { 575 minMem, err1 := tx.ZPeekMin(bucket, key) 576 assertErr(t, err1, finalExpectErr) 577 578 if expectErr == nil { 579 require.Equal(t, &SortedSetMember{ 580 Value: expectVal, 581 Score: expectScore, 582 }, minMem) 583 } 584 return err1 585 }) 586 assertErr(t, err, finalExpectErr) 587 } 588 589 func txZKeys(t *testing.T, db *DB, bucket, pattern string, f func(key string) bool, expectVal int, expectErr error) { 590 err := db.View(func(tx *Tx) error { 591 patternMatchNum := 0 592 err := tx.ZKeys(bucket, pattern, func(key string) bool { 593 patternMatchNum += 1 594 return f(key) 595 }) 596 if expectErr != nil { 597 assert.ErrorIs(t, expectErr, err) 598 } else { 599 assert.NoError(t, err) 600 assert.Equal(t, expectVal, patternMatchNum) 601 } 602 return nil 603 }) 604 require.NoError(t, err) 605 } 606 607 func txPop(t *testing.T, db *DB, bucket string, key, expectVal []byte, expectErr error, isLeft bool) { 608 err := db.Update(func(tx *Tx) error { 609 var item []byte 610 var err error 611 612 if isLeft { 613 item, err = tx.LPop(bucket, key) 614 } else { 615 item, err = tx.RPop(bucket, key) 616 } 617 618 if expectErr != nil { 619 require.Equal(t, expectErr, err) 620 } else { 621 require.Equal(t, expectVal, item) 622 } 623 624 return nil 625 }) 626 require.NoError(t, err) 627 } 628 629 func txPush(t *testing.T, db *DB, bucket string, key, val []byte, isLeft bool, expectErr error, finalExpectErr error) { 630 err := db.Update(func(tx *Tx) error { 631 var err error 632 633 if isLeft { 634 err = tx.LPush(bucket, key, val) 635 } else { 636 err = tx.RPush(bucket, key, val) 637 } 638 639 assertErr(t, err, expectErr) 640 641 return nil 642 }) 643 assertErr(t, err, finalExpectErr) 644 } 645 646 func txMPush(t *testing.T, db *DB, bucket string, key []byte, vals [][]byte, isLeft bool, expectErr error, finalExpectErr error) { 647 err := db.Update(func(tx *Tx) error { 648 var err error 649 650 if isLeft { 651 err = tx.LPush(bucket, key, vals...) 652 } else { 653 err = tx.RPush(bucket, key, vals...) 654 } 655 656 assertErr(t, err, expectErr) 657 658 return nil 659 }) 660 assertErr(t, err, finalExpectErr) 661 } 662 663 func txPushRaw(t *testing.T, db *DB, bucket string, key, val []byte, isLeft bool, expectErr error, finalExpectErr error) { 664 err := db.Update(func(tx *Tx) error { 665 var err error 666 667 if isLeft { 668 err = tx.LPushRaw(bucket, key, val) 669 } else { 670 err = tx.RPushRaw(bucket, key, val) 671 } 672 673 assertErr(t, err, expectErr) 674 675 return nil 676 }) 677 assertErr(t, err, finalExpectErr) 678 } 679 680 func txExpireList(t *testing.T, db *DB, bucket string, key []byte, ttl uint32, expectErr error) { 681 err := db.Update(func(tx *Tx) error { 682 err := tx.ExpireList(bucket, key, ttl) 683 assertErr(t, err, expectErr) 684 return nil 685 }) 686 require.NoError(t, err) 687 } 688 689 func txGetListTTL(t *testing.T, db *DB, bucket string, key []byte, expectVal uint32, expectErr error) { 690 err := db.View(func(tx *Tx) error { 691 ttl, err := tx.GetListTTL(bucket, key) 692 assertErr(t, err, expectErr) 693 require.Equal(t, ttl, expectVal) 694 return nil 695 }) 696 require.NoError(t, err) 697 } 698 699 func txLKeys(t *testing.T, db *DB, bucket, pattern string, expectLen int, expectErr error, keysOperation func(keys []string) bool) { 700 err := db.View(func(tx *Tx) error { 701 var keys []string 702 err := tx.LKeys(bucket, pattern, func(key string) bool { 703 keys = append(keys, key) 704 return keysOperation(keys) 705 }) 706 assertErr(t, err, expectErr) 707 require.Equal(t, expectLen, len(keys)) 708 return nil 709 }) 710 require.NoError(t, err) 711 } 712 713 func txLRange(t *testing.T, db *DB, bucket string, key []byte, start, end, expectLen int, expectVal [][]byte, expectErr error) { 714 err := db.View(func(tx *Tx) error { 715 list, err := tx.LRange(bucket, key, start, end) 716 assertErr(t, err, expectErr) 717 718 require.Equal(t, expectLen, len(list)) 719 720 if len(expectVal) > 0 { 721 for i, val := range list { 722 assert.Equal(t, expectVal[i], val) 723 } 724 } 725 726 return nil 727 }) 728 require.NoError(t, err) 729 } 730 731 func txLSize(t *testing.T, db *DB, bucket string, key []byte, expectVal int, expectErr error) { 732 err := db.View(func(tx *Tx) error { 733 size, err := tx.LSize(bucket, key) 734 assertErr(t, err, expectErr) 735 736 require.Equal(t, expectVal, size) 737 738 return nil 739 }) 740 require.NoError(t, err) 741 } 742 743 func txLTrim(t *testing.T, db *DB, bucket string, key []byte, start int, end int, expectErr error) { 744 err := db.Update(func(tx *Tx) error { 745 err := tx.LTrim(bucket, key, start, end) 746 assertErr(t, err, expectErr) 747 return nil 748 }) 749 require.NoError(t, err) 750 } 751 752 func txIterateBuckets(t *testing.T, db *DB, ds uint16, pattern string, f func(key string) bool, expectErr error, containsKey ...string) { 753 err := db.View(func(tx *Tx) error { 754 var elements []string 755 err := tx.IterateBuckets(ds, pattern, func(key string) bool { 756 if f != nil && !f(key) { 757 return false 758 } 759 elements = append(elements, key) 760 return true 761 }) 762 if err != nil { 763 assert.Equal(t, expectErr, err) 764 } else { 765 assert.NoError(t, err) 766 for _, key := range containsKey { 767 assert.Contains(t, elements, key) 768 } 769 } 770 return nil 771 }) 772 require.NoError(t, err) 773 } 774 775 func TestDB_GetKeyNotFound(t *testing.T) { 776 runNutsDBTest(t, nil, func(t *testing.T, db *DB) { 777 bucket := "bucket" 778 txCreateBucket(t, db, DataStructureBTree, bucket, nil) 779 txGet(t, db, bucket, GetTestBytes(0), nil, ErrKeyNotFound) 780 txPut(t, db, bucket, GetTestBytes(1), GetRandomBytes(24), Persistent, nil, nil) 781 txGet(t, db, bucket, GetTestBytes(0), nil, ErrKeyNotFound) 782 }) 783 } 784 785 func TestDB_Backup(t *testing.T) { 786 runNutsDBTest(t, nil, func(t *testing.T, db *DB) { 787 backUpDir := "/tmp/nutsdb-backup" 788 require.NoError(t, db.Backup(backUpDir)) 789 }) 790 } 791 792 func TestDB_BackupTarGZ(t *testing.T) { 793 runNutsDBTest(t, nil, func(t *testing.T, db *DB) { 794 backUpFile := "/tmp/nutsdb-backup/backup.tar.gz" 795 f, err := os.Create(backUpFile) 796 require.NoError(t, err) 797 require.NoError(t, db.BackupTarGZ(f)) 798 }) 799 } 800 801 func TestDB_Close(t *testing.T) { 802 runNutsDBTest(t, nil, func(t *testing.T, db *DB) { 803 require.NoError(t, db.Close()) 804 require.Equal(t, ErrDBClosed, db.Close()) 805 }) 806 } 807 808 func TestDB_ErrThenReadWrite(t *testing.T) { 809 runNutsDBTest(t, nil, func(t *testing.T, db *DB) { 810 bucket := "testForDeadLock" 811 err = db.View( 812 func(tx *Tx) error { 813 return fmt.Errorf("err happened") 814 }) 815 require.NotNil(t, err) 816 817 err = db.View( 818 func(tx *Tx) error { 819 key := []byte("key1") 820 _, err := tx.Get(bucket, key) 821 if err != nil { 822 return err 823 } 824 825 return nil 826 }) 827 require.NotNil(t, err) 828 829 notice := make(chan struct{}) 830 go func() { 831 err = db.Update( 832 func(tx *Tx) error { 833 notice <- struct{}{} 834 835 return nil 836 }) 837 require.NoError(t, err) 838 }() 839 840 select { 841 case <-notice: 842 case <-time.After(1 * time.Second): 843 t.Fatalf("exist deadlock") 844 } 845 }) 846 } 847 848 func TestDB_ErrorHandler(t *testing.T) { 849 opts := DefaultOptions 850 handleErrCalled := false 851 opts.ErrorHandler = ErrorHandlerFunc(func(err error) { 852 handleErrCalled = true 853 }) 854 855 runNutsDBTest(t, &opts, func(t *testing.T, db *DB) { 856 err = db.View( 857 func(tx *Tx) error { 858 return fmt.Errorf("err happened") 859 }) 860 require.NotNil(t, err) 861 require.Equal(t, handleErrCalled, true) 862 }) 863 } 864 865 func TestDB_CommitBuffer(t *testing.T) { 866 bucket := "bucket" 867 868 opts := DefaultOptions 869 opts.CommitBufferSize = 8 * MB 870 runNutsDBTest(t, &opts, func(t *testing.T, db *DB) { 871 require.Equal(t, int64(8*MB), db.opt.CommitBufferSize) 872 // When the database starts, the commit buffer should be allocated with the size of CommitBufferSize. 873 require.Equal(t, 0, db.commitBuffer.Len()) 874 require.Equal(t, db.opt.CommitBufferSize, int64(db.commitBuffer.Cap())) 875 txCreateBucket(t, db, DataStructureBTree, bucket, nil) 876 txPut(t, db, bucket, GetTestBytes(0), GetRandomBytes(24), Persistent, nil, nil) 877 878 // When tx is committed, content of commit buffer should be empty, but do not release memory 879 require.Equal(t, 0, db.commitBuffer.Len()) 880 require.Equal(t, db.opt.CommitBufferSize, int64(db.commitBuffer.Cap())) 881 }) 882 883 opts = DefaultOptions 884 opts.CommitBufferSize = 1 * KB 885 runNutsDBTest(t, &opts, func(t *testing.T, db *DB) { 886 require.Equal(t, int64(1*KB), db.opt.CommitBufferSize) 887 888 txCreateBucket(t, db, DataStructureBTree, bucket, nil) 889 err := db.Update(func(tx *Tx) error { 890 // making this tx big enough, it should not use the commit buffer 891 for i := 0; i < 1000; i++ { 892 err := tx.Put(bucket, GetTestBytes(i), GetRandomBytes(1024), Persistent) 893 require.NoError(t, err) 894 } 895 return nil 896 }) 897 require.NoError(t, err) 898 899 require.Equal(t, 0, db.commitBuffer.Len()) 900 require.Equal(t, db.opt.CommitBufferSize, int64(db.commitBuffer.Cap())) 901 }) 902 } 903 904 func TestDB_DeleteBucket(t *testing.T) { 905 runNutsDBTest(t, nil, func(t *testing.T, db *DB) { 906 bucket := "bucket" 907 txCreateBucket(t, db, DataStructureBTree, bucket, nil) 908 key := GetTestBytes(0) 909 val := GetTestBytes(0) 910 txPut(t, db, bucket, key, val, Persistent, nil, nil) 911 txGet(t, db, bucket, key, val, nil) 912 913 txDeleteBucket(t, db, DataStructureBTree, bucket, nil) 914 txPut(t, db, bucket, key, val, Persistent, ErrorBucketNotExist, nil) 915 }) 916 } 917 918 func withDBOption(t *testing.T, opt Options, fn func(t *testing.T, db *DB)) { 919 db, err := Open(opt) 920 require.NoError(t, err) 921 922 defer func() { 923 os.RemoveAll(db.opt.Dir) 924 db.Close() 925 }() 926 927 fn(t, db) 928 } 929 930 func withDefaultDB(t *testing.T, fn func(t *testing.T, db *DB)) { 931 tmpdir, _ := os.MkdirTemp("", "nutsdb") 932 opt := DefaultOptions 933 opt.Dir = tmpdir 934 opt.SegmentSize = 8 * 1024 935 936 withDBOption(t, opt, fn) 937 } 938 939 func withRAMIdxDB(t *testing.T, fn func(t *testing.T, db *DB)) { 940 tmpdir, _ := os.MkdirTemp("", "nutsdb") 941 opt := DefaultOptions 942 opt.Dir = tmpdir 943 opt.EntryIdxMode = HintKeyAndRAMIdxMode 944 945 withDBOption(t, opt, fn) 946 } 947 948 func TestDB_HintKeyValAndRAMIdxMode_RestartDB(t *testing.T) { 949 opts := DefaultOptions 950 runNutsDBTest(t, &opts, func(t *testing.T, db *DB) { 951 bucket := "bucket" 952 txCreateBucket(t, db, DataStructureBTree, bucket, nil) 953 954 key := GetTestBytes(0) 955 val := GetTestBytes(0) 956 957 txPut(t, db, bucket, key, val, Persistent, nil, nil) 958 txGet(t, db, bucket, key, val, nil) 959 960 db.Close() 961 // restart db with HintKeyValAndRAMIdxMode EntryIdxMode 962 db, err := Open(db.opt) 963 require.NoError(t, err) 964 txGet(t, db, bucket, key, val, nil) 965 }) 966 } 967 968 func TestDB_HintKeyAndRAMIdxMode_RestartDB(t *testing.T) { 969 opts := DefaultOptions 970 opts.EntryIdxMode = HintKeyAndRAMIdxMode 971 runNutsDBTest(t, &opts, func(t *testing.T, db *DB) { 972 bucket := "bucket" 973 txCreateBucket(t, db, DataStructureBTree, bucket, nil) 974 key := GetTestBytes(0) 975 val := GetTestBytes(0) 976 977 txPut(t, db, bucket, key, val, Persistent, nil, nil) 978 txGet(t, db, bucket, key, val, nil) 979 db.Close() 980 981 // restart db with HintKeyAndRAMIdxMode EntryIdxMode 982 db, err := Open(db.opt) 983 require.NoError(t, err) 984 txGet(t, db, bucket, key, val, nil) 985 }) 986 } 987 988 func TestDB_HintKeyAndRAMIdxMode_LruCache(t *testing.T) { 989 opts := DefaultOptions 990 opts.EntryIdxMode = HintKeyAndRAMIdxMode 991 lruCacheSizes := []int{0, 5000, 10000, 20000} 992 993 for _, lruCacheSize := range lruCacheSizes { 994 opts.HintKeyAndRAMIdxCacheSize = lruCacheSize 995 runNutsDBTest(t, &opts, func(t *testing.T, db *DB) { 996 bucket := "bucket" 997 txCreateBucket(t, db, DataStructureBTree, bucket, nil) 998 for i := 0; i < 10000; i++ { 999 key := []byte(fmt.Sprintf("%10d", i)) 1000 val := []byte(fmt.Sprintf("%10d", i)) 1001 txPut(t, db, bucket, key, val, Persistent, nil, nil) 1002 txGet(t, db, bucket, key, val, nil) 1003 txGet(t, db, bucket, key, val, nil) 1004 } 1005 db.Close() 1006 }) 1007 } 1008 } 1009 1010 func TestDB_ChangeMode_RestartDB(t *testing.T) { 1011 changeModeRestart := func(firstMode EntryIdxMode, secondMode EntryIdxMode) { 1012 opts := DefaultOptions 1013 opts.EntryIdxMode = firstMode 1014 var err error 1015 1016 runNutsDBTest(t, &opts, func(t *testing.T, db *DB) { 1017 bucket := "bucket" 1018 txCreateBucket(t, db, DataStructureBTree, bucket, nil) 1019 txCreateBucket(t, db, DataStructureList, bucket, nil) 1020 txCreateBucket(t, db, DataStructureSet, bucket, nil) 1021 txCreateBucket(t, db, DataStructureSortedSet, bucket, nil) 1022 1023 // k-v 1024 for i := 0; i < 10; i++ { 1025 txPut(t, db, bucket, GetTestBytes(i), GetTestBytes(i), Persistent, nil, nil) 1026 } 1027 1028 // list 1029 for i := 0; i < 10; i++ { 1030 txPush(t, db, bucket, GetTestBytes(0), GetTestBytes(i), true, nil, nil) 1031 } 1032 1033 err = db.Update(func(tx *Tx) error { 1034 return tx.LRem(bucket, GetTestBytes(0), 1, GetTestBytes(5)) 1035 }) 1036 require.NoError(t, err) 1037 1038 for i := 0; i < 2; i++ { 1039 txPop(t, db, bucket, GetTestBytes(0), GetTestBytes(9-i), nil, true) 1040 } 1041 1042 for i := 0; i < 2; i++ { 1043 txPop(t, db, bucket, GetTestBytes(0), GetTestBytes(i), nil, false) 1044 } 1045 1046 // set 1047 for i := 0; i < 10; i++ { 1048 txSAdd(t, db, bucket, GetTestBytes(0), GetTestBytes(i), nil, nil) 1049 } 1050 1051 for i := 0; i < 3; i++ { 1052 txSRem(t, db, bucket, GetTestBytes(0), GetTestBytes(i), nil) 1053 } 1054 1055 // zset 1056 for i := 0; i < 10; i++ { 1057 txZAdd(t, db, bucket, GetTestBytes(0), GetTestBytes(i), float64(i), nil, nil) 1058 } 1059 1060 for i := 0; i < 3; i++ { 1061 txZRem(t, db, bucket, GetTestBytes(0), GetTestBytes(i), nil) 1062 } 1063 1064 require.NoError(t, db.Close()) 1065 1066 opts.EntryIdxMode = secondMode 1067 db, err = Open(opts) 1068 require.NoError(t, err) 1069 1070 // k-v 1071 for i := 0; i < 10; i++ { 1072 txGet(t, db, bucket, GetTestBytes(i), GetTestBytes(i), nil) 1073 } 1074 1075 // list 1076 txPop(t, db, bucket, GetTestBytes(0), GetTestBytes(7), nil, true) 1077 txPop(t, db, bucket, GetTestBytes(0), GetTestBytes(6), nil, true) 1078 txPop(t, db, bucket, GetTestBytes(0), GetTestBytes(4), nil, true) 1079 txPop(t, db, bucket, GetTestBytes(0), GetTestBytes(2), nil, false) 1080 1081 err = db.View(func(tx *Tx) error { 1082 size, err := tx.LSize(bucket, GetTestBytes(0)) 1083 require.NoError(t, err) 1084 require.Equal(t, 1, size) 1085 return nil 1086 }) 1087 require.NoError(t, err) 1088 1089 // set 1090 for i := 0; i < 3; i++ { 1091 txSIsMember(t, db, bucket, GetTestBytes(0), GetTestBytes(i), false) 1092 } 1093 1094 for i := 3; i < 10; i++ { 1095 txSIsMember(t, db, bucket, GetTestBytes(0), GetTestBytes(i), true) 1096 } 1097 1098 // zset 1099 for i := 0; i < 3; i++ { 1100 txZScore(t, db, bucket, GetTestBytes(0), GetTestBytes(i), float64(i), ErrSortedSetMemberNotExist) 1101 } 1102 1103 for i := 3; i < 10; i++ { 1104 txZScore(t, db, bucket, GetTestBytes(0), GetTestBytes(i), float64(i), nil) 1105 } 1106 }) 1107 } 1108 1109 // HintKeyValAndRAMIdxMode to HintKeyAndRAMIdxMode 1110 changeModeRestart(HintKeyValAndRAMIdxMode, HintKeyAndRAMIdxMode) 1111 // HintKeyAndRAMIdxMode to HintKeyValAndRAMIdxMode 1112 changeModeRestart(HintKeyAndRAMIdxMode, HintKeyValAndRAMIdxMode) 1113 } 1114 1115 func TestTx_SmallFile(t *testing.T) { 1116 opts := DefaultOptions 1117 opts.SegmentSize = 100 1118 opts.EntryIdxMode = HintKeyAndRAMIdxMode 1119 runNutsDBTest(t, &opts, func(t *testing.T, db *DB) { 1120 bucket := "bucket" 1121 txCreateBucket(t, db, DataStructureBTree, bucket, nil) 1122 1123 err := db.Update(func(tx *Tx) error { 1124 for i := 0; i < 100; i++ { 1125 err := tx.Put(bucket, GetTestBytes(i), GetTestBytes(i), Persistent) 1126 if err != nil { 1127 return err 1128 } 1129 } 1130 return nil 1131 }) 1132 require.Nil(t, err) 1133 require.NoError(t, db.Close()) 1134 db, _ = Open(opts) 1135 1136 txGet(t, db, bucket, GetTestBytes(10), GetTestBytes(10), nil) 1137 }) 1138 } 1139 1140 func TestDB_DataStructureBTreeWriteRecordLimit(t *testing.T) { 1141 opts := DefaultOptions 1142 limitCount := int64(1000) 1143 opts.MaxWriteRecordCount = limitCount 1144 bucket1 := "bucket1" 1145 bucket2 := "bucket2" 1146 // Iterate over different EntryIdxModes 1147 for _, idxMode := range []EntryIdxMode{HintKeyValAndRAMIdxMode, HintKeyAndRAMIdxMode} { 1148 opts.EntryIdxMode = idxMode 1149 runNutsDBTest(t, &opts, func(t *testing.T, db *DB) { 1150 txCreateBucket(t, db, DataStructureBTree, bucket1, nil) 1151 txCreateBucket(t, db, DataStructureBTree, bucket2, nil) 1152 1153 // Add limitCount records 1154 err := db.Update(func(tx *Tx) error { 1155 for i := 0; i < int(limitCount); i++ { 1156 key := []byte(strconv.Itoa(i)) 1157 value := []byte(strconv.Itoa(i)) 1158 err = tx.Put(bucket1, key, value, Persistent) 1159 assertErr(t, err, nil) 1160 } 1161 return nil 1162 }) 1163 require.NoError(t, err) 1164 // Trigger the limit 1165 txPut(t, db, bucket1, []byte("key1"), []byte("value1"), Persistent, nil, ErrTxnExceedWriteLimit) 1166 // Add a key that is within the limit 1167 txPut(t, db, bucket1, []byte("0"), []byte("000"), Persistent, nil, nil) 1168 // Delete and add one item 1169 txDel(t, db, bucket1, []byte("0"), nil) 1170 txPut(t, db, bucket1, []byte("key1"), []byte("value1"), Persistent, nil, nil) 1171 // Add an item to another bucket 1172 txPut(t, db, bucket2, []byte("key2"), []byte("value2"), Persistent, nil, ErrTxnExceedWriteLimit) 1173 // Delete bucket1 1174 txDeleteBucket(t, db, DataStructureBTree, bucket1, nil) 1175 // Add data to bucket2 1176 err = db.Update(func(tx *Tx) error { 1177 for i := 0; i < (int(limitCount) - 1); i++ { 1178 key := []byte(strconv.Itoa(i)) 1179 value := []byte(strconv.Itoa(i)) 1180 err = tx.Put(bucket2, key, value, Persistent) 1181 assertErr(t, err, nil) 1182 } 1183 return nil 1184 }) 1185 require.NoError(t, err) 1186 // Add items to bucket2 1187 txPut(t, db, bucket2, []byte("key1"), []byte("value1"), Persistent, nil, nil) 1188 txPut(t, db, bucket2, []byte("key2"), []byte("value2"), Persistent, nil, ErrTxnExceedWriteLimit) 1189 }) 1190 } 1191 } 1192 1193 func TestDB_DataStructureListWriteRecordLimit(t *testing.T) { 1194 // Set options 1195 opts := DefaultOptions 1196 limitCount := int64(1000) 1197 opts.MaxWriteRecordCount = limitCount 1198 // Define bucket names 1199 bucket1 := "bucket1" 1200 bucket2 := "bucket2" 1201 // Iterate over EntryIdxMode options 1202 for _, idxMode := range []EntryIdxMode{HintKeyValAndRAMIdxMode, HintKeyAndRAMIdxMode} { 1203 1204 opts.EntryIdxMode = idxMode 1205 runNutsDBTest(t, &opts, func(t *testing.T, db *DB) { 1206 txCreateBucket(t, db, DataStructureList, bucket1, nil) 1207 txCreateBucket(t, db, DataStructureList, bucket2, nil) 1208 // Add limitCount records 1209 err := db.Update(func(tx *Tx) error { 1210 for i := 0; i < int(limitCount); i++ { 1211 key := []byte("0") 1212 value := []byte(strconv.Itoa(i)) 1213 err = tx.LPush(bucket1, key, value) 1214 assertErr(t, err, nil) 1215 } 1216 return nil 1217 }) 1218 require.NoError(t, err) 1219 // Trigger the limit 1220 txPush(t, db, bucket1, []byte("0"), []byte("value1"), false, nil, ErrTxnExceedWriteLimit) 1221 // Test LRem 1222 err = db.Update(func(tx *Tx) error { 1223 err := tx.LRem(bucket1, []byte("0"), 1, []byte("0")) 1224 assertErr(t, err, nil) 1225 return nil 1226 }) 1227 require.NoError(t, err) 1228 txPush(t, db, bucket1, []byte("0"), []byte("value1"), true, nil, nil) 1229 txPush(t, db, bucket1, []byte("0"), []byte("value1"), false, nil, ErrTxnExceedWriteLimit) 1230 // Test for DataLPopFlag 1231 err = db.Update(func(tx *Tx) error { 1232 _, err := tx.LPop(bucket1, []byte("0")) 1233 assertErr(t, err, nil) 1234 return nil 1235 }) 1236 require.NoError(t, err) 1237 txPush(t, db, bucket1, []byte("0"), []byte("value1"), false, nil, nil) 1238 txPush(t, db, bucket1, []byte("0"), []byte("value1"), false, nil, ErrTxnExceedWriteLimit) 1239 // Test for DataLTrimFlag 1240 err = db.Update(func(tx *Tx) error { 1241 err := tx.LTrim(bucket1, []byte("0"), 0, 0) 1242 assertErr(t, err, nil) 1243 return nil 1244 }) 1245 require.NoError(t, err) 1246 err = db.Update(func(tx *Tx) error { 1247 for i := 0; i < int(limitCount)-2; i++ { 1248 key := []byte("0") 1249 value := []byte(strconv.Itoa(i)) 1250 err = tx.RPush(bucket1, key, value) 1251 assertErr(t, err, nil) 1252 } 1253 return nil 1254 }) 1255 require.NoError(t, err) 1256 txPush(t, db, bucket1, []byte("0"), []byte("value11"), false, nil, nil) 1257 txPush(t, db, bucket1, []byte("0"), []byte("value11"), false, nil, ErrTxnExceedWriteLimit) 1258 // Test for LRemByIndex 1259 err = db.Update(func(tx *Tx) error { 1260 err := tx.LRemByIndex(bucket1, []byte("0"), 0, 1, 2) 1261 assertErr(t, err, nil) 1262 return nil 1263 }) 1264 require.NoError(t, err) 1265 err = db.Update(func(tx *Tx) error { 1266 for i := 0; i < 2; i++ { 1267 key := []byte("0") 1268 value := []byte(strconv.Itoa(i)) 1269 err = tx.RPush(bucket1, key, value) 1270 assertErr(t, err, nil) 1271 } 1272 return nil 1273 }) 1274 require.NoError(t, err) 1275 txPush(t, db, bucket2, []byte("0"), []byte("value11"), false, nil, nil) 1276 txPush(t, db, bucket1, []byte("0"), []byte("value11"), false, nil, ErrTxnExceedWriteLimit) 1277 // Delete bucket 1278 txDeleteBucket(t, db, DataStructureList, bucket1, nil) 1279 // Add data to another bucket 1280 err = db.Update(func(tx *Tx) error { 1281 for i := 0; i < int(limitCount)-1; i++ { 1282 key := []byte(strconv.Itoa(i)) 1283 value := []byte(strconv.Itoa(i)) 1284 err = tx.RPush(bucket2, key, value) 1285 assertErr(t, err, nil) 1286 } 1287 return nil 1288 }) 1289 require.NoError(t, err) 1290 txPush(t, db, bucket2, []byte("key1"), []byte("value1"), false, nil, ErrTxnExceedWriteLimit) 1291 }) 1292 } 1293 } 1294 1295 func TestDB_DataStructureSetWriteRecordLimit(t *testing.T) { 1296 // Set default options and limitCount. 1297 opts := DefaultOptions 1298 limitCount := int64(1000) 1299 opts.MaxWriteRecordCount = limitCount 1300 // Define bucket names. 1301 bucket1 := "bucket1" 1302 bucket2 := "bucket2" 1303 // Loop through EntryIdxModes. 1304 for _, idxMode := range []EntryIdxMode{HintKeyValAndRAMIdxMode, HintKeyAndRAMIdxMode} { 1305 opts.EntryIdxMode = idxMode 1306 runNutsDBTest(t, &opts, func(t *testing.T, db *DB) { 1307 txCreateBucket(t, db, DataStructureSet, bucket1, nil) 1308 txCreateBucket(t, db, DataStructureSet, bucket2, nil) 1309 1310 // Add limitCount records to bucket1. 1311 err := db.Update(func(tx *Tx) error { 1312 for i := 0; i < int(limitCount); i++ { 1313 key := []byte("0") 1314 value := []byte(strconv.Itoa(i)) 1315 err := tx.SAdd(bucket1, key, value) 1316 assertErr(t, err, nil) 1317 } 1318 return nil 1319 }) 1320 require.NoError(t, err) 1321 // Try to add one more item to bucket1 and check for ErrTxnExceedWriteLimit. 1322 txSAdd(t, db, bucket1, []byte("key1"), []byte("value1"), nil, ErrTxnExceedWriteLimit) 1323 // Remove one item and add another item to bucket1. 1324 txSRem(t, db, bucket1, []byte("0"), []byte("0"), nil) 1325 txSAdd(t, db, bucket1, []byte("key1"), []byte("value1"), nil, nil) 1326 // Add two more items to bucket1 and check for ErrTxnExceedWriteLimit. 1327 txSAdd(t, db, bucket1, []byte("key1"), []byte("value1"), nil, nil) 1328 txSAdd(t, db, bucket1, []byte("key11"), []byte("value11"), nil, ErrTxnExceedWriteLimit) 1329 // Test for SPOP, SPOP two items from bucket1. 1330 err = db.Update(func(tx *Tx) error { 1331 _, err := tx.SPop(bucket1, []byte("0")) 1332 assertErr(t, err, nil) 1333 _, err = tx.SPop(bucket1, []byte("key1")) 1334 assertErr(t, err, nil) 1335 return nil 1336 }) 1337 require.NoError(t, err) 1338 // Add two items to bucket1 and check for ErrTxnExceedWriteLimit. 1339 txSAdd(t, db, bucket1, []byte("1"), []byte("value1"), nil, nil) 1340 txSAdd(t, db, bucket1, []byte("1"), []byte("value2"), nil, nil) 1341 txSAdd(t, db, bucket1, []byte("1"), []byte("value3"), nil, ErrTxnExceedWriteLimit) 1342 // Delete bucket1. 1343 txDeleteBucket(t, db, DataStructureSet, bucket1, nil) 1344 // Add data to bucket2. 1345 txSAdd(t, db, bucket2, []byte("key1"), []byte("value1"), nil, nil) 1346 err = db.Update(func(tx *Tx) error { 1347 for i := 0; i < int(limitCount)-1; i++ { 1348 value := []byte(strconv.Itoa(i)) 1349 err = tx.SAdd(bucket2, []byte("2"), value) 1350 assertErr(t, err, nil) 1351 } 1352 return nil 1353 }) 1354 require.NoError(t, err) 1355 // Try to add one more item to bucket2 and check for ErrTxnExceedWriteLimit. 1356 txSAdd(t, db, bucket2, []byte("key2"), []byte("value2"), nil, ErrTxnExceedWriteLimit) 1357 }) 1358 } 1359 } 1360 1361 func TestDB_DataStructureSortedSetWriteRecordLimit(t *testing.T) { 1362 // Set up options 1363 opts := DefaultOptions 1364 limitCount := int64(1000) 1365 opts.MaxWriteRecordCount = limitCount 1366 // Set up bucket names and score 1367 bucket1 := "bucket1" 1368 bucket2 := "bucket2" 1369 score := 1.0 1370 // Iterate over EntryIdxMode options 1371 for _, idxMode := range []EntryIdxMode{HintKeyValAndRAMIdxMode, HintKeyAndRAMIdxMode} { 1372 opts.EntryIdxMode = idxMode 1373 runNutsDBTest(t, &opts, func(t *testing.T, db *DB) { 1374 txCreateBucket(t, db, DataStructureSortedSet, bucket1, nil) 1375 // Add limitCount records 1376 err := db.Update(func(tx *Tx) error { 1377 for i := 0; i < int(limitCount); i++ { 1378 key := []byte("0") 1379 value := []byte(strconv.Itoa(i)) 1380 err := tx.ZAdd(bucket1, key, score+float64(i), value) 1381 assertErr(t, err, nil) 1382 } 1383 return nil 1384 }) 1385 require.NoError(t, err) 1386 // Trigger the limit 1387 txZAdd(t, db, bucket1, []byte("key1"), []byte("value1"), score, nil, ErrTxnExceedWriteLimit) 1388 // Delete and add one item 1389 txZRem(t, db, bucket1, []byte("0"), []byte("0"), nil) 1390 txZAdd(t, db, bucket1, []byte("key1"), []byte("value1"), score, nil, nil) 1391 // Add some data is ok 1392 txZAdd(t, db, bucket1, []byte("key1"), []byte("value1"), score, nil, nil) 1393 // Trigger the limit 1394 txZAdd(t, db, bucket1, []byte("key2"), []byte("value2"), score, nil, ErrTxnExceedWriteLimit) 1395 // Test for ZRemRangeByRank 1396 err = db.Update(func(tx *Tx) error { 1397 err := tx.ZRemRangeByRank(bucket1, []byte("0"), 1, 3) 1398 assert.NoError(t, err) 1399 return nil 1400 }) 1401 assert.NoError(t, err) 1402 txZAdd(t, db, bucket1, []byte("0"), []byte("value1"), score, nil, nil) 1403 txZAdd(t, db, bucket1, []byte("0"), []byte("value2"), score, nil, nil) 1404 txZAdd(t, db, bucket1, []byte("0"), []byte("value3"), score+float64(1000), nil, nil) 1405 // Trigger the limit 1406 txZAdd(t, db, bucket1, []byte("0"), []byte("value4"), score, nil, ErrTxnExceedWriteLimit) 1407 // Test for ZPop 1408 txZPop(t, db, bucket1, []byte("0"), true, []byte("value3"), score+float64(1000), nil) 1409 txZAdd(t, db, bucket1, []byte("key3"), []byte("value3"), score, nil, nil) 1410 // Delete bucket 1411 txDeleteBucket(t, db, DataStructureSortedSet, bucket1, nil) 1412 // Add data to another bucket 1413 txCreateBucket(t, db, DataStructureSortedSet, bucket1, nil) 1414 txCreateBucket(t, db, DataStructureSortedSet, bucket2, nil) 1415 txZAdd(t, db, bucket2, []byte("key1"), []byte("value1"), score, nil, nil) 1416 // Add data to bucket1 1417 err = db.Update(func(tx *Tx) error { 1418 for i := 0; i < int(limitCount)-1; i++ { 1419 key := []byte(strconv.Itoa(i)) 1420 value := []byte(strconv.Itoa(i)) 1421 err = tx.ZAdd(bucket1, key, score, value) 1422 assertErr(t, err, nil) 1423 } 1424 return nil 1425 }) 1426 require.NoError(t, err) 1427 // Trigger the limit 1428 txZAdd(t, db, bucket2, []byte("key1"), []byte("value2"), score, nil, ErrTxnExceedWriteLimit) 1429 }) 1430 } 1431 } 1432 1433 func TestDB_AllDsWriteRecordLimit(t *testing.T) { 1434 // Set up options 1435 opts := DefaultOptions 1436 limitCount := int64(1000) 1437 opts.MaxWriteRecordCount = limitCount 1438 // Set up bucket names and score 1439 bucket1 := "bucket1" 1440 bucket2 := "bucket2" 1441 score := 1.0 1442 // Iterate over EntryIdxMode options 1443 for _, idxMode := range []EntryIdxMode{HintKeyValAndRAMIdxMode, HintKeyAndRAMIdxMode} { 1444 opts.EntryIdxMode = idxMode 1445 runNutsDBTest(t, &opts, func(t *testing.T, db *DB) { 1446 txCreateBucket(t, db, DataStructureBTree, bucket1, nil) 1447 txCreateBucket(t, db, DataStructureList, bucket1, nil) 1448 txCreateBucket(t, db, DataStructureSet, bucket1, nil) 1449 txCreateBucket(t, db, DataStructureSortedSet, bucket1, nil) 1450 txCreateBucket(t, db, DataStructureList, bucket2, nil) 1451 1452 // Add limitCount records 1453 err := db.Update(func(tx *Tx) error { 1454 for i := 0; i < int(limitCount); i++ { 1455 key := []byte(strconv.Itoa(i)) 1456 value := []byte(strconv.Itoa(i)) 1457 err = tx.Put(bucket1, key, value, Persistent) 1458 assertErr(t, err, nil) 1459 } 1460 return nil 1461 }) 1462 require.NoError(t, err) 1463 // Trigger the limit 1464 txPush(t, db, bucket1, []byte("0"), []byte("value1"), false, nil, ErrTxnExceedWriteLimit) 1465 // Delete item and add one 1466 txDel(t, db, bucket1, []byte("0"), nil) 1467 txPush(t, db, bucket1, []byte("0"), []byte("value1"), false, nil, nil) 1468 // Trigger the limit 1469 txSAdd(t, db, bucket1, []byte("key1"), []byte("value1"), nil, ErrTxnExceedWriteLimit) 1470 // Delete item and add one 1471 txDel(t, db, bucket1, []byte("1"), nil) 1472 txSAdd(t, db, bucket1, []byte("key1"), []byte("value1"), nil, nil) 1473 // Trigger the limit 1474 txZAdd(t, db, bucket1, []byte("key1"), []byte("value1"), score, nil, ErrTxnExceedWriteLimit) 1475 // Delete item and add one 1476 txDel(t, db, bucket1, []byte("2"), nil) 1477 txZAdd(t, db, bucket1, []byte("key1"), []byte("value1"), score, nil, nil) 1478 // Delete bucket 1479 txDeleteBucket(t, db, DataStructureSortedSet, bucket1, nil) 1480 // Add data to another bucket 1481 txPush(t, db, bucket2, []byte("key1"), []byte("value1"), false, nil, nil) 1482 // Trigger the limit 1483 txPush(t, db, bucket2, []byte("key2"), []byte("value2"), false, nil, ErrTxnExceedWriteLimit) 1484 }) 1485 } 1486 } 1487 1488 func txIncrement(t *testing.T, db *DB, bucket string, key []byte, expectErr error, finalExpectErr error) { 1489 err := db.Update(func(tx *Tx) error { 1490 err := tx.Incr(bucket, key) 1491 assertErr(t, err, expectErr) 1492 return nil 1493 }) 1494 assertErr(t, err, finalExpectErr) 1495 } 1496 1497 func txDecrement(t *testing.T, db *DB, bucket string, key []byte, expectErr error, finalExpectErr error) { 1498 err := db.Update(func(tx *Tx) error { 1499 err := tx.Decr(bucket, key) 1500 assertErr(t, err, expectErr) 1501 return nil 1502 }) 1503 assertErr(t, err, finalExpectErr) 1504 } 1505 1506 func txIncrementBy(t *testing.T, db *DB, bucket string, key []byte, value int64, expectErr error, finalExpectErr error) { 1507 err := db.Update(func(tx *Tx) error { 1508 err := tx.IncrBy(bucket, key, value) 1509 assertErr(t, err, expectErr) 1510 return nil 1511 }) 1512 assertErr(t, err, finalExpectErr) 1513 } 1514 1515 func txDecrementBy(t *testing.T, db *DB, bucket string, key []byte, value int64, expectErr error, finalExpectErr error) { 1516 err := db.Update(func(tx *Tx) error { 1517 err := tx.DecrBy(bucket, key, value) 1518 assertErr(t, err, expectErr) 1519 return nil 1520 }) 1521 assertErr(t, err, finalExpectErr) 1522 } 1523 1524 func txPutIfNotExists(t *testing.T, db *DB, bucket string, key, value []byte, expectedErr, finalExpectErr error) { 1525 err := db.Update(func(tx *Tx) error { 1526 err := tx.PutIfNotExists(bucket, key, value, Persistent) 1527 assertErr(t, err, expectedErr) 1528 return nil 1529 }) 1530 assertErr(t, err, finalExpectErr) 1531 } 1532 1533 func txPutIfExists(t *testing.T, db *DB, bucket string, key, value []byte, expectedErr, finalExpectErr error) { 1534 err := db.Update(func(tx *Tx) error { 1535 err := tx.PutIfExists(bucket, key, value, Persistent) 1536 assertErr(t, err, expectedErr) 1537 return nil 1538 }) 1539 assertErr(t, err, finalExpectErr) 1540 } 1541 1542 func txValueLen(t *testing.T, db *DB, bucket string, key []byte, expectLength int, expectErr error) { 1543 err := db.View(func(tx *Tx) error { 1544 length, err := tx.ValueLen(bucket, key) 1545 if expectErr != nil { 1546 require.Equal(t, expectErr, err) 1547 } else { 1548 require.NoError(t, err) 1549 } 1550 require.EqualValuesf(t, expectLength, length, "err Tx ValueLen. got %s want %s", length, expectLength) 1551 return nil 1552 }) 1553 require.NoError(t, err) 1554 } 1555 1556 func txGetSet(t *testing.T, db *DB, bucket string, key, value []byte, expectOldValue []byte, expectErr error) { 1557 err := db.Update(func(tx *Tx) error { 1558 oldValue, err := tx.GetSet(bucket, key, value) 1559 assertErr(t, err, expectErr) 1560 require.EqualValuesf(t, oldValue, expectOldValue, "err Tx GetSet. got %s want %s", string(oldValue), string(expectOldValue)) 1561 return nil 1562 }) 1563 require.NoError(t, err) 1564 } 1565 1566 func txGetBit(t *testing.T, db *DB, bucket string, key []byte, offset int, expectVal byte, expectErr error, finalExpectErr error) { 1567 err := db.View(func(tx *Tx) error { 1568 value, err := tx.GetBit(bucket, key, offset) 1569 assertErr(t, err, expectErr) 1570 require.Equal(t, expectVal, value) 1571 return nil 1572 }) 1573 assertErr(t, err, finalExpectErr) 1574 } 1575 1576 func txSetBit(t *testing.T, db *DB, bucket string, key []byte, offset int, value byte, expectErr error, finalExpectErr error) { 1577 err := db.Update(func(tx *Tx) error { 1578 err := tx.SetBit(bucket, key, offset, value) 1579 assertErr(t, err, expectErr) 1580 return nil 1581 }) 1582 assertErr(t, err, finalExpectErr) 1583 } 1584 1585 func txGetTTL(t *testing.T, db *DB, bucket string, key []byte, expectedTTL int64, expectedErr error) { 1586 err := db.View(func(tx *Tx) error { 1587 ttl, err := tx.GetTTL(bucket, key) 1588 assertErr(t, err, expectedErr) 1589 1590 // If diff between expectedTTL and realTTL lesser than 1s, We'll consider as equal 1591 diff := int(math.Abs(float64(ttl - expectedTTL))) 1592 assert.LessOrEqual(t, diff, 1) 1593 return nil 1594 }) 1595 require.NoError(t, err) 1596 } 1597 1598 func txPersist(t *testing.T, db *DB, bucket string, key []byte, expectedErr error) { 1599 err := db.Update(func(tx *Tx) error { 1600 err := tx.Persist(bucket, key) 1601 assertErr(t, err, expectedErr) 1602 return nil 1603 }) 1604 require.NoError(t, err) 1605 } 1606 1607 func txMSet(t *testing.T, db *DB, bucket string, args [][]byte, ttl uint32, expectErr error, finalExpectErr error) { 1608 err := db.Update(func(tx *Tx) error { 1609 err := tx.MSet(bucket, ttl, args...) 1610 assertErr(t, err, expectErr) 1611 return nil 1612 }) 1613 assertErr(t, err, finalExpectErr) 1614 } 1615 1616 func txMGet(t *testing.T, db *DB, bucket string, keys [][]byte, expectValues [][]byte, expectErr error, finalExpectErr error) { 1617 err := db.View(func(tx *Tx) error { 1618 values, err := tx.MGet(bucket, keys...) 1619 assertErr(t, err, expectErr) 1620 require.EqualValues(t, expectValues, values) 1621 return nil 1622 }) 1623 assertErr(t, err, finalExpectErr) 1624 } 1625 1626 func txAppend(t *testing.T, db *DB, bucket string, key, appendage []byte, expectErr error, expectFinalErr error) { 1627 err := db.Update(func(tx *Tx) error { 1628 err := tx.Append(bucket, key, appendage) 1629 assertErr(t, err, expectErr) 1630 return nil 1631 }) 1632 assertErr(t, err, expectFinalErr) 1633 } 1634 1635 func txGetRange(t *testing.T, db *DB, bucket string, key []byte, start, end int, expectVal []byte, expectErr error, expectFinalErr error) { 1636 err := db.View(func(tx *Tx) error { 1637 value, err := tx.GetRange(bucket, key, start, end) 1638 assertErr(t, err, expectErr) 1639 require.EqualValues(t, expectVal, value) 1640 return nil 1641 }) 1642 assertErr(t, err, expectFinalErr) 1643 }