github.com/zuoyebang/bitalosdb@v1.1.1-0.20240516111551-79a8c4d8ce20/db_test.go (about) 1 // Copyright 2021 The Bitalosdb author(hustxrb@163.com) and other contributors. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package bitalosdb 16 17 import ( 18 "bytes" 19 "errors" 20 "fmt" 21 "os" 22 "strings" 23 "sync" 24 "testing" 25 "time" 26 27 "github.com/zuoyebang/bitalosdb/internal/vfs" 28 29 "github.com/stretchr/testify/require" 30 "golang.org/x/exp/rand" 31 ) 32 33 func try(initialSleep, maxTotalSleep time.Duration, f func() error) error { 34 totalSleep := time.Duration(0) 35 for d := initialSleep; ; d *= 2 { 36 time.Sleep(d) 37 totalSleep += d 38 if err := f(); err == nil || totalSleep >= maxTotalSleep { 39 return err 40 } 41 } 42 } 43 44 func TestTry(t *testing.T) { 45 c := make(chan struct{}) 46 go func() { 47 time.Sleep(1 * time.Millisecond) 48 close(c) 49 }() 50 51 attemptsMu := sync.Mutex{} 52 attempts := 0 53 54 err := try(100*time.Microsecond, 20*time.Second, func() error { 55 attemptsMu.Lock() 56 attempts++ 57 attemptsMu.Unlock() 58 59 select { 60 default: 61 return errors.New("timed out") 62 case <-c: 63 return nil 64 } 65 }) 66 require.NoError(t, err) 67 68 attemptsMu.Lock() 69 a := attempts 70 attemptsMu.Unlock() 71 72 if a == 0 { 73 t.Fatalf("attempts: got 0, want > 0") 74 } 75 } 76 77 func TestBasicBitowerWrites(t *testing.T) { 78 dir := testDirname 79 defer os.RemoveAll(dir) 80 os.RemoveAll(dir) 81 82 db := openTestDB(testDirname, nil) 83 defer func() { 84 require.NoError(t, db.Close()) 85 }() 86 87 names := []string{ 88 "Alatar", 89 "Gandalf", 90 "Pallando", 91 "Radagast", 92 "Saruman", 93 "Joe", 94 } 95 wantMap := map[string]string{} 96 97 batchNew := func() *BatchBitower { 98 return newBatchBitowerByIndex(db, int(testSlotId)) 99 } 100 101 inBatch, batch, pending := false, batchNew(), [][]string(nil) 102 set0 := func(k, v string) error { 103 return db.Set(makeTestSlotKey([]byte(k)), []byte(v), nil) 104 } 105 del0 := func(k string) error { 106 return db.Delete(makeTestSlotKey([]byte(k)), nil) 107 } 108 set1 := func(k, v string) error { 109 batch.Set(makeTestSlotKey([]byte(k)), []byte(v), nil) 110 return nil 111 } 112 del1 := func(k string) error { 113 batch.Delete(makeTestSlotKey([]byte(k)), nil) 114 return nil 115 } 116 set, del := set0, del0 117 118 testCases := []string{ 119 "set Gandalf Grey", 120 "set Saruman White", 121 "set Radagast Brown", 122 "delete Saruman", 123 "set Gandalf White", 124 "batch", 125 " set Alatar AliceBlue", 126 "apply", 127 "delete Pallando", 128 "set Alatar AntiqueWhite", 129 "set Pallando PapayaWhip", 130 "batch", 131 "apply", 132 "set Pallando PaleVioletRed", 133 "batch", 134 " delete Alatar", 135 " set Gandalf GhostWhite", 136 " set Saruman Seashell", 137 " delete Saruman", 138 " set Saruman SeaGreen", 139 " set Radagast RosyBrown", 140 " delete Pallando", 141 "apply", 142 "delete Radagast", 143 "delete Radagast", 144 "delete Radagast", 145 "set Gandalf Goldenrod", 146 "set Pallando PeachPuff", 147 "batch", 148 " delete Joe", 149 " delete Saruman", 150 " delete Radagast", 151 " delete Pallando", 152 " delete Gandalf", 153 " delete Alatar", 154 "apply", 155 "set Joe Plumber", 156 } 157 for i, tc := range testCases { 158 s := strings.Split(strings.TrimSpace(tc), " ") 159 switch s[0] { 160 case "set": 161 if err := set(s[1], s[2]); err != nil { 162 t.Fatalf("#%d %s: %v", i, tc, err) 163 } 164 if inBatch { 165 pending = append(pending, s) 166 } else { 167 wantMap[s[1]] = s[2] 168 } 169 case "delete": 170 if err := del(s[1]); err != nil { 171 t.Fatalf("#%d %s: %v", i, tc, err) 172 } 173 if inBatch { 174 pending = append(pending, s) 175 } else { 176 delete(wantMap, s[1]) 177 } 178 case "batch": 179 inBatch, batch, set, del = true, batchNew(), set1, del1 180 case "apply": 181 if err := db.ApplyBitower(batch, NoSync); err != nil { 182 t.Fatalf("#%d %s: %v", i, tc, err) 183 } 184 for _, p := range pending { 185 switch p[0] { 186 case "set": 187 wantMap[p[1]] = p[2] 188 case "delete": 189 delete(wantMap, p[1]) 190 } 191 } 192 inBatch, pending, set, del = false, nil, set0, del0 193 default: 194 t.Fatalf("#%d %s: bad test case: %q", i, tc, s) 195 } 196 197 fail := false 198 for _, name := range names { 199 key := makeTestSlotKey([]byte(name)) 200 g, closer, err := db.Get(key) 201 if err != nil && err != ErrNotFound { 202 t.Errorf("#%d %s: Get(%q): %v", i, tc, name, err) 203 fail = true 204 } 205 _, err = db.Exist(key) 206 if err != nil && err != ErrNotFound { 207 t.Errorf("#%d %s: Exist(%q): %v", i, tc, name, err) 208 fail = true 209 } 210 got, gOK := string(g), err == nil 211 want, wOK := wantMap[name] 212 if got != want || gOK != wOK { 213 t.Errorf("#%d %s: Get(%q): got %q, %t, want %q, %t", 214 i, tc, name, got, gOK, want, wOK) 215 fail = true 216 } 217 if closer != nil { 218 closer() 219 } 220 } 221 if fail { 222 return 223 } 224 } 225 } 226 227 func TestEmptyMemtable(t *testing.T) { 228 dir := testDirname 229 defer os.RemoveAll(dir) 230 os.RemoveAll(dir) 231 db, err := Open(dir, &Options{}) 232 require.NoError(t, err) 233 234 checkEmpty := func(index int) { 235 d := db.bitowers[index] 236 d.mu.Lock() 237 empty := true 238 for i := range d.mu.mem.queue { 239 if !d.mu.mem.queue[i].empty() { 240 empty = false 241 break 242 } 243 } 244 d.mu.Unlock() 245 require.Equal(t, true, empty) 246 } 247 248 for i := 0; i < len(db.bitowers); i++ { 249 checkEmpty(i) 250 } 251 252 require.NoError(t, db.Close()) 253 } 254 255 func TestNotEmptyMemtable(t *testing.T) { 256 dir := testDirname 257 defer os.RemoveAll(dir) 258 os.RemoveAll(dir) 259 db, err := Open(dir, &Options{}) 260 require.NoError(t, err) 261 262 key := makeTestKey([]byte("a")) 263 require.NoError(t, db.Set(key, key, nil)) 264 265 index := db.getBitowerIndexByKey(key) 266 d := db.bitowers[index] 267 d.mu.Lock() 268 empty := true 269 for i := range d.mu.mem.queue { 270 if !d.mu.mem.queue[i].empty() { 271 empty = false 272 break 273 } 274 } 275 d.mu.Unlock() 276 require.Equal(t, false, empty) 277 require.NoError(t, db.Close()) 278 } 279 280 func TestRandomWrites(t *testing.T) { 281 dir := testDirname 282 defer os.RemoveAll(dir) 283 os.RemoveAll(dir) 284 d, err := Open(dir, &Options{ 285 FS: vfs.Default, 286 MemTableSize: 1 << 20, 287 }) 288 require.NoError(t, err) 289 290 keys := [64][]byte{} 291 wants := [64]int{} 292 for k := range keys { 293 keys[k] = makeTestIntKey(k) 294 wants[k] = -1 295 } 296 xxx := bytes.Repeat([]byte("x"), 512) 297 298 rng := rand.New(rand.NewSource(123)) 299 const N = 1000 300 for i := 0; i < N; i++ { 301 k := rng.Intn(len(keys)) 302 if rng.Intn(20) != 0 { 303 wants[k] = rng.Intn(len(xxx) + 1) 304 if err := d.Set(keys[k], xxx[:wants[k]], nil); err != nil { 305 t.Fatalf("i=%d: Set: %v", i, err) 306 } 307 } else { 308 wants[k] = -1 309 if err := d.Delete(keys[k], nil); err != nil { 310 t.Fatalf("i=%d: Delete: %v", i, err) 311 } 312 } 313 314 if i != N-1 || rng.Intn(50) != 0 { 315 continue 316 } 317 for k := range keys { 318 got := -1 319 if v, closer, err := d.Get(keys[k]); err != nil { 320 if err != ErrNotFound { 321 t.Fatalf("Get: %v", err) 322 } 323 } else { 324 got = len(v) 325 closer() 326 } 327 exist, err := d.Exist(keys[k]) 328 if exist { 329 t.Fatalf("Exist: %v", err) 330 } 331 if got != wants[k] { 332 t.Errorf("i=%d, k=%d: got %d, want %d", i, k, got, wants[k]) 333 } 334 } 335 } 336 337 require.NoError(t, d.Close()) 338 } 339 340 func TestGetNoCache(t *testing.T) { 341 dir := testDirname 342 defer os.RemoveAll(dir) 343 os.RemoveAll(dir) 344 d, err := Open(dir, &Options{ 345 CacheSize: 0, 346 FS: vfs.Default, 347 }) 348 require.NoError(t, err) 349 350 key := makeTestKey([]byte("a")) 351 require.NoError(t, d.Set(key, []byte("aa"), nil)) 352 require.NoError(t, d.Flush()) 353 require.NoError(t, verifyGet(d, key, []byte("aa"))) 354 require.NoError(t, d.Close()) 355 } 356 357 func TestLogData(t *testing.T) { 358 defer os.RemoveAll(testDirname) 359 os.RemoveAll(testDirname) 360 361 db := openTestDB(testDirname, nil) 362 val := testRandBytes(10) 363 for i := 0; i < 100; i++ { 364 require.NoError(t, db.Set(makeTestIntKey(i), val, NoSync)) 365 } 366 for i := range db.bitowers { 367 require.NoError(t, db.LogData([]byte("foo"), i, Sync)) 368 } 369 require.NoError(t, db.Close()) 370 371 db = openTestDB(testDirname, nil) 372 for i := 0; i < 100; i++ { 373 require.NoError(t, verifyGet(db, makeTestIntKey(i), val)) 374 } 375 require.NoError(t, db.Close()) 376 } 377 378 func TestDeleteGet(t *testing.T) { 379 dir := testDirname 380 defer os.RemoveAll(dir) 381 os.RemoveAll(dir) 382 d, err := Open(dir, &Options{}) 383 require.NoError(t, err) 384 385 key := makeTestKey([]byte("key")) 386 val := []byte("val") 387 388 require.NoError(t, d.Set(key, val, nil)) 389 require.NoError(t, verifyGet(d, key, val)) 390 391 key2 := makeTestKey([]byte("key2")) 392 val2 := []byte("val2") 393 394 require.NoError(t, d.Set(key2, val2, nil)) 395 require.NoError(t, verifyGet(d, key2, val2)) 396 397 require.NoError(t, d.Delete(key2, nil)) 398 require.NoError(t, verifyGetNotFound(d, key2)) 399 400 require.NoError(t, d.Close()) 401 } 402 403 func TestDeleteFlush(t *testing.T) { 404 dir := testDirname 405 defer os.RemoveAll(dir) 406 os.RemoveAll(dir) 407 d, err := Open(dir, &Options{ 408 FS: vfs.Default, 409 }) 410 require.NoError(t, err) 411 defer func() { 412 require.NoError(t, d.Close()) 413 }() 414 415 key := makeTestKey([]byte("key")) 416 valFirst := []byte("first") 417 valSecond := []byte("second") 418 key2 := makeTestKey([]byte("key2")) 419 val2 := []byte("val2") 420 421 require.NoError(t, d.Set(key, valFirst, nil)) 422 require.NoError(t, d.Set(key2, val2, nil)) 423 require.NoError(t, d.Flush()) 424 425 require.NoError(t, d.Set(key, valSecond, nil)) 426 require.NoError(t, d.Delete(key2, nil)) 427 require.NoError(t, d.Set(key2, val2, nil)) 428 require.NoError(t, d.Flush()) 429 430 require.NoError(t, d.Delete(key, nil)) 431 require.NoError(t, d.Delete(key2, nil)) 432 require.NoError(t, d.Flush()) 433 434 require.NoError(t, verifyGetNotFound(d, key)) 435 require.NoError(t, verifyGetNotFound(d, key2)) 436 } 437 438 func TestUnremovableDelete(t *testing.T) { 439 dir := testDirname 440 defer os.RemoveAll(dir) 441 os.RemoveAll(dir) 442 d, err := Open(dir, &Options{ 443 FS: vfs.Default, 444 }) 445 require.NoError(t, err) 446 defer func() { 447 require.NoError(t, d.Close()) 448 }() 449 450 key := makeTestKey([]byte("key")) 451 valFirst := []byte("valFirst") 452 valSecond := []byte("valSecond") 453 454 require.NoError(t, d.Set(key, valFirst, nil)) 455 require.NoError(t, d.Set(key, valSecond, nil)) 456 require.NoError(t, d.Flush()) 457 458 require.NoError(t, verifyGet(d, key, valSecond)) 459 460 require.NoError(t, d.Delete(key, nil)) 461 require.NoError(t, d.Flush()) 462 require.NoError(t, verifyGetNotFound(d, key)) 463 } 464 465 func TestAsyncFlush(t *testing.T) { 466 dir := testDirname 467 defer os.RemoveAll(dir) 468 os.RemoveAll(dir) 469 db := openTestDB(testDirname, nil) 470 flushed, err := db.AsyncFlush() 471 require.NoError(t, err) 472 if flushed != nil { 473 t.Fatalf("empty flush flushed is not nil") 474 } 475 476 val := testRandBytes(100) 477 for i := 0; i < 100; i++ { 478 key := makeTestIntKey(i) 479 require.NoError(t, db.Set(key, val, NoSync)) 480 } 481 482 flushed, err = db.AsyncFlush() 483 require.NoError(t, err) 484 <-flushed 485 486 for i := 0; i < 100; i++ { 487 key := makeTestIntKey(i) 488 require.NoError(t, verifyGet(db, key, val)) 489 } 490 491 require.NoError(t, err) 492 require.NoError(t, db.Close()) 493 } 494 495 func TestAsyncFlushDataAndWrite(t *testing.T) { 496 dir := testDirname 497 defer os.RemoveAll(dir) 498 os.RemoveAll(dir) 499 d, err := Open(dir, &Options{}) 500 require.NoError(t, err) 501 502 d.Set(makeTestKey([]byte("a")), []byte("100"), nil) 503 d.Set(makeTestKey([]byte("b")), []byte("200"), nil) 504 505 var val []byte 506 var closer func() 507 for _, k := range [][]byte{[]byte("a"), []byte("b")} { 508 val, closer, err = d.Get(makeTestKey(k)) 509 if closer != nil { 510 closer() 511 } 512 } 513 514 require.NoError(t, d.Flush()) 515 516 d.Set(makeTestKey([]byte("c")), []byte("300"), nil) 517 d.Set(makeTestKey([]byte("d")), []byte("400"), nil) 518 519 for _, k := range [][]byte{[]byte("a"), []byte("b"), []byte("c"), []byte("d")} { 520 val, closer, err = d.Get(makeTestKey(k)) 521 fmt.Println(string(k), string(val)) 522 if closer != nil { 523 closer() 524 } 525 } 526 527 require.NoError(t, err) 528 require.NoError(t, d.Close()) 529 } 530 531 func TestFlushEmpty(t *testing.T) { 532 dir := testDirname 533 defer os.RemoveAll(dir) 534 os.RemoveAll(dir) 535 d, err := Open(dir, &Options{}) 536 require.NoError(t, err) 537 538 require.NoError(t, d.Flush()) 539 require.NoError(t, d.Close()) 540 } 541 542 func TestDBConcurrentCommitCompactFlush(t *testing.T) { 543 for _, disableWAL := range []bool{false, true} { 544 t.Run(fmt.Sprintf("disableWAL=%t", disableWAL), func(t *testing.T) { 545 dir := testDirname 546 defer os.RemoveAll(dir) 547 os.RemoveAll(dir) 548 d, err := Open(dir, &Options{ 549 FS: vfs.Default, 550 DisableWAL: disableWAL, 551 }) 552 require.NoError(t, err) 553 554 const n = 100 555 var wg sync.WaitGroup 556 wg.Add(n) 557 for i := 0; i < n; i++ { 558 go func(i int) { 559 defer wg.Done() 560 _ = d.Set(makeTestIntKey(i), nil, NoSync) 561 var err error 562 switch i % 2 { 563 case 0: 564 err = d.Flush() 565 case 1: 566 _, err = d.AsyncFlush() 567 } 568 require.NoError(t, err) 569 }(i) 570 } 571 wg.Wait() 572 573 require.NoError(t, d.Close()) 574 }) 575 } 576 } 577 578 func TestDBApplyBatchMismatch(t *testing.T) { 579 dir := testDirname 580 defer os.RemoveAll(dir) 581 os.RemoveAll(dir) 582 srcDB, err := Open(dir, &Options{ 583 FS: vfs.Default, 584 }) 585 require.NoError(t, err) 586 587 dir1 := testDirname + "1" 588 defer os.RemoveAll(dir1) 589 os.RemoveAll(dir1) 590 applyDB, err := Open(dir1, &Options{ 591 FS: vfs.Default, 592 }) 593 require.NoError(t, err) 594 595 b := srcDB.NewBatch() 596 b.Set(makeTestKey([]byte("test")), nil, nil) 597 598 err = applyDB.Apply(b, nil) 599 if err == nil || !strings.Contains(err.Error(), "batch db mismatch:") { 600 t.Fatalf("expected error, but found %v", err) 601 } 602 603 require.NoError(t, srcDB.Close()) 604 require.NoError(t, applyDB.Close()) 605 } 606 607 func TestMetaFlushBitable(t *testing.T) { 608 defer os.RemoveAll(testDirname) 609 os.RemoveAll(testDirname) 610 611 testOptsUseBitable = true 612 db := openTestDB(testDirname, nil) 613 require.Equal(t, uint8(0), db.meta.meta.GetFieldFlushedBitable()) 614 require.Equal(t, false, db.isFlushedBitable()) 615 db.setFlushedBitable() 616 require.Equal(t, uint8(1), db.meta.meta.GetFieldFlushedBitable()) 617 require.Equal(t, true, db.isFlushedBitable()) 618 require.NoError(t, db.Close()) 619 testOptsUseBitable = true 620 db = openTestDB(testDirname, nil) 621 require.Equal(t, uint8(1), db.meta.meta.GetFieldFlushedBitable()) 622 require.Equal(t, true, db.isFlushedBitable()) 623 require.NoError(t, db.Close()) 624 }