github.com/petermattis/pebble@v0.0.0-20190905164901-ab51a2166067/db_test.go (about) 1 // Copyright 2012 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 // of this source code is governed by a BSD-style license that can be found in 3 // the LICENSE file. 4 5 package pebble 6 7 import ( 8 "bytes" 9 "errors" 10 "fmt" 11 "io" 12 "os" 13 "strconv" 14 "strings" 15 "sync" 16 "testing" 17 "time" 18 19 "github.com/petermattis/pebble/cache" 20 "github.com/petermattis/pebble/internal/base" 21 "github.com/petermattis/pebble/vfs" 22 "github.com/stretchr/testify/require" 23 "golang.org/x/exp/rand" 24 ) 25 26 // try repeatedly calls f, sleeping between calls with exponential back-off, 27 // until f returns a nil error or the total sleep time is greater than or equal 28 // to maxTotalSleep. It always calls f at least once. 29 func try(initialSleep, maxTotalSleep time.Duration, f func() error) error { 30 totalSleep := time.Duration(0) 31 for d := initialSleep; ; d *= 2 { 32 time.Sleep(d) 33 totalSleep += d 34 if err := f(); err == nil || totalSleep >= maxTotalSleep { 35 return err 36 } 37 } 38 } 39 40 func TestTry(t *testing.T) { 41 c := make(chan struct{}) 42 go func() { 43 time.Sleep(1 * time.Millisecond) 44 close(c) 45 }() 46 47 attemptsMu := sync.Mutex{} 48 attempts := 0 49 50 err := try(100*time.Microsecond, 20*time.Second, func() error { 51 attemptsMu.Lock() 52 attempts++ 53 attemptsMu.Unlock() 54 55 select { 56 default: 57 return errors.New("timed out") 58 case <-c: 59 return nil 60 } 61 }) 62 if err != nil { 63 t.Fatal(err) 64 } 65 66 attemptsMu.Lock() 67 a := attempts 68 attemptsMu.Unlock() 69 70 if a == 0 { 71 t.Fatalf("attempts: got 0, want > 0") 72 } 73 } 74 75 // cloneFileSystem returns a new memory-backed file system whose root contains 76 // a copy of the directory dirname in the source file system srcFS. The copy 77 // is not recursive; directories under dirname are not copied. 78 // 79 // Changes to the resultant file system do not modify the source file system. 80 // 81 // For example, if srcFS contained: 82 // - /bar 83 // - /baz/0 84 // - /foo/x 85 // - /foo/y 86 // - /foo/z/A 87 // - /foo/z/B 88 // then calling cloneFileSystem(srcFS, "/foo") would result in a file system 89 // containing: 90 // - /x 91 // - /y 92 func cloneFileSystem(srcFS vfs.FS, dirname string) (vfs.FS, error) { 93 if len(dirname) == 0 || dirname[len(dirname)-1] != os.PathSeparator { 94 dirname += string(os.PathSeparator) 95 } 96 97 dstFS := vfs.NewMem() 98 list, err := srcFS.List(dirname) 99 if err != nil { 100 return nil, err 101 } 102 for _, name := range list { 103 srcFile, err := srcFS.Open(dirname + name) 104 if err != nil { 105 return nil, err 106 } 107 stat, err := srcFile.Stat() 108 if err != nil { 109 return nil, err 110 } 111 if stat.IsDir() { 112 err = srcFile.Close() 113 if err != nil { 114 return nil, err 115 } 116 continue 117 } 118 data := make([]byte, stat.Size()) 119 _, err = io.ReadFull(srcFile, data) 120 if err != nil { 121 return nil, err 122 } 123 err = srcFile.Close() 124 if err != nil { 125 return nil, err 126 } 127 dstFile, err := dstFS.Create(name) 128 if err != nil { 129 return nil, err 130 } 131 _, err = dstFile.Write(data) 132 if err != nil { 133 return nil, err 134 } 135 err = dstFile.Close() 136 if err != nil { 137 return nil, err 138 } 139 } 140 return dstFS, nil 141 } 142 143 func TestBasicReads(t *testing.T) { 144 testCases := []struct { 145 dirname string 146 wantMap map[string]string 147 }{ 148 { 149 "db-stage-1", 150 map[string]string{ 151 "aaa": "", 152 "bar": "", 153 "baz": "", 154 "foo": "", 155 "quux": "", 156 "zzz": "", 157 }, 158 }, 159 { 160 "db-stage-2", 161 map[string]string{ 162 "aaa": "", 163 "bar": "", 164 "baz": "three", 165 "foo": "four", 166 "quux": "", 167 "zzz": "", 168 }, 169 }, 170 { 171 "db-stage-3", 172 map[string]string{ 173 "aaa": "", 174 "bar": "", 175 "baz": "three", 176 "foo": "four", 177 "quux": "", 178 "zzz": "", 179 }, 180 }, 181 { 182 "db-stage-4", 183 map[string]string{ 184 "aaa": "", 185 "bar": "", 186 "baz": "", 187 "foo": "five", 188 "quux": "six", 189 "zzz": "", 190 }, 191 }, 192 } 193 for _, tc := range testCases { 194 fs, err := cloneFileSystem(vfs.Default, "testdata/"+tc.dirname) 195 if err != nil { 196 t.Errorf("%s: cloneFileSystem failed: %v", tc.dirname, err) 197 continue 198 } 199 d, err := Open("", &Options{ 200 FS: fs, 201 }) 202 if err != nil { 203 t.Errorf("%s: Open failed: %v", tc.dirname, err) 204 continue 205 } 206 for key, want := range tc.wantMap { 207 got, err := d.Get([]byte(key)) 208 if err != nil && err != ErrNotFound { 209 t.Errorf("%s: Get(%q) failed: %v", tc.dirname, key, err) 210 continue 211 } 212 if string(got) != string(want) { 213 t.Errorf("%s: Get(%q): got %q, want %q", tc.dirname, key, got, want) 214 continue 215 } 216 } 217 err = d.Close() 218 if err != nil { 219 t.Errorf("%s: Close failed: %v", tc.dirname, err) 220 continue 221 } 222 } 223 } 224 225 func TestBasicWrites(t *testing.T) { 226 d, err := Open("", &Options{ 227 FS: vfs.NewMem(), 228 }) 229 if err != nil { 230 t.Fatal(err) 231 } 232 233 names := []string{ 234 "Alatar", 235 "Gandalf", 236 "Pallando", 237 "Radagast", 238 "Saruman", 239 "Joe", 240 } 241 wantMap := map[string]string{} 242 243 inBatch, batch, pending := false, &Batch{}, [][]string(nil) 244 set0 := func(k, v string) error { 245 return d.Set([]byte(k), []byte(v), nil) 246 } 247 del0 := func(k string) error { 248 return d.Delete([]byte(k), nil) 249 } 250 set1 := func(k, v string) error { 251 batch.Set([]byte(k), []byte(v), nil) 252 return nil 253 } 254 del1 := func(k string) error { 255 batch.Delete([]byte(k), nil) 256 return nil 257 } 258 set, del := set0, del0 259 260 testCases := []string{ 261 "set Gandalf Grey", 262 "set Saruman White", 263 "set Radagast Brown", 264 "delete Saruman", 265 "set Gandalf White", 266 "batch", 267 " set Alatar AliceBlue", 268 "apply", 269 "delete Pallando", 270 "set Alatar AntiqueWhite", 271 "set Pallando PapayaWhip", 272 "batch", 273 "apply", 274 "set Pallando PaleVioletRed", 275 "batch", 276 " delete Alatar", 277 " set Gandalf GhostWhite", 278 " set Saruman Seashell", 279 " delete Saruman", 280 " set Saruman SeaGreen", 281 " set Radagast RosyBrown", 282 " delete Pallando", 283 "apply", 284 "delete Radagast", 285 "delete Radagast", 286 "delete Radagast", 287 "set Gandalf Goldenrod", 288 "set Pallando PeachPuff", 289 "batch", 290 " delete Joe", 291 " delete Saruman", 292 " delete Radagast", 293 " delete Pallando", 294 " delete Gandalf", 295 " delete Alatar", 296 "apply", 297 "set Joe Plumber", 298 } 299 for i, tc := range testCases { 300 s := strings.Split(strings.TrimSpace(tc), " ") 301 switch s[0] { 302 case "set": 303 if err := set(s[1], s[2]); err != nil { 304 t.Fatalf("#%d %s: %v", i, tc, err) 305 } 306 if inBatch { 307 pending = append(pending, s) 308 } else { 309 wantMap[s[1]] = s[2] 310 } 311 case "delete": 312 if err := del(s[1]); err != nil { 313 t.Fatalf("#%d %s: %v", i, tc, err) 314 } 315 if inBatch { 316 pending = append(pending, s) 317 } else { 318 delete(wantMap, s[1]) 319 } 320 case "batch": 321 inBatch, batch, set, del = true, &Batch{}, set1, del1 322 case "apply": 323 if err := d.Apply(batch, nil); err != nil { 324 t.Fatalf("#%d %s: %v", i, tc, err) 325 } 326 for _, p := range pending { 327 switch p[0] { 328 case "set": 329 wantMap[p[1]] = p[2] 330 case "delete": 331 delete(wantMap, p[1]) 332 } 333 } 334 inBatch, pending, set, del = false, nil, set0, del0 335 default: 336 t.Fatalf("#%d %s: bad test case: %q", i, tc, s) 337 } 338 339 fail := false 340 for _, name := range names { 341 g, err := d.Get([]byte(name)) 342 if err != nil && err != ErrNotFound { 343 t.Errorf("#%d %s: Get(%q): %v", i, tc, name, err) 344 fail = true 345 } 346 got, gOK := string(g), err == nil 347 want, wOK := wantMap[name] 348 if got != want || gOK != wOK { 349 t.Errorf("#%d %s: Get(%q): got %q, %t, want %q, %t", 350 i, tc, name, got, gOK, want, wOK) 351 fail = true 352 } 353 } 354 if fail { 355 return 356 } 357 } 358 359 if err := d.Close(); err != nil { 360 t.Fatal(err) 361 } 362 } 363 364 func TestRandomWrites(t *testing.T) { 365 d, err := Open("", &Options{ 366 FS: vfs.NewMem(), 367 MemTableSize: 8 * 1024, 368 }) 369 if err != nil { 370 t.Fatal(err) 371 } 372 373 keys := [64][]byte{} 374 wants := [64]int{} 375 for k := range keys { 376 keys[k] = []byte(strconv.Itoa(k)) 377 wants[k] = -1 378 } 379 xxx := bytes.Repeat([]byte("x"), 512) 380 381 rng := rand.New(rand.NewSource(123)) 382 const N = 1000 383 for i := 0; i < N; i++ { 384 k := rng.Intn(len(keys)) 385 if rng.Intn(20) != 0 { 386 wants[k] = rng.Intn(len(xxx) + 1) 387 if err := d.Set(keys[k], xxx[:wants[k]], nil); err != nil { 388 t.Fatalf("i=%d: Set: %v", i, err) 389 } 390 } else { 391 wants[k] = -1 392 if err := d.Delete(keys[k], nil); err != nil { 393 t.Fatalf("i=%d: Delete: %v", i, err) 394 } 395 } 396 397 if i != N-1 || rng.Intn(50) != 0 { 398 continue 399 } 400 for k := range keys { 401 got := -1 402 if v, err := d.Get(keys[k]); err != nil { 403 if err != ErrNotFound { 404 t.Fatalf("Get: %v", err) 405 } 406 } else { 407 got = len(v) 408 } 409 if got != wants[k] { 410 t.Errorf("i=%d, k=%d: got %d, want %d", i, k, got, wants[k]) 411 } 412 } 413 } 414 415 if err := d.Close(); err != nil { 416 t.Fatal(err) 417 } 418 } 419 420 func TestLargeBatch(t *testing.T) { 421 d, err := Open("", &Options{ 422 FS: vfs.NewMem(), 423 MemTableSize: 1400, 424 MemTableStopWritesThreshold: 100, 425 }) 426 if err != nil { 427 t.Fatal(err) 428 } 429 430 verifyLSM := func(expected string) func() error { 431 return func() error { 432 d.mu.Lock() 433 s := d.mu.versions.currentVersion().String() 434 d.mu.Unlock() 435 if expected != s { 436 if testing.Verbose() { 437 fmt.Println(strings.TrimSpace(s)) 438 } 439 return fmt.Errorf("expected %s, but found %s", expected, s) 440 } 441 return nil 442 } 443 } 444 445 // Write two keys with values that are larger than the memtable size. 446 if err := d.Set([]byte("a"), bytes.Repeat([]byte("a"), 512), nil); err != nil { 447 t.Fatal(err) 448 } 449 450 // Verify this results in one L0 table being created. 451 err = try(100*time.Microsecond, 20*time.Second, verifyLSM("0: a-a\n")) 452 if err != nil { 453 t.Fatal(err) 454 } 455 456 if err := d.Set([]byte("b"), bytes.Repeat([]byte("b"), 512), nil); err != nil { 457 t.Fatal(err) 458 } 459 460 // Verify this results in a second L0 table being created. 461 err = try(100*time.Microsecond, 20*time.Second, verifyLSM("0: a-a b-b\n")) 462 if err != nil { 463 t.Fatal(err) 464 } 465 466 if err := d.Close(); err != nil { 467 t.Fatal(err) 468 } 469 } 470 471 func TestGetMerge(t *testing.T) { 472 d, err := Open("", &Options{ 473 FS: vfs.NewMem(), 474 }) 475 if err != nil { 476 t.Fatal(err) 477 } 478 479 key := []byte("a") 480 verify := func(expected string) { 481 val, err := d.Get(key) 482 if err != nil { 483 t.Fatal(err) 484 } 485 if expected != string(val) { 486 t.Fatalf("expected %s, but got %s", expected, val) 487 } 488 } 489 490 const val = "1" 491 for i := 1; i <= 3; i++ { 492 if err := d.Merge(key, []byte(val), nil); err != nil { 493 t.Fatal(err) 494 } 495 expected := strings.Repeat(val, i) 496 verify(expected) 497 498 if err := d.Flush(); err != nil { 499 t.Fatal(err) 500 } 501 verify(expected) 502 } 503 504 if err := d.Close(); err != nil { 505 t.Fatal(err) 506 } 507 } 508 509 func TestIterLeak(t *testing.T) { 510 for _, leak := range []bool{true, false} { 511 t.Run(fmt.Sprintf("leak=%t", leak), func(t *testing.T) { 512 for _, flush := range []bool{true, false} { 513 t.Run(fmt.Sprintf("flush=%t", flush), func(t *testing.T) { 514 d, err := Open("", &Options{ 515 FS: vfs.NewMem(), 516 }) 517 if err != nil { 518 t.Fatal(err) 519 } 520 521 if err := d.Set([]byte("a"), []byte("a"), nil); err != nil { 522 t.Fatal(err) 523 } 524 if flush { 525 if err := d.Flush(); err != nil { 526 t.Fatal(err) 527 } 528 } 529 iter := d.NewIter(nil) 530 iter.First() 531 if !leak { 532 if err := iter.Close(); err != nil { 533 t.Fatal(err) 534 } 535 if err := d.Close(); err != nil { 536 t.Fatal(err) 537 } 538 } else { 539 if err := d.Close(); err == nil { 540 t.Fatalf("expected failure, but found success") 541 } else if !strings.HasPrefix(err.Error(), "leaked iterators:") { 542 t.Fatalf("expected leaked iterators, but found %+v", err) 543 } else { 544 t.Log(err.Error()) 545 } 546 } 547 }) 548 } 549 }) 550 } 551 } 552 553 func TestCacheEvict(t *testing.T) { 554 cache := cache.New(10 << 20) 555 d, err := Open("", &Options{ 556 Cache: cache, 557 FS: vfs.NewMem(), 558 }) 559 if err != nil { 560 t.Fatal(err) 561 } 562 563 for i := 0; i < 1000; i++ { 564 key := []byte(fmt.Sprintf("%04d", i)) 565 if err := d.Set(key, key, nil); err != nil { 566 t.Fatal(err) 567 } 568 } 569 570 if err := d.Flush(); err != nil { 571 t.Fatal(err) 572 } 573 iter := d.NewIter(nil) 574 for iter.First(); iter.Valid(); iter.Next() { 575 } 576 if err := iter.Close(); err != nil { 577 t.Fatal(err) 578 } 579 if size := cache.Size(); size == 0 { 580 t.Fatalf("expected non-zero cache size") 581 } 582 583 for i := 0; i < 1000; i++ { 584 key := []byte(fmt.Sprintf("%04d", i)) 585 if err := d.Delete(key, nil); err != nil { 586 t.Fatal(err) 587 } 588 } 589 590 if err := d.Compact([]byte("0"), []byte("1")); err != nil { 591 t.Fatal(err) 592 } 593 594 if size := cache.Size(); size != 0 { 595 t.Fatalf("expected empty cache, but found %d", size) 596 } 597 598 if err := d.Close(); err != nil { 599 t.Fatal(err) 600 } 601 } 602 603 func TestFlushEmpty(t *testing.T) { 604 d, err := Open("", &Options{ 605 FS: vfs.NewMem(), 606 }) 607 if err != nil { 608 t.Fatal(err) 609 } 610 // Flushing an empty memtable should not fail. 611 if err := d.Flush(); err != nil { 612 t.Fatal(err) 613 } 614 if err := d.Close(); err != nil { 615 t.Fatal(err) 616 } 617 } 618 619 func TestRollManifest(t *testing.T) { 620 d, err := Open("", &Options{ 621 MaxManifestFileSize: 1, 622 L0CompactionThreshold: 10, 623 FS: vfs.NewMem(), 624 }) 625 if err != nil { 626 t.Fatal(err) 627 } 628 629 manifestFileNumber := func() uint64 { 630 d.mu.Lock() 631 defer d.mu.Unlock() 632 return d.mu.versions.manifestFileNum 633 } 634 635 current := func() string { 636 f, err := d.opts.FS.Open(base.MakeFilename(d.dirname, fileTypeCurrent, 0)) 637 if err != nil { 638 t.Fatal(err) 639 } 640 defer f.Close() 641 stat, err := f.Stat() 642 if err != nil { 643 t.Fatal(err) 644 } 645 n := stat.Size() 646 b := make([]byte, n) 647 if _, err = f.ReadAt(b, 0); err != nil { 648 t.Fatal(err) 649 } 650 return string(b) 651 } 652 653 lastManifestNum := manifestFileNumber() 654 for i := 0; i < 5; i++ { 655 if err := d.Set([]byte("a"), nil, nil); err != nil { 656 t.Fatal(err) 657 } 658 if err := d.Flush(); err != nil { 659 t.Fatal(err) 660 } 661 num := manifestFileNumber() 662 if lastManifestNum == num { 663 t.Fatalf("manifest failed to roll: %d == %d", lastManifestNum, num) 664 } 665 lastManifestNum = num 666 667 expectedCurrent := fmt.Sprintf("MANIFEST-%06d\n", lastManifestNum) 668 if v := current(); expectedCurrent != v { 669 t.Fatalf("expected %s, but found %s", expectedCurrent, v) 670 } 671 } 672 673 files, err := d.opts.FS.List("") 674 if err != nil { 675 t.Fatal(err) 676 } 677 var manifests []string 678 for _, filename := range files { 679 fileType, _, ok := base.ParseFilename(filename) 680 if !ok { 681 continue 682 } 683 if fileType == fileTypeManifest { 684 manifests = append(manifests, filename) 685 } 686 } 687 expected := []string{fmt.Sprintf("MANIFEST-%06d", lastManifestNum)} 688 require.EqualValues(t, expected, manifests) 689 690 if err := d.Close(); err != nil { 691 t.Fatal(err) 692 } 693 } 694 695 func TestDBClosed(t *testing.T) { 696 d, err := Open("", &Options{ 697 FS: vfs.NewMem(), 698 }) 699 if err != nil { 700 t.Fatal(err) 701 } 702 if err := d.Close(); err != nil { 703 t.Fatal(err) 704 } 705 706 catch := func(f func()) (err error) { 707 defer func() { 708 if r := recover(); r != nil { 709 err = r.(error) 710 } 711 }() 712 f() 713 return nil 714 } 715 716 require.EqualValues(t, ErrClosed, catch(func() { _ = d.Close() })) 717 718 require.EqualValues(t, ErrClosed, catch(func() { _ = d.Compact(nil, nil) })) 719 require.EqualValues(t, ErrClosed, catch(func() { _ = d.Flush() })) 720 require.EqualValues(t, ErrClosed, catch(func() { _, _ = d.AsyncFlush() })) 721 722 require.EqualValues(t, ErrClosed, catch(func() { _, _ = d.Get(nil) })) 723 require.EqualValues(t, ErrClosed, catch(func() { _ = d.Delete(nil, nil) })) 724 require.EqualValues(t, ErrClosed, catch(func() { _ = d.DeleteRange(nil, nil, nil) })) 725 require.EqualValues(t, ErrClosed, catch(func() { _ = d.LogData(nil, nil) })) 726 require.EqualValues(t, ErrClosed, catch(func() { _ = d.Merge(nil, nil, nil) })) 727 require.EqualValues(t, ErrClosed, catch(func() { _ = d.Set(nil, nil, nil) })) 728 729 require.EqualValues(t, ErrClosed, catch(func() { _ = d.NewSnapshot() })) 730 731 b := d.NewIndexedBatch() 732 require.EqualValues(t, ErrClosed, catch(func() { _ = b.Commit(nil) })) 733 require.EqualValues(t, ErrClosed, catch(func() { _ = d.Apply(b, nil) })) 734 require.EqualValues(t, ErrClosed, catch(func() { _ = b.NewIter(nil) })) 735 } 736 737 func TestDBConcurrentCommitCompactFlush(t *testing.T) { 738 d, err := Open("", &Options{ 739 FS: vfs.NewMem(), 740 }) 741 if err != nil { 742 t.Fatal(err) 743 } 744 745 // Concurrently commit, compact, and flush in order to stress the locking around 746 // those operations. 747 const n = 1000 748 var wg sync.WaitGroup 749 wg.Add(n) 750 for i := 0; i < n; i++ { 751 go func(i int) { 752 defer wg.Done() 753 _ = d.Set([]byte(fmt.Sprint(i)), nil, nil) 754 var err error 755 switch i % 3 { 756 case 0: 757 err = d.Compact(nil, []byte("\xff")) 758 case 1: 759 err = d.Flush() 760 case 2: 761 _, err = d.AsyncFlush() 762 } 763 if err != nil { 764 t.Fatal(err) 765 } 766 }(i) 767 } 768 wg.Wait() 769 770 if err := d.Close(); err != nil { 771 t.Fatal(err) 772 } 773 }