github.com/pingcap/badger@v1.5.1-0.20230103063557-828f39b09b6d/db_test.go (about) 1 /* 2 * Copyright 2017 Dgraph Labs, Inc. and Contributors 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package badger 18 19 import ( 20 "bytes" 21 "flag" 22 "fmt" 23 "io/ioutil" 24 "log" 25 "math" 26 "math/rand" 27 "os" 28 "path/filepath" 29 "regexp" 30 "sort" 31 "sync" 32 "testing" 33 "time" 34 35 "github.com/pingcap/badger/options" 36 "github.com/pingcap/badger/table/sstable" 37 "github.com/pingcap/badger/y" 38 "github.com/stretchr/testify/require" 39 ) 40 41 var mmap = flag.Bool("vlog_mmap", true, "Specify if value log must be memory-mapped") 42 43 func getTestCompression(tp options.CompressionType) []options.CompressionType { 44 tps := make([]options.CompressionType, DefaultOptions.TableBuilderOptions.MaxLevels) 45 for i := range tps { 46 tps[i] = tp 47 } 48 return tps 49 } 50 51 func getTestOptions(dir string) Options { 52 opt := DefaultOptions 53 opt.TableBuilderOptions.MaxTableSize = 4 << 15 // Force more compaction. 54 opt.MaxMemTableSize = 4 << 15 // Force more compaction. 55 opt.LevelOneSize = 4 << 15 // Force more compaction. 56 opt.Dir = dir 57 opt.ValueDir = dir 58 opt.SyncWrites = false 59 opt.TableBuilderOptions.CompressionPerLevel = getTestCompression(options.ZSTD) 60 return opt 61 } 62 63 func getItemValue(t *testing.T, item *Item) (val []byte) { 64 v, err := item.Value() 65 if err != nil { 66 t.Error(err) 67 } 68 if v == nil { 69 return nil 70 } 71 vSize := item.ValueSize() 72 require.Equal(t, vSize, len(v)) 73 another, err := item.ValueCopy(nil) 74 require.NoError(t, err) 75 require.Equal(t, v, another) 76 return v 77 } 78 79 func txnSet(t *testing.T, kv *DB, key []byte, val []byte, meta byte) { 80 txn := kv.NewTransaction(true) 81 require.NoError(t, txn.SetWithMeta(key, val, meta)) 82 require.NoError(t, txn.Commit()) 83 } 84 85 func txnDelete(t *testing.T, kv *DB, key []byte) { 86 txn := kv.NewTransaction(true) 87 require.NoError(t, txn.Delete(key)) 88 require.NoError(t, txn.Commit()) 89 } 90 91 // Opens a badger db and runs a a test on it. 92 func runBadgerTest(t *testing.T, opts *Options, test func(t *testing.T, db *DB)) { 93 dir, err := ioutil.TempDir("", "badger") 94 require.NoError(t, err) 95 defer os.RemoveAll(dir) 96 if opts == nil { 97 opts = new(Options) 98 *opts = getTestOptions(dir) 99 } 100 db, err := Open(*opts) 101 require.NoError(t, err) 102 defer db.Close() 103 test(t, db) 104 } 105 106 func TestWrite(t *testing.T) { 107 runBadgerTest(t, nil, func(t *testing.T, db *DB) { 108 for i := 0; i < 100; i++ { 109 txnSet(t, db, []byte(fmt.Sprintf("key%d", i)), []byte(fmt.Sprintf("val%d", i)), 0x00) 110 } 111 }) 112 } 113 114 func TestUpdateAndView(t *testing.T) { 115 runBadgerTest(t, nil, func(t *testing.T, db *DB) { 116 err := db.Update(func(txn *Txn) error { 117 for i := 0; i < 10; i++ { 118 err := txn.Set([]byte(fmt.Sprintf("key%d", i)), []byte(fmt.Sprintf("val%d", i))) 119 if err != nil { 120 return err 121 } 122 } 123 return nil 124 }) 125 require.NoError(t, err) 126 127 err = db.View(func(txn *Txn) error { 128 for i := 0; i < 10; i++ { 129 item, err := txn.Get([]byte(fmt.Sprintf("key%d", i))) 130 if err != nil { 131 return err 132 } 133 134 val, err := item.Value() 135 if err != nil { 136 return err 137 } 138 expected := []byte(fmt.Sprintf("val%d", i)) 139 require.Equal(t, expected, val, 140 "Invalid value for key %q. expected: %q, actual: %q", 141 item.Key(), expected, val) 142 } 143 return nil 144 }) 145 require.NoError(t, err) 146 }) 147 } 148 149 func TestConcurrentWrite(t *testing.T) { 150 runBadgerTest(t, nil, func(t *testing.T, db *DB) { 151 // Not a benchmark. Just a simple test for concurrent writes. 152 n := 20 153 m := 500 154 var wg sync.WaitGroup 155 for i := 0; i < n; i++ { 156 wg.Add(1) 157 go func(i int) { 158 defer wg.Done() 159 for j := 0; j < m; j++ { 160 txnSet(t, db, []byte(fmt.Sprintf("k%05d_%08d", i, j)), 161 []byte(fmt.Sprintf("v%05d_%08d", i, j)), byte(j%127)) 162 } 163 }(i) 164 } 165 wg.Wait() 166 167 t.Log("Starting iteration") 168 txn := db.NewTransaction(true) 169 it := txn.NewIterator(DefaultIteratorOptions) 170 defer it.Close() 171 var i, j int 172 for it.Rewind(); it.Valid(); it.Next() { 173 item := it.Item() 174 k := item.Key() 175 if k == nil { 176 break // end of iteration. 177 } 178 179 require.EqualValues(t, fmt.Sprintf("k%05d_%08d", i, j), string(k)) 180 v := getItemValue(t, item) 181 require.EqualValues(t, fmt.Sprintf("v%05d_%08d", i, j), string(v)) 182 require.Equal(t, item.UserMeta(), []byte{byte(j % 127)}) 183 j++ 184 if j == m { 185 i++ 186 j = 0 187 } 188 } 189 require.EqualValues(t, n, i) 190 require.EqualValues(t, 0, j) 191 }) 192 } 193 194 func TestGet(t *testing.T) { 195 runBadgerTest(t, nil, func(t *testing.T, db *DB) { 196 txnSet(t, db, []byte("key1"), []byte("val1"), 0x08) 197 198 txn := db.NewTransaction(false) 199 item, err := txn.Get([]byte("key1")) 200 require.NoError(t, err) 201 require.EqualValues(t, "val1", getItemValue(t, item)) 202 require.Equal(t, []byte{0x08}, item.UserMeta()) 203 txn.Discard() 204 205 txnSet(t, db, []byte("key1"), []byte("val2"), 0x09) 206 207 txn = db.NewTransaction(false) 208 item, err = txn.Get([]byte("key1")) 209 require.NoError(t, err) 210 require.EqualValues(t, "val2", getItemValue(t, item)) 211 require.Equal(t, []byte{0x09}, item.UserMeta()) 212 txn.Discard() 213 214 txnDelete(t, db, []byte("key1")) 215 216 txn = db.NewTransaction(false) 217 _, err = txn.Get([]byte("key1")) 218 require.Equal(t, ErrKeyNotFound, err) 219 txn.Discard() 220 221 txnSet(t, db, []byte("key1"), []byte("val3"), 0x01) 222 223 txn = db.NewTransaction(false) 224 item, err = txn.Get([]byte("key1")) 225 require.NoError(t, err) 226 require.EqualValues(t, "val3", getItemValue(t, item)) 227 require.Equal(t, []byte{0x01}, item.UserMeta()) 228 txn.Discard() 229 230 longVal := make([]byte, 1000) 231 txnSet(t, db, []byte("key1"), y.Copy(longVal), 0x00) 232 233 txn = db.NewTransaction(false) 234 item, err = txn.Get([]byte("key1")) 235 require.NoError(t, err) 236 require.EqualValues(t, longVal, getItemValue(t, item)) 237 txn.Discard() 238 }) 239 } 240 241 func TestGetAfterDelete(t *testing.T) { 242 runBadgerTest(t, nil, func(t *testing.T, db *DB) { 243 // populate with one entry 244 key := []byte("key") 245 txnSet(t, db, key, []byte("val1"), 0x00) 246 require.NoError(t, db.Update(func(txn *Txn) error { 247 err := txn.Delete(key) 248 require.NoError(t, err) 249 250 _, err = txn.Get(key) 251 require.Equal(t, ErrKeyNotFound, err) 252 return nil 253 })) 254 }) 255 } 256 257 func TestTxnTooBig(t *testing.T) { 258 runBadgerTest(t, nil, func(t *testing.T, db *DB) { 259 data := func(i int) []byte { 260 return []byte(fmt.Sprintf("%b", i)) 261 } 262 // n := 500000 263 n := 1000 264 txn := db.NewTransaction(true) 265 for i := 0; i < n; { 266 if err := txn.Set(data(i), data(i)); err != nil { 267 require.NoError(t, txn.Commit()) 268 txn = db.NewTransaction(true) 269 } else { 270 i++ 271 } 272 } 273 require.NoError(t, txn.Commit()) 274 275 txn = db.NewTransaction(true) 276 for i := 0; i < n; { 277 if err := txn.Delete(data(i)); err != nil { 278 require.NoError(t, txn.Commit()) 279 txn = db.NewTransaction(true) 280 } else { 281 i++ 282 } 283 } 284 require.NoError(t, txn.Commit()) 285 }) 286 } 287 288 func TestForceCompactL0(t *testing.T) { 289 dir, err := ioutil.TempDir("", "badger") 290 require.NoError(t, err) 291 defer os.RemoveAll(dir) 292 293 opts := getTestOptions(dir) 294 opts.ValueLogFileSize = 15 << 20 295 db, err := OpenManaged(opts) 296 require.NoError(t, err) 297 298 data := func(i int) []byte { 299 return []byte(fmt.Sprintf("%b", i)) 300 } 301 n := 80 302 m := 45 // Increasing would cause ErrTxnTooBig 303 sz := 128 304 v := make([]byte, sz) 305 for i := 0; i < n; i += 2 { 306 version := uint64(i) 307 txn := db.NewTransactionAt(version, true) 308 for j := 0; j < m; j++ { 309 require.NoError(t, txn.SetEntry(&Entry{ 310 Key: y.KeyWithTs(data(j), version+1), 311 Value: v, 312 })) 313 } 314 require.NoError(t, txn.Commit()) 315 } 316 db.Close() 317 318 db, err = OpenManaged(opts) 319 defer db.Close() 320 require.Equal(t, len(db.lc.levels[0].tables), 0) 321 } 322 323 // Put a lot of data to move some data to disk. 324 // WARNING: This test might take a while but it should pass! 325 func TestGetMore(t *testing.T) { 326 runBadgerTest(t, nil, func(t *testing.T, db *DB) { 327 328 data := func(i int) []byte { 329 return []byte(fmt.Sprintf("%b", i)) 330 } 331 n := 10000 332 333 txn := db.NewTransaction(true) 334 for i := 0; i < n; i++ { 335 require.NoError(t, txn.Set(data(i), data(i))) 336 } 337 require.NoError(t, txn.Commit()) 338 339 require.NoError(t, db.validate()) 340 341 for i := 0; i < n; i++ { 342 txn := db.NewTransaction(false) 343 item, err := txn.Get(data(i)) 344 if err != nil { 345 t.Errorf("key %v", data(i)) 346 } 347 require.EqualValues(t, string(data(i)), string(getItemValue(t, item))) 348 txn.Discard() 349 } 350 351 // Overwrite 352 txn = db.NewTransaction(true) 353 for i := 0; i < n; i++ { 354 require.NoError(t, txn.Set(data(i), 355 // Use a long value that will certainly exceed value threshold. 356 []byte(fmt.Sprintf("zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz%9d", i)))) 357 } 358 require.NoError(t, txn.Commit()) 359 360 require.NoError(t, db.validate()) 361 362 for i := 0; i < n; i++ { 363 expectedValue := fmt.Sprintf("zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz%9d", i) 364 k := data(i) 365 txn := db.NewTransaction(false) 366 item, err := txn.Get(k) 367 if err != nil { 368 t.Error(err) 369 } 370 got := string(getItemValue(t, item)) 371 if expectedValue != got { 372 373 vs := db.get(y.KeyWithTs(k, math.MaxUint64)) 374 fmt.Printf("wanted=%q Item: %s\n", k, item) 375 fmt.Printf("on re-run, got version: %+v\n", vs) 376 377 txn := db.NewTransaction(false) 378 itr := txn.NewIterator(DefaultIteratorOptions) 379 for itr.Seek(k); itr.Valid(); itr.Next() { 380 item := itr.Item() 381 fmt.Printf("item=%s\n", item) 382 if !bytes.Equal(item.Key(), k) { 383 break 384 } 385 } 386 itr.Close() 387 txn.Discard() 388 } 389 require.EqualValues(t, expectedValue, string(getItemValue(t, item)), "wanted=%q Item: %s\n", k, item) 390 txn.Discard() 391 } 392 393 // MultiGet 394 var multiGetKeys [][]byte 395 var expectedValues []string 396 for i := 0; i < n; i += 100 { 397 multiGetKeys = append(multiGetKeys, data(i)) 398 // Set a long value to make sure we have enough sst tables. 399 expectedValues = append(expectedValues, fmt.Sprintf("zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz%9d", i)) 400 } 401 txn1 := db.NewTransaction(false) 402 items, err := txn1.MultiGet(multiGetKeys) 403 require.NoError(t, err) 404 for i, item := range items { 405 val, err1 := item.Value() 406 require.NoError(t, err1) 407 require.Equal(t, expectedValues[i], string(val)) 408 } 409 txn1.Discard() 410 411 // "Delete" key. 412 txn = db.NewTransaction(true) 413 for i := 0; i < n; i++ { 414 if (i % 10000) == 0 { 415 fmt.Printf("Deleting i=%d\n", i) 416 } 417 require.NoError(t, txn.Delete(data(i))) 418 } 419 require.NoError(t, txn.Commit()) 420 421 db.validate() 422 for i := 0; i < n; i++ { 423 if (i % 10000) == 0 { 424 // Display some progress. Right now, it's not very fast with no caching. 425 fmt.Printf("Testing i=%d\n", i) 426 } 427 k := data(i) 428 txn := db.NewTransaction(false) 429 _, err := txn.Get([]byte(k)) 430 require.Equal(t, ErrKeyNotFound, err, "should not have found k: %q", k) 431 txn.Discard() 432 } 433 }) 434 } 435 436 // Put a lot of data to move some data to disk. 437 // WARNING: This test might take a while but it should pass! 438 func TestExistsMore(t *testing.T) { 439 runBadgerTest(t, nil, func(t *testing.T, db *DB) { 440 // n := 500000 441 n := 10000 442 m := 45 443 for i := 0; i < n; i += m { 444 if (i % 1000) == 0 { 445 t.Logf("Putting i=%d\n", i) 446 } 447 txn := db.NewTransaction(true) 448 for j := i; j < i+m && j < n; j++ { 449 require.NoError(t, txn.Set([]byte(fmt.Sprintf("%09d", j)), 450 []byte(fmt.Sprintf("%09d", j)))) 451 } 452 require.NoError(t, txn.Commit()) 453 } 454 db.validate() 455 456 for i := 0; i < n; i++ { 457 if (i % 1000) == 0 { 458 fmt.Printf("Testing i=%d\n", i) 459 } 460 k := fmt.Sprintf("%09d", i) 461 require.NoError(t, db.View(func(txn *Txn) error { 462 _, err := txn.Get([]byte(k)) 463 require.NoError(t, err) 464 return nil 465 })) 466 } 467 require.NoError(t, db.View(func(txn *Txn) error { 468 _, err := txn.Get([]byte("non-exists")) 469 require.Error(t, err) 470 return nil 471 })) 472 473 // "Delete" key. 474 for i := 0; i < n; i += m { 475 if (i % 1000) == 0 { 476 fmt.Printf("Deleting i=%d\n", i) 477 } 478 txn := db.NewTransaction(true) 479 for j := i; j < i+m && j < n; j++ { 480 require.NoError(t, txn.Delete([]byte(fmt.Sprintf("%09d", j)))) 481 } 482 require.NoError(t, txn.Commit()) 483 } 484 db.validate() 485 for i := 0; i < n; i++ { 486 if (i % 10000) == 0 { 487 // Display some progress. Right now, it's not very fast with no caching. 488 fmt.Printf("Testing i=%d\n", i) 489 } 490 k := fmt.Sprintf("%09d", i) 491 492 require.NoError(t, db.View(func(txn *Txn) error { 493 _, err := txn.Get([]byte(k)) 494 require.Error(t, err) 495 return nil 496 })) 497 } 498 fmt.Println("Done and closing") 499 }) 500 } 501 502 func TestIterate2Basic(t *testing.T) { 503 runBadgerTest(t, nil, func(t *testing.T, db *DB) { 504 505 bkey := func(i int) []byte { 506 return []byte(fmt.Sprintf("%09d", i)) 507 } 508 bval := func(i int) []byte { 509 return []byte(fmt.Sprintf("%025d", i)) 510 } 511 512 // n := 500000 513 n := 10000 514 for i := 0; i < n; i++ { 515 if (i % 1000) == 0 { 516 t.Logf("Put i=%d\n", i) 517 } 518 txnSet(t, db, bkey(i), bval(i), byte(i%127)) 519 } 520 521 txn := db.NewTransaction(false) 522 it := txn.NewIterator(DefaultIteratorOptions) 523 { 524 var count int 525 rewind := true 526 t.Log("Starting first basic iteration") 527 for it.Rewind(); it.Valid(); it.Next() { 528 item := it.Item() 529 key := item.Key() 530 if rewind && count == 5000 { 531 // Rewind would skip /head/ key, and it.Next() would skip 0. 532 count = 1 533 it.Rewind() 534 t.Log("Rewinding from 5000 to zero.") 535 rewind = false 536 continue 537 } 538 require.EqualValues(t, bkey(count), string(key)) 539 val := getItemValue(t, item) 540 require.EqualValues(t, bval(count), string(val)) 541 require.Equal(t, []byte{byte(count % 127)}, item.UserMeta()) 542 count++ 543 } 544 require.EqualValues(t, n, count) 545 } 546 547 { 548 t.Log("Starting second basic iteration") 549 idx := 5030 550 for it.Seek(bkey(idx)); it.Valid(); it.Next() { 551 item := it.Item() 552 require.EqualValues(t, bkey(idx), string(item.Key())) 553 require.EqualValues(t, bval(idx), string(getItemValue(t, item))) 554 idx++ 555 } 556 } 557 it.Close() 558 }) 559 } 560 561 func TestLoad(t *testing.T) { 562 testLoad := func(t *testing.T, opt Options) { 563 dir, err := ioutil.TempDir("", "badger-test") 564 require.NoError(t, err) 565 defer os.RemoveAll(dir) 566 opt.Dir = dir 567 opt.ValueDir = dir 568 n := 10000 569 570 { 571 kv, _ := Open(getTestOptions(dir)) 572 for i := 0; i < n; i++ { 573 if (i % 10000) == 0 { 574 fmt.Printf("Putting i=%d\n", i) 575 } 576 k := []byte(fmt.Sprintf("%09d", i)) 577 txnSet(t, kv, k, k, 0x00) 578 } 579 kv.Close() 580 } 581 582 kv, err := Open(getTestOptions(dir)) 583 require.NoError(t, err) 584 require.Equal(t, uint64(10001), kv.orc.readTs()) 585 for i := 0; i < n; i++ { 586 if (i % 10000) == 0 { 587 fmt.Printf("Testing i=%d\n", i) 588 } 589 k := fmt.Sprintf("%09d", i) 590 require.NoError(t, kv.View(func(txn *Txn) error { 591 item, err := txn.Get([]byte(k)) 592 require.NoError(t, err) 593 require.EqualValues(t, k, string(getItemValue(t, item))) 594 return nil 595 })) 596 } 597 598 kv.Close() 599 summary := kv.lc.getSummary() 600 601 // Check that files are garbage collected. 602 idMap := getIDMap(dir) 603 for fileID := range idMap { 604 // Check that name is in summary.filenames. 605 require.True(t, summary.fileIDs[fileID], "%d", fileID) 606 } 607 require.EqualValues(t, len(idMap), len(summary.fileIDs)) 608 609 var fileIDs []uint64 610 for k := range summary.fileIDs { // Map to array. 611 fileIDs = append(fileIDs, k) 612 } 613 sort.Slice(fileIDs, func(i, j int) bool { return fileIDs[i] < fileIDs[j] }) 614 fmt.Printf("FileIDs: %v\n", fileIDs) 615 } 616 617 t.Run("Without compression", func(t *testing.T) { 618 opt := getTestOptions("") 619 opt.TableBuilderOptions.CompressionPerLevel = getTestCompression(options.None) 620 testLoad(t, opt) 621 }) 622 t.Run("With compression", func(t *testing.T) { 623 opt := getTestOptions("") 624 testLoad(t, opt) 625 }) 626 } 627 628 func TestIterateDeleted(t *testing.T) { 629 runBadgerTest(t, nil, func(t *testing.T, db *DB) { 630 txnSet(t, db, []byte("Key1"), []byte("Value1"), 0x00) 631 txnSet(t, db, []byte("Key2"), []byte("Value2"), 0x00) 632 633 txn := db.NewTransaction(false) 634 idxIt := txn.NewIterator(DefaultIteratorOptions) 635 636 count := 0 637 txn2 := db.NewTransaction(true) 638 prefix := []byte("Key") 639 for idxIt.Seek(prefix); idxIt.ValidForPrefix(prefix); idxIt.Next() { 640 key := idxIt.Item().Key() 641 count++ 642 newKey := make([]byte, len(key)) 643 copy(newKey, key) 644 require.NoError(t, txn2.Delete(newKey)) 645 } 646 require.Equal(t, 2, count) 647 require.NoError(t, txn2.Commit()) 648 idxIt.Close() 649 t.Run(fmt.Sprintf("Prefetch=%t", false), func(t *testing.T) { 650 txn := db.NewTransaction(false) 651 idxIt := txn.NewIterator(DefaultIteratorOptions) 652 var estSize int64 653 var idxKeys []string 654 for idxIt.Seek(prefix); idxIt.Valid(); idxIt.Next() { 655 item := idxIt.Item() 656 key := item.Key() 657 estSize += item.EstimatedSize() 658 if !bytes.HasPrefix(key, prefix) { 659 break 660 } 661 idxKeys = append(idxKeys, string(key)) 662 t.Logf("%+v\n", idxIt.Item()) 663 } 664 require.Equal(t, 0, len(idxKeys)) 665 require.Equal(t, int64(0), estSize) 666 idxIt.Close() 667 }) 668 }) 669 } 670 671 func TestDeleteWithoutSyncWrite(t *testing.T) { 672 dir, err := ioutil.TempDir("", "badger") 673 require.NoError(t, err) 674 defer os.RemoveAll(dir) 675 opt := DefaultOptions 676 opt.Dir = dir 677 opt.ValueDir = dir 678 kv, err := Open(opt) 679 if err != nil { 680 t.Error(err) 681 t.Fail() 682 } 683 684 key := []byte("k1") 685 // Set a value with size > value threshold so that its written to value log. 686 txnSet(t, kv, key, []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789FOOBARZOGZOG"), 0x00) 687 txnDelete(t, kv, key) 688 kv.Close() 689 690 // Reopen KV 691 kv, err = Open(opt) 692 if err != nil { 693 t.Error(err) 694 t.Fail() 695 } 696 defer kv.Close() 697 698 require.NoError(t, kv.View(func(txn *Txn) error { 699 _, err := txn.Get(key) 700 require.Equal(t, ErrKeyNotFound, err) 701 return nil 702 })) 703 } 704 705 func TestPidFile(t *testing.T) { 706 runBadgerTest(t, nil, func(t *testing.T, db *DB) { 707 // Reopen database 708 _, err := Open(getTestOptions(db.opt.Dir)) 709 require.Error(t, err) 710 require.Contains(t, err.Error(), "Another process is using this Badger database") 711 }) 712 } 713 714 func TestBigKeyValuePairs(t *testing.T) { 715 runBadgerTest(t, nil, func(t *testing.T, db *DB) { 716 bigK := make([]byte, maxKeySize+1) 717 bigV := make([]byte, db.opt.ValueLogFileSize+1) 718 small := make([]byte, 10) 719 720 txn := db.NewTransaction(true) 721 require.Regexp(t, regexp.MustCompile("Key.*exceeded"), txn.Set(bigK, small)) 722 require.Regexp(t, regexp.MustCompile("Value.*exceeded"), txn.Set(small, bigV)) 723 724 require.NoError(t, txn.Set(small, small)) 725 require.Regexp(t, regexp.MustCompile("Key.*exceeded"), txn.Set(bigK, bigV)) 726 727 require.NoError(t, db.View(func(txn *Txn) error { 728 _, err := txn.Get(small) 729 require.Equal(t, ErrKeyNotFound, err) 730 return nil 731 })) 732 }) 733 } 734 735 func TestSetIfAbsentAsync(t *testing.T) { 736 dir, err := ioutil.TempDir("", "badger") 737 require.NoError(t, err) 738 defer os.RemoveAll(dir) 739 kv, _ := Open(getTestOptions(dir)) 740 741 bkey := func(i int) []byte { 742 return []byte(fmt.Sprintf("%09d", i)) 743 } 744 745 n := 1000 746 for i := 0; i < n; i++ { 747 // if (i % 10) == 0 { 748 // t.Logf("Put i=%d\n", i) 749 // } 750 txn := kv.NewTransaction(true) 751 _, err = txn.Get(bkey(i)) 752 require.Equal(t, ErrKeyNotFound, err) 753 require.NoError(t, txn.SetWithMeta(bkey(i), nil, byte(i%127))) 754 require.NoError(t, txn.Commit()) 755 } 756 757 require.NoError(t, kv.Close()) 758 kv, err = Open(getTestOptions(dir)) 759 require.NoError(t, err) 760 761 txn := kv.NewTransaction(false) 762 var count int 763 it := txn.NewIterator(DefaultIteratorOptions) 764 defer it.Close() 765 { 766 t.Log("Starting first basic iteration") 767 for it.Rewind(); it.Valid(); it.Next() { 768 count++ 769 } 770 require.EqualValues(t, n, count) 771 } 772 require.Equal(t, n, count) 773 require.NoError(t, kv.Close()) 774 } 775 776 func TestGetSetRace(t *testing.T) { 777 runBadgerTest(t, nil, func(t *testing.T, db *DB) { 778 779 data := make([]byte, 4096) 780 _, err := rand.Read(data) 781 require.NoError(t, err) 782 783 var ( 784 numOp = 100 785 wg sync.WaitGroup 786 keyCh = make(chan string) 787 ) 788 789 // writer 790 wg.Add(1) 791 go func() { 792 defer func() { 793 wg.Done() 794 close(keyCh) 795 }() 796 797 for i := 0; i < numOp; i++ { 798 key := fmt.Sprintf("%d", i) 799 txnSet(t, db, []byte(key), data, 0x00) 800 keyCh <- key 801 } 802 }() 803 804 // reader 805 wg.Add(1) 806 go func() { 807 defer wg.Done() 808 809 for key := range keyCh { 810 require.NoError(t, db.View(func(txn *Txn) error { 811 item, err := txn.Get([]byte(key)) 812 require.NoError(t, err) 813 _, err = item.Value() 814 require.NoError(t, err) 815 return nil 816 })) 817 } 818 }() 819 820 wg.Wait() 821 }) 822 } 823 824 func randBytes(n int) []byte { 825 recv := make([]byte, n) 826 in, err := rand.Read(recv) 827 if err != nil { 828 log.Fatal(err) 829 } 830 return recv[:in] 831 } 832 833 var benchmarkData = []struct { 834 key, value []byte 835 }{ 836 {randBytes(100), nil}, 837 {randBytes(1000), []byte("foo")}, 838 {[]byte("foo"), randBytes(1000)}, 839 {[]byte(""), randBytes(1000)}, 840 {nil, randBytes(1000000)}, 841 {randBytes(100000), nil}, 842 {randBytes(1000000), nil}, 843 } 844 845 func TestLargeKeys(t *testing.T) { 846 dir, err := ioutil.TempDir("", "badger") 847 require.NoError(t, err) 848 defer os.RemoveAll(dir) 849 850 opts := new(Options) 851 *opts = DefaultOptions 852 opts.ValueLogFileSize = 1024 * 1024 * 1024 853 opts.Dir = dir 854 opts.ValueDir = dir 855 856 db, err := Open(*opts) 857 if err != nil { 858 t.Fatal(err) 859 } 860 defer db.Close() 861 862 for i := 0; i < 1000; i++ { 863 tx := db.NewTransaction(true) 864 for _, kv := range benchmarkData { 865 k := make([]byte, len(kv.key)) 866 copy(k, kv.key) 867 868 v := make([]byte, len(kv.value)) 869 copy(v, kv.value) 870 if err := tx.Set(k, v); err != nil { 871 // Skip over this record. 872 } 873 } 874 if err := tx.Commit(); err != nil { 875 t.Fatalf("#%d: batchSet err: %v", i, err) 876 } 877 } 878 } 879 880 func TestCreateDirs(t *testing.T) { 881 dir, err := ioutil.TempDir("", "parent") 882 require.NoError(t, err) 883 defer os.RemoveAll(dir) 884 885 opts := DefaultOptions 886 dir = filepath.Join(dir, "badger") 887 opts.Dir = dir 888 opts.ValueDir = dir 889 db, err := Open(opts) 890 require.NoError(t, err) 891 db.Close() 892 _, err = os.Stat(dir) 893 require.NoError(t, err) 894 } 895 896 func TestGetSetDeadlock(t *testing.T) { 897 dir, err := ioutil.TempDir("", "badger") 898 fmt.Println(dir) 899 require.NoError(t, err) 900 defer os.RemoveAll(dir) 901 902 opt := DefaultOptions 903 opt.Dir = dir 904 opt.ValueDir = dir 905 opt.ValueLogFileSize = 1 << 20 906 db, err := Open(opt) 907 require.NoError(t, err) 908 defer db.Close() 909 910 val := make([]byte, 1<<19) 911 key := []byte("key1") 912 require.NoError(t, db.Update(func(txn *Txn) error { 913 rand.Read(val) 914 require.NoError(t, txn.Set(key, val)) 915 return nil 916 })) 917 918 timeout, done := time.After(10*time.Second), make(chan bool) 919 920 go func() { 921 db.Update(func(txn *Txn) error { 922 item, err := txn.Get(key) 923 require.NoError(t, err) 924 _, err = item.Value() // This take a RLock on file 925 require.NoError(t, err) 926 927 rand.Read(val) 928 require.NoError(t, txn.Set(key, val)) 929 require.NoError(t, txn.Set([]byte("key2"), val)) 930 return nil 931 }) 932 done <- true 933 }() 934 935 select { 936 case <-timeout: 937 t.Fatal("db.Update did not finish within 10s, assuming deadlock.") 938 case <-done: 939 t.Log("db.Update finished.") 940 } 941 } 942 943 func TestWriteDeadlock(t *testing.T) { 944 dir, err := ioutil.TempDir("", "badger") 945 fmt.Println(dir) 946 require.NoError(t, err) 947 defer os.RemoveAll(dir) 948 949 opt := DefaultOptions 950 opt.Dir = dir 951 opt.ValueDir = dir 952 opt.ValueLogFileSize = 10 << 20 953 db, err := Open(opt) 954 require.NoError(t, err) 955 defer db.Close() 956 957 print := func(count *int) { 958 *count++ 959 if *count%100 == 0 { 960 fmt.Printf("%05d\r", *count) 961 } 962 } 963 964 var count int 965 val := make([]byte, 1000) 966 require.NoError(t, db.Update(func(txn *Txn) error { 967 for i := 0; i < 1500; i++ { 968 key := fmt.Sprintf("%d", i) 969 rand.Read(val) 970 require.NoError(t, txn.Set([]byte(key), y.Copy(val))) 971 print(&count) 972 } 973 return nil 974 })) 975 976 count = 0 977 fmt.Println("\nWrites done. Iteration and updates starting...") 978 err = db.Update(func(txn *Txn) error { 979 it := txn.NewIterator(DefaultIteratorOptions) 980 defer it.Close() 981 for it.Rewind(); it.Valid(); it.Next() { 982 item := it.Item() 983 984 // Using Value() would cause deadlock. 985 // item.Value() 986 out, err := item.ValueCopy(nil) 987 require.NoError(t, err) 988 require.Equal(t, len(val), len(out)) 989 990 key := y.Copy(item.Key()) 991 rand.Read(val) 992 require.NoError(t, txn.Set(key, val)) 993 print(&count) 994 } 995 return nil 996 }) 997 require.NoError(t, err) 998 } 999 1000 func TestReadOnly(t *testing.T) { 1001 dir, err := ioutil.TempDir("", "badger") 1002 require.NoError(t, err) 1003 defer os.RemoveAll(dir) 1004 opts := getTestOptions(dir) 1005 1006 // Create the DB 1007 db, err := Open(opts) 1008 require.NoError(t, err) 1009 for i := 0; i < 10000; i++ { 1010 txnSet(t, db, []byte(fmt.Sprintf("key%d", i)), []byte(fmt.Sprintf("value%d", i)), 0x00) 1011 } 1012 1013 // Attempt a read-only open while it's open read-write. 1014 opts.ReadOnly = true 1015 _, err = Open(opts) 1016 require.Error(t, err) 1017 if err == ErrWindowsNotSupported { 1018 return 1019 } 1020 require.Contains(t, err.Error(), "Another process is using this Badger database") 1021 db.Close() 1022 1023 // Open one read-only 1024 opts.ReadOnly = true 1025 kv1, err := Open(opts) 1026 require.NoError(t, err) 1027 defer kv1.Close() 1028 1029 // Open another read-only 1030 kv2, err := Open(opts) 1031 require.NoError(t, err) 1032 defer kv2.Close() 1033 1034 // Attempt a read-write open while it's open for read-only 1035 opts.ReadOnly = false 1036 _, err = Open(opts) 1037 require.Error(t, err) 1038 require.Contains(t, err.Error(), "Another process is using this Badger database") 1039 1040 // Get a thing from the DB 1041 txn1 := kv1.NewTransaction(true) 1042 v1, err := txn1.Get([]byte("key1")) 1043 require.NoError(t, err) 1044 b1, err := v1.Value() 1045 require.NoError(t, err) 1046 require.Equal(t, b1, []byte("value1")) 1047 err = txn1.Commit() 1048 require.NoError(t, err) 1049 1050 // Get a thing from the DB via the other connection 1051 txn2 := kv2.NewTransaction(true) 1052 v2, err := txn2.Get([]byte("key2000")) 1053 require.NoError(t, err) 1054 b2, err := v2.Value() 1055 require.NoError(t, err) 1056 require.Equal(t, b2, []byte("value2000")) 1057 err = txn2.Commit() 1058 require.NoError(t, err) 1059 1060 // Attempt to set a value on a read-only connection 1061 txn := kv1.NewTransaction(true) 1062 err = txn.SetWithMeta([]byte("key"), []byte("value"), 0x00) 1063 require.Error(t, err) 1064 require.Contains(t, err.Error(), "No sets or deletes are allowed in a read-only transaction") 1065 err = txn.Commit() 1066 require.NoError(t, err) 1067 } 1068 1069 func TestLSMOnly(t *testing.T) { 1070 dir, err := ioutil.TempDir("", "badger") 1071 require.NoError(t, err) 1072 defer os.RemoveAll(dir) 1073 1074 opts := LSMOnlyOptions 1075 opts.Dir = dir 1076 opts.ValueDir = dir 1077 1078 dopts := DefaultOptions 1079 require.NotEqual(t, dopts.ValueThreshold, opts.ValueThreshold) 1080 require.NotEqual(t, dopts.ValueLogFileSize, opts.ValueLogFileSize) 1081 1082 dopts.ValueThreshold = 1 << 16 1083 _, err = Open(dopts) 1084 require.Equal(t, ErrValueThreshold, err) 1085 1086 db, err := Open(opts) 1087 require.NoError(t, err) 1088 if err != nil { 1089 t.Fatal(err) 1090 } 1091 defer db.Close() 1092 1093 for i := 0; i < 5000; i++ { 1094 value := make([]byte, 64000) 1095 _, err = rand.Read(value) 1096 require.NoError(t, err) 1097 1098 txnSet(t, db, []byte(fmt.Sprintf("key%d", i)), value, 0x00) 1099 } 1100 } 1101 1102 func TestMinReadTs(t *testing.T) { 1103 runBadgerTest(t, nil, func(t *testing.T, db *DB) { 1104 for i := 0; i < 10; i++ { 1105 require.NoError(t, db.Update(func(txn *Txn) error { 1106 return txn.Set([]byte("x"), []byte("y")) 1107 })) 1108 } 1109 time.Sleep(time.Second) 1110 require.Equal(t, uint64(10), db.orc.readTs()) 1111 min := db.getCompactSafeTs() 1112 require.Equal(t, uint64(9), min) 1113 1114 readTxn := db.NewTransaction(false) 1115 for i := 0; i < 10; i++ { 1116 require.NoError(t, db.Update(func(txn *Txn) error { 1117 return txn.Set([]byte("x"), []byte("y")) 1118 })) 1119 } 1120 require.Equal(t, uint64(20), db.orc.readTs()) 1121 time.Sleep(time.Second) 1122 require.Equal(t, min, db.getCompactSafeTs()) 1123 readTxn.Discard() 1124 time.Sleep(time.Second) 1125 require.Equal(t, uint64(19), db.getCompactSafeTs()) 1126 // // The minReadTS can only be increase by newer txn done. 1127 // readTxn = db.NewTransaction(false) 1128 // readTxn.Discard() 1129 // time.Sleep(time.Second) 1130 // require.Equal(t, uint64(20), db.getCompactSafeTs()) 1131 1132 for i := 0; i < 10; i++ { 1133 db.View(func(txn *Txn) error { 1134 return nil 1135 }) 1136 } 1137 time.Sleep(time.Second) 1138 require.Equal(t, uint64(20), db.getCompactSafeTs()) 1139 }) 1140 } 1141 1142 type testFilter struct{} 1143 1144 var ( 1145 userMetaDrop = []byte{0, 0} 1146 userMetaDelete = []byte{0} 1147 ) 1148 1149 func (f *testFilter) Filter(key, val, userMeta []byte) Decision { 1150 if bytes.Equal(userMeta, userMetaDrop) { 1151 return DecisionDrop 1152 } else if bytes.Equal(userMeta, userMetaDelete) { 1153 return DecisionMarkTombstone 1154 } 1155 return DecisionKeep 1156 } 1157 1158 func TestCompactionFilter(t *testing.T) { 1159 dir, err := ioutil.TempDir("", "badger") 1160 require.NoError(t, err) 1161 defer os.RemoveAll(dir) 1162 opts := getTestOptions(dir) 1163 opts.ValueThreshold = 8 * 1024 1164 opts.TableBuilderOptions.MaxTableSize = 32 * 1024 1165 opts.MaxMemTableSize = 32 * 1024 1166 opts.NumMemtables = 2 1167 opts.NumLevelZeroTables = 1 1168 opts.NumLevelZeroTablesStall = 2 1169 opts.CompactionFilterFactory = func(targetLevel int, smallest, biggest []byte) CompactionFilter { 1170 return &testFilter{} 1171 } 1172 // This case depend on level's size, so disable compression for now. 1173 opts.TableBuilderOptions.CompressionPerLevel = getTestCompression(options.None) 1174 db, err := Open(opts) 1175 require.NoError(t, err) 1176 defer db.Close() 1177 val := make([]byte, 1024*4) 1178 // Insert 50 entries that will be kept. 1179 for i := 0; i < 50; i++ { 1180 err = db.Update(func(txn *Txn) error { 1181 key := []byte(fmt.Sprintf("key%d", i)) 1182 // Entries without userMeta will result in DecisionKeep in testFilter. 1183 return txn.Set(key, val) 1184 }) 1185 require.NoError(t, err) 1186 } 1187 // Insert keys for delete decision and drop decision. 1188 for i := 0; i < 100; i++ { 1189 db.Update(func(txn *Txn) error { 1190 key := []byte(fmt.Sprintf("key%d", i)) 1191 if i%2 == 0 { 1192 // Entries with userMetaDelete will result in DecisionMarkTombstone in testFilter. 1193 txn.SetWithMetaSlice(key, val, userMetaDelete) 1194 } else { 1195 // Entries with userMetaDrop will result in DecisionDrop in testFilter. 1196 txn.SetWithMetaSlice(key, val, userMetaDrop) 1197 } 1198 return nil 1199 }) 1200 } 1201 var deleteNotFoundCount, dropAppearOldCount int 1202 err = db.View(func(txn *Txn) error { 1203 for i := 0; i < 50; i++ { 1204 key := []byte(fmt.Sprintf("key%d", i)) 1205 item, _ := txn.Get(key) 1206 if i%2 == 0 { 1207 // For delete decision, the old value can not be read. 1208 if item == nil { 1209 deleteNotFoundCount++ 1210 } 1211 } else { 1212 // For dropped entry, since no tombstone left, the old value appear again. 1213 if item != nil && len(item.UserMeta()) == 0 { 1214 dropAppearOldCount++ 1215 } 1216 } 1217 } 1218 return nil 1219 }) 1220 require.True(t, deleteNotFoundCount > 0) 1221 // Since compaction always pick upper level first, the never has chance to reappear old value. 1222 // require.True(t, dropAppearOldCount > 0) 1223 } 1224 1225 func (f *testFilter) Guards() []Guard { 1226 return []Guard{ 1227 { 1228 Prefix: nil, 1229 MatchLen: 1, 1230 MinSize: 64, 1231 }, 1232 { 1233 Prefix: []byte("b"), 1234 MatchLen: 4, 1235 MinSize: 128, 1236 }, 1237 { 1238 Prefix: []byte("bx"), 1239 MatchLen: 5, 1240 MinSize: 128, 1241 }, 1242 { 1243 Prefix: []byte("c"), 1244 MatchLen: 3, 1245 MinSize: 128, 1246 }, 1247 } 1248 } 1249 1250 func TestSearchGuard(t *testing.T) { 1251 filter := &testFilter{} 1252 guards := filter.Guards() 1253 tests := []struct { 1254 key string 1255 guardPrefix string 1256 }{ 1257 {"0", ""}, 1258 {"br", "b"}, 1259 {"bn", "b"}, 1260 {"bx", "bx"}, 1261 {"crz", "c"}, 1262 } 1263 for _, tt := range tests { 1264 guard := searchGuard([]byte(tt.key), guards) 1265 require.Equal(t, string(guard.Prefix), tt.guardPrefix) 1266 } 1267 } 1268 1269 func TestShouldFinishFile(t *testing.T) { 1270 tests1 := []struct { 1271 key []byte 1272 lastKey []byte 1273 finish bool 1274 }{ 1275 {[]byte("k2"), nil, false}, 1276 {[]byte("k2"), []byte("k1"), true}, 1277 {[]byte("k12"), []byte("k1"), true}, 1278 {[]byte("k234"), []byte("k233"), false}, 1279 {[]byte("k241"), []byte("k233"), true}, 1280 {[]byte("l233"), []byte("k233"), true}, 1281 } 1282 guard := &Guard{Prefix: []byte("k"), MatchLen: 3, MinSize: 64} 1283 for _, tt := range tests1 { 1284 finish := shouldFinishFile(y.KeyWithTs(tt.key, 0), y.KeyWithTs(tt.lastKey, 0), guard, 100, 100) 1285 require.Equal(t, tt.finish, finish) 1286 } 1287 // A guard prefix change always finish the file even if the MinSize has not been reached. 1288 require.Equal(t, shouldFinishFile(y.KeyWithTs([]byte("l11"), 0), y.KeyWithTs([]byte("k11"), 0), guard, 1, 100), true) 1289 // guard prefix match, but matchLen changed, must reach MinSize to finish file. 1290 require.Equal(t, shouldFinishFile(y.KeyWithTs([]byte("k12"), 0), y.KeyWithTs([]byte("k11"), 0), guard, 1, 100), false) 1291 require.Equal(t, shouldFinishFile(y.KeyWithTs([]byte("k12"), 0), y.KeyWithTs([]byte("k11"), 0), guard, 65, 100), true) 1292 // table max size has reached always finish the file. 1293 require.Equal(t, shouldFinishFile(y.KeyWithTs([]byte("k111"), 0), y.KeyWithTs([]byte("k110"), 0), guard, 33, 32), true) 1294 } 1295 1296 func TestIterateVLog(t *testing.T) { 1297 dir, err := ioutil.TempDir("", "badger") 1298 require.NoError(t, err) 1299 defer os.RemoveAll(dir) 1300 opts := getTestOptions(dir) 1301 opts.TableBuilderOptions.MaxTableSize = 1 << 20 1302 opts.MaxMemTableSize = 1 << 20 1303 opts.ValueLogFileSize = 1 << 20 1304 opts.ValueThreshold = 1000 1305 opts.ValueLogMaxNumFiles = 1000 1306 db, err := Open(opts) 1307 require.NoError(t, err) 1308 defer db.Close() 1309 for i := 0; i < 3000; i++ { 1310 err = db.Update(func(txn *Txn) error { 1311 key := []byte(fmt.Sprintf("key%d", i)) 1312 val := make([]byte, 1024) 1313 return txn.Set(key, val) 1314 }) 1315 require.NoError(t, err) 1316 } 1317 1318 var iterKeys [][]byte 1319 var iterVals [][]byte 1320 err = db.IterateVLog(0, func(e Entry) { 1321 key := y.SafeCopy(nil, e.Key.UserKey) 1322 iterKeys = append(iterKeys, key) 1323 iterVals = append(iterVals, y.SafeCopy(nil, e.Value)) 1324 }) 1325 require.Nil(t, err) 1326 require.Equal(t, 3000, len(iterKeys)) 1327 expectedVal := make([]byte, 1024) 1328 for i, key := range iterKeys { 1329 require.Equal(t, fmt.Sprintf("key%d", i), string(key)) 1330 require.Equal(t, expectedVal, iterVals[i]) 1331 } 1332 offset := db.GetVLogOffset() 1333 for i := 3000; i < 5000; i++ { 1334 err = db.Update(func(txn *Txn) error { 1335 key := []byte(fmt.Sprintf("key%d", i)) 1336 return txn.Set(key, make([]byte, 999)) 1337 }) 1338 require.NoError(t, err) 1339 } 1340 iterKeys = iterKeys[:0] 1341 iterVals = iterVals[:0] 1342 err = db.IterateVLog(offset, func(e Entry) { 1343 key := y.SafeCopy(nil, e.Key.UserKey) 1344 iterKeys = append(iterKeys, key) 1345 iterVals = append(iterVals, y.SafeCopy(nil, e.Value)) 1346 }) 1347 require.Len(t, iterKeys, 2000) 1348 expectedVal = make([]byte, 999) 1349 for i, key := range iterKeys { 1350 require.Equal(t, fmt.Sprintf("key%d", i+3000), string(key)) 1351 require.Equal(t, expectedVal, iterVals[i]) 1352 } 1353 } 1354 1355 func TestMultiGet(t *testing.T) { 1356 dir, err := ioutil.TempDir("", "badger") 1357 require.NoError(t, err) 1358 defer os.RemoveAll(dir) 1359 opts := getTestOptions(dir) 1360 opts.ValueThreshold = 512 1361 db, err := Open(opts) 1362 require.NoError(t, err) 1363 defer db.Close() 1364 var keys [][]byte 1365 for i := 0; i < 1000; i++ { 1366 keys = append(keys, []byte(fmt.Sprintf("key%d", i))) 1367 } 1368 for i := 0; i < 1000; i++ { 1369 err = db.Update(func(txn *Txn) error { 1370 val := make([]byte, 513) 1371 return txn.SetWithMetaSlice(keys[i], val, make([]byte, 16)) 1372 }) 1373 require.NoError(t, err) 1374 } 1375 txn := db.NewTransaction(false) 1376 defer txn.Discard() 1377 items, err := txn.MultiGet(keys) 1378 require.NoError(t, err) 1379 for i := range keys { 1380 require.NotNil(t, items[i]) 1381 } 1382 // Update more data to trigger compaction. 1383 for i := 0; i < 1000; i++ { 1384 err = db.Update(func(txn *Txn) error { 1385 val := make([]byte, 500) 1386 return txn.Set(keys[i], val) 1387 }) 1388 require.NoError(t, err) 1389 } 1390 // items can be safely accessed. 1391 total := 0 1392 for _, item := range items { 1393 total += int(item.UserMeta()[15]) 1394 } 1395 require.Equal(t, 0, total) 1396 } 1397 1398 func buildSst(t *testing.T, keys [][]byte, vals [][]byte) *os.File { 1399 filename := fmt.Sprintf("%s%s%d.sst", os.TempDir(), string(os.PathSeparator), rand.Int63()) 1400 f, err := y.OpenSyncedFile(filename, true) 1401 require.NoError(t, err) 1402 builder := sstable.NewExternalTableBuilder(f, nil, DefaultOptions.TableBuilderOptions, options.ZSTD) 1403 1404 for i, k := range keys { 1405 err := builder.Add(y.KeyWithTs(k, 0), y.ValueStruct{Value: vals[i], Meta: 0, UserMeta: []byte{0}}) 1406 require.NoError(t, err) 1407 } 1408 _, err = builder.Finish() 1409 require.NoError(t, err) 1410 return f 1411 } 1412 1413 func TestIngestSimple(t *testing.T) { 1414 var ingestKeys [][]byte 1415 for i := 1000; i < 2000; i++ { 1416 ingestKeys = append(ingestKeys, []byte(fmt.Sprintf("key%04d", i))) 1417 } 1418 f := buildSst(t, ingestKeys, ingestKeys) 1419 defer os.Remove(f.Name()) 1420 1421 dir, err := ioutil.TempDir("", "badger") 1422 require.NoError(t, err) 1423 defer os.RemoveAll(dir) 1424 opts := getTestOptions(dir) 1425 opts.ValueThreshold = 512 1426 db, err := Open(opts) 1427 require.NoError(t, err) 1428 1429 var keys [][]byte 1430 for i := 0; i < 1000; i++ { 1431 keys = append(keys, []byte(fmt.Sprintf("key%d", i))) 1432 } 1433 for i := 0; i < 1000; i++ { 1434 err = db.Update(func(txn *Txn) error { 1435 return txn.SetWithMetaSlice(keys[i], keys[i], []byte{0}) 1436 }) 1437 require.NoError(t, err) 1438 } 1439 1440 cnt, err := db.IngestExternalFiles([]ExternalTableSpec{{f.Name()}}) 1441 require.NoError(t, err) 1442 require.Equal(t, 1, cnt) 1443 1444 txn := db.NewTransaction(false) 1445 for _, k := range append(keys, ingestKeys...) { 1446 item, err := txn.Get(k) 1447 require.NoError(t, err) 1448 v, err := item.Value() 1449 require.NoError(t, err) 1450 require.Equal(t, k, v) 1451 } 1452 } 1453 1454 func TestIngestOverwrite(t *testing.T) { 1455 var ingestKeys, ingestVals [][]byte 1456 for i := 0; i < 1000; i++ { 1457 ingestKeys = append(ingestKeys, []byte(fmt.Sprintf("key%07d", i))) 1458 ingestVals = append(ingestVals, []byte(fmt.Sprintf("val%07d", i))) 1459 } 1460 ingestKeys = append(ingestKeys, []byte(fmt.Sprintf("key%07d", 9000))) 1461 ingestVals = append(ingestVals, []byte(fmt.Sprintf("val%07d", 9000))) 1462 f := buildSst(t, ingestKeys, ingestVals) 1463 defer os.Remove(f.Name()) 1464 1465 dir, err := ioutil.TempDir("", "badger") 1466 require.NoError(t, err) 1467 defer os.RemoveAll(dir) 1468 opts := getTestOptions(dir) 1469 opts.ValueThreshold = 512 1470 opts.NumLevelZeroTables = 1 1471 opts.NumLevelZeroTablesStall = 2 1472 opts.LevelOneSize *= 5 1473 db, err := Open(opts) 1474 require.NoError(t, err) 1475 1476 for i := 1000; i < 20000; i++ { 1477 if i == 9000 { 1478 continue 1479 } 1480 key := []byte(fmt.Sprintf("key%07d", i)) 1481 err = db.Update(func(txn *Txn) error { 1482 return txn.SetWithMetaSlice(key, key, []byte{0}) 1483 }) 1484 require.NoError(t, err) 1485 } 1486 1487 cnt, err := db.IngestExternalFiles([]ExternalTableSpec{{f.Name()}}) 1488 require.NoError(t, err) 1489 require.Equal(t, 1, cnt) 1490 1491 txn := db.NewTransaction(false) 1492 for i, k := range ingestKeys { 1493 item, err := txn.Get(k) 1494 require.NoError(t, err) 1495 v, err := item.Value() 1496 require.NoError(t, err) 1497 require.Equal(t, ingestVals[i], v) 1498 } 1499 } 1500 1501 func TestIngestWhileWrite(t *testing.T) { 1502 var ingestKeys, ingestVals [][][]byte 1503 var files []*os.File 1504 for n := 0; n < 10; n++ { 1505 var keys, vals [][]byte 1506 for i := n * 1000; i < (n+1)*1000; i++ { 1507 keys = append(keys, []byte(fmt.Sprintf("key%05d", i))) 1508 vals = append(vals, []byte(fmt.Sprintf("val%05d", i))) 1509 } 1510 f := buildSst(t, keys, vals) 1511 files = append(files, f) 1512 ingestKeys = append(ingestKeys, keys) 1513 ingestVals = append(ingestVals, vals) 1514 } 1515 defer func() { 1516 for _, f := range files { 1517 os.Remove(f.Name()) 1518 } 1519 }() 1520 1521 dir, err := ioutil.TempDir("", "badger") 1522 require.NoError(t, err) 1523 defer os.RemoveAll(dir) 1524 opts := getTestOptions(dir) 1525 opts.ValueThreshold = 512 1526 db, err := Open(opts) 1527 require.NoError(t, err) 1528 1529 stop, done := make(chan struct{}), make(chan struct{}) 1530 go func() { 1531 for { 1532 select { 1533 case <-stop: 1534 close(done) 1535 return 1536 default: 1537 } 1538 1539 err = db.Update(func(txn *Txn) error { 1540 for n := 0; n < 10; n++ { 1541 key := []byte(fmt.Sprintf("key%05d", n*1000+1)) 1542 if err := txn.Set(key, key); err != nil { 1543 return err 1544 } 1545 } 1546 return nil 1547 }) 1548 require.NoError(t, err) 1549 } 1550 }() 1551 1552 specs := make([]ExternalTableSpec, len(files)) 1553 for i := range specs { 1554 specs[i] = ExternalTableSpec{ 1555 Filename: files[i].Name(), 1556 } 1557 } 1558 cnt, err := db.IngestExternalFiles(specs) 1559 require.NoError(t, err) 1560 require.Equal(t, len(files), cnt) 1561 close(stop) 1562 <-done 1563 1564 txn := db.NewTransaction(false) 1565 for i, keys := range ingestKeys { 1566 vals := ingestVals[i] 1567 for j, k := range keys { 1568 val := vals[j] 1569 item, err := txn.Get(k) 1570 require.NoError(t, err) 1571 v, err := item.Value() 1572 require.NoError(t, err) 1573 if !bytes.Equal(val, v) { 1574 require.Equal(t, k, v) 1575 } 1576 } 1577 } 1578 } 1579 1580 func TestIngestSplit(t *testing.T) { 1581 var ingestKeys [][]byte 1582 var files []*os.File 1583 { 1584 keys := [][]byte{[]byte("c"), []byte("d")} 1585 ingestKeys = append(ingestKeys, keys...) 1586 files = append(files, buildSst(t, keys, keys)) 1587 } 1588 { 1589 keys := [][]byte{[]byte("e"), []byte("h")} 1590 ingestKeys = append(ingestKeys, keys...) 1591 files = append(files, buildSst(t, keys, keys)) 1592 } 1593 { 1594 keys := [][]byte{[]byte("l"), []byte("o")} 1595 ingestKeys = append(ingestKeys, keys...) 1596 files = append(files, buildSst(t, keys, keys)) 1597 } 1598 defer func() { 1599 for _, f := range files { 1600 os.Remove(f.Name()) 1601 } 1602 }() 1603 1604 dir, err := ioutil.TempDir("", "badger") 1605 require.NoError(t, err) 1606 defer os.RemoveAll(dir) 1607 opts := getTestOptions(dir) 1608 opts.ValueThreshold = 512 1609 opts.NumLevelZeroTables = 10 1610 opts.NumLevelZeroTablesStall = 20 1611 db, err := Open(opts) 1612 require.NoError(t, err) 1613 1614 keys := [][]byte{[]byte("a"), []byte("b"), []byte("i"), []byte("k"), []byte("x"), []byte("z")} 1615 err = db.Update(func(txn *Txn) error { 1616 for _, k := range keys { 1617 if err := txn.Set(k, k); err != nil { 1618 return err 1619 } 1620 } 1621 return nil 1622 }) 1623 require.NoError(t, err) 1624 1625 wg := db.flushMemTable() 1626 wg.Wait() 1627 1628 l0 := db.lc.levels[0] 1629 l1 := db.lc.levels[1] 1630 l0.Lock() 1631 l1.Lock() 1632 l1.tables = append(l1.tables, l0.tables[0]) 1633 l0.tables[0] = nil 1634 l0.tables = l0.tables[:0] 1635 l0.Unlock() 1636 l1.Unlock() 1637 1638 specs := make([]ExternalTableSpec, len(files)) 1639 for i := range specs { 1640 specs[i] = ExternalTableSpec{ 1641 Filename: files[i].Name(), 1642 } 1643 } 1644 cnt, err := db.IngestExternalFiles(specs) 1645 require.NoError(t, err) 1646 require.Equal(t, 3, cnt) 1647 1648 txn := db.NewTransaction(false) 1649 for _, k := range append(keys, ingestKeys...) { 1650 item, err := txn.Get(k) 1651 require.NoError(t, err, string(k)) 1652 v, err := item.Value() 1653 require.NoError(t, err) 1654 require.Equal(t, k, v) 1655 } 1656 1657 l1.RLock() 1658 tblCnt := 0 1659 for _, t := range l1.tables { 1660 if t.(*sstable.Table).HasGlobalTs() { 1661 tblCnt++ 1662 } 1663 } 1664 l1.RUnlock() 1665 require.Equal(t, 2, tblCnt) 1666 } 1667 1668 func TestDeleteRange(t *testing.T) { 1669 runBadgerTest(t, nil, func(t *testing.T, db *DB) { 1670 data := func(i int) []byte { 1671 return []byte(fmt.Sprintf("%06d", i)) 1672 } 1673 n := 20000 1674 1675 txn := db.NewTransaction(true) 1676 for i := 0; i < n; i++ { 1677 require.NoError(t, txn.Set(data(i), make([]byte, 128))) 1678 } 1679 require.NoError(t, txn.Commit()) 1680 require.NoError(t, db.validate()) 1681 1682 db.DeleteFilesInRange(data(0), data(n/2)) 1683 1684 // wait for compaction. 1685 time.Sleep(2 * time.Second) 1686 1687 txn = db.NewTransaction(false) 1688 for i := 0; i < n/4; i++ { 1689 _, err := txn.Get(data(i)) 1690 require.Equal(t, ErrKeyNotFound, err) 1691 } 1692 for i := n / 2; i < n; i++ { 1693 _, err := txn.Get(data(i)) 1694 require.NoError(t, err) 1695 } 1696 }) 1697 } 1698 1699 func ExampleOpen() { 1700 dir, err := ioutil.TempDir("", "badger") 1701 if err != nil { 1702 log.Fatal(err) 1703 } 1704 defer os.RemoveAll(dir) 1705 opts := DefaultOptions 1706 opts.Dir = dir 1707 opts.ValueDir = dir 1708 db, err := Open(opts) 1709 if err != nil { 1710 log.Fatal(err) 1711 } 1712 defer db.Close() 1713 1714 err = db.View(func(txn *Txn) error { 1715 _, err := txn.Get([]byte("key")) 1716 // We expect ErrKeyNotFound 1717 fmt.Println(err) 1718 return nil 1719 }) 1720 1721 if err != nil { 1722 log.Fatal(err) 1723 } 1724 1725 txn := db.NewTransaction(true) // Read-write txn 1726 err = txn.Set([]byte("key"), []byte("value")) 1727 if err != nil { 1728 log.Fatal(err) 1729 } 1730 err = txn.Commit() 1731 if err != nil { 1732 log.Fatal(err) 1733 } 1734 1735 err = db.View(func(txn *Txn) error { 1736 item, err := txn.Get([]byte("key")) 1737 if err != nil { 1738 return err 1739 } 1740 val, err := item.Value() 1741 if err != nil { 1742 return err 1743 } 1744 fmt.Printf("%s\n", string(val)) 1745 return nil 1746 }) 1747 1748 if err != nil { 1749 log.Fatal(err) 1750 } 1751 1752 // Output: 1753 // Key not found 1754 // value 1755 } 1756 1757 func ExampleTxn_NewIterator() { 1758 dir, err := ioutil.TempDir("", "badger") 1759 if err != nil { 1760 log.Fatal(err) 1761 } 1762 defer os.RemoveAll(dir) 1763 1764 opts := DefaultOptions 1765 opts.Dir = dir 1766 opts.ValueDir = dir 1767 1768 db, err := Open(opts) 1769 if err != nil { 1770 log.Fatal(err) 1771 } 1772 defer db.Close() 1773 1774 bkey := func(i int) []byte { 1775 return []byte(fmt.Sprintf("%09d", i)) 1776 } 1777 bval := func(i int) []byte { 1778 return []byte(fmt.Sprintf("%025d", i)) 1779 } 1780 1781 txn := db.NewTransaction(true) 1782 1783 // Fill in 1000 items 1784 n := 1000 1785 for i := 0; i < n; i++ { 1786 err := txn.Set(bkey(i), bval(i)) 1787 if err != nil { 1788 log.Fatal(err) 1789 } 1790 } 1791 1792 err = txn.Commit() 1793 if err != nil { 1794 log.Fatal(err) 1795 } 1796 1797 // Iterate over 1000 items 1798 var count int 1799 err = db.View(func(txn *Txn) error { 1800 it := txn.NewIterator(DefaultIteratorOptions) 1801 defer it.Close() 1802 for it.Rewind(); it.Valid(); it.Next() { 1803 count++ 1804 } 1805 return nil 1806 }) 1807 if err != nil { 1808 log.Fatal(err) 1809 } 1810 fmt.Printf("Counted %d elements", count) 1811 // Output: 1812 // Counted 1000 elements 1813 } 1814 1815 func TestRemoteCompaction(t *testing.T) { 1816 dir, err := ioutil.TempDir("", "badger") 1817 require.NoError(t, err) 1818 defer os.RemoveAll(dir) 1819 remoteAddr := "127.0.0.1:4080" 1820 compactionServer, err := NewCompactionServer(remoteAddr) 1821 require.NoError(t, err) 1822 go compactionServer.Run() 1823 defer compactionServer.Close() 1824 opts := getTestOptions(dir) 1825 opts.ValueThreshold = 0 1826 opts.RemoteCompactionAddr = remoteAddr 1827 opts.TableBuilderOptions.MaxTableSize = 32 * 1024 1828 opts.MaxMemTableSize = 32 * 1024 1829 opts.NumMemtables = 2 1830 opts.NumLevelZeroTables = 1 1831 opts.NumLevelZeroTablesStall = 2 1832 // This case depend on level's size, so disable compression for now. 1833 opts.TableBuilderOptions.CompressionPerLevel = getTestCompression(options.None) 1834 db, err := Open(opts) 1835 require.NoError(t, err) 1836 defer db.Close() 1837 for i := 0; i < 128; i++ { 1838 err = db.Update(func(txn *Txn) error { 1839 key := []byte(fmt.Sprintf("key%03d", i)) 1840 val := make([]byte, 1024) 1841 copy(val, key) 1842 return txn.Set(key, val) 1843 }) 1844 require.NoError(t, err) 1845 } 1846 1847 for i := 0; i < 512; i++ { 1848 err = db.Update(func(txn *Txn) error { 1849 key := []byte(fmt.Sprintf("key%03d", rand.Intn(128))) 1850 val := make([]byte, 1024*4) 1851 copy(val, key) 1852 return txn.Set(key, val) 1853 }) 1854 require.NoError(t, err) 1855 } 1856 err = db.View(func(txn *Txn) error { 1857 it := txn.NewIterator(DefaultIteratorOptions) 1858 defer it.Close() 1859 var i int 1860 for it.Rewind(); it.Valid(); it.Next() { 1861 require.EqualValues(t, string(it.Item().Key()), fmt.Sprintf("key%03d", i)) 1862 require.True(t, bytes.HasPrefix(it.Item().vptr, it.Item().Key())) 1863 i++ 1864 } 1865 return nil 1866 }) 1867 } 1868 1869 func TestNonDirectIO(t *testing.T) { 1870 // if the dir is a tmpfs (or other file system which doesn't support directio), badger 1871 // should still work 1872 dir, err := ioutil.TempDir("", "badger") 1873 require.NoError(t, err) 1874 defer os.RemoveAll(dir) 1875 remoteAddr := "127.0.0.1:4080" 1876 compactionServer, err := NewCompactionServer(remoteAddr) 1877 require.NoError(t, err) 1878 go compactionServer.Run() 1879 defer compactionServer.Close() 1880 opts := getTestOptions(dir) 1881 db, err := Open(opts) 1882 require.NoError(t, err) 1883 defer db.Close() 1884 1885 for i := 0; i < 512; i++ { 1886 err = db.Update(func(txn *Txn) error { 1887 key := []byte(fmt.Sprintf("key%03d", rand.Intn(128))) 1888 val := make([]byte, 1024*4) 1889 copy(val, key) 1890 return txn.Set(key, val) 1891 }) 1892 require.NoError(t, err) 1893 } 1894 // flushing memTable shouldn't hang forever even on file system which doesn't support directio 1895 db.flushMemTable().Wait() 1896 }