github.com/weaviate/weaviate@v1.24.6/adapters/repos/db/lsmkv/strategies_replace_integration_test.go (about) 1 // _ _ 2 // __ _____ __ ___ ___ __ _| |_ ___ 3 // \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ 4 // \ V V / __/ (_| |\ V /| | (_| | || __/ 5 // \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| 6 // 7 // Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. 8 // 9 // CONTACT: hello@weaviate.io 10 // 11 12 //go:build integrationTest 13 // +build integrationTest 14 15 package lsmkv 16 17 import ( 18 "context" 19 "fmt" 20 "testing" 21 22 "github.com/stretchr/testify/assert" 23 "github.com/stretchr/testify/require" 24 "github.com/weaviate/weaviate/entities/cyclemanager" 25 ) 26 27 func TestReplaceStrategy(t *testing.T) { 28 ctx := testCtx() 29 tests := bucketIntegrationTests{ 30 { 31 name: "replaceInsertAndUpdate", 32 f: replaceInsertAndUpdate, 33 opts: []BucketOption{ 34 WithStrategy(StrategyReplace), 35 }, 36 }, 37 { 38 name: "replaceInsertAndUpdate_WithSecondaryKeys", 39 f: replaceInsertAndUpdate_WithSecondaryKeys, 40 opts: []BucketOption{ 41 WithStrategy(StrategyReplace), 42 WithSecondaryIndices(1), 43 }, 44 }, 45 { 46 name: "replaceInsertAndDelete", 47 f: replaceInsertAndDelete, 48 opts: []BucketOption{ 49 WithStrategy(StrategyReplace), 50 }, 51 }, 52 { 53 name: "replaceCursors", 54 f: replaceCursors, 55 opts: []BucketOption{ 56 WithStrategy(StrategyReplace), 57 }, 58 }, 59 } 60 tests.run(ctx, t) 61 } 62 63 func replaceInsertAndUpdate(ctx context.Context, t *testing.T, opts []BucketOption) { 64 dirName := t.TempDir() 65 66 t.Run("memtable-only", func(t *testing.T) { 67 b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, 68 cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) 69 require.Nil(t, err) 70 71 // so big it effectively never triggers as part of this test 72 b.SetMemtableThreshold(1e9) 73 74 t.Run("set original values and verify", func(t *testing.T) { 75 key1 := []byte("key-1") 76 key2 := []byte("key-2") 77 key3 := []byte("key-3") 78 orig1 := []byte("original value for key1") 79 orig2 := []byte("original value for key2") 80 orig3 := []byte("original value for key3") 81 82 err = b.Put(key1, orig1) 83 require.Nil(t, err) 84 err = b.Put(key2, orig2) 85 require.Nil(t, err) 86 err = b.Put(key3, orig3) 87 require.Nil(t, err) 88 89 assert.Equal(t, 3, b.Count()) 90 assert.Equal(t, 0, b.CountAsync()) 91 92 res, err := b.Get(key1) 93 require.Nil(t, err) 94 assert.Equal(t, res, orig1) 95 res, err = b.Get(key2) 96 require.Nil(t, err) 97 assert.Equal(t, res, orig2) 98 res, err = b.Get(key3) 99 require.Nil(t, err) 100 assert.Equal(t, res, orig3) 101 }) 102 103 t.Run("replace some, keep one", func(t *testing.T) { 104 key1 := []byte("key-1") 105 key2 := []byte("key-2") 106 key3 := []byte("key-3") 107 orig1 := []byte("original value for key1") 108 replaced2 := []byte("updated value for key2") 109 replaced3 := []byte("updated value for key3") 110 111 err = b.Put(key2, replaced2) 112 require.Nil(t, err) 113 err = b.Put(key3, replaced3) 114 require.Nil(t, err) 115 116 assert.Equal(t, 3, b.Count()) 117 assert.Equal(t, 0, b.CountAsync()) 118 119 res, err := b.Get(key1) 120 require.Nil(t, err) 121 assert.Equal(t, res, orig1) 122 res, err = b.Get(key2) 123 require.Nil(t, err) 124 assert.Equal(t, res, replaced2) 125 res, err = b.Get(key3) 126 require.Nil(t, err) 127 assert.Equal(t, res, replaced3) 128 }) 129 }) 130 131 t.Run("with single flush in between updates", func(t *testing.T) { 132 b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, 133 cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) 134 require.Nil(t, err) 135 136 // so big it effectively never triggers as part of this test 137 b.SetMemtableThreshold(1e9) 138 139 t.Run("set original values and verify", func(t *testing.T) { 140 key1 := []byte("key-1") 141 key2 := []byte("key-2") 142 key3 := []byte("key-3") 143 orig1 := []byte("original value for key1") 144 orig2 := []byte("original value for key2") 145 orig3 := []byte("original value for key3") 146 147 err = b.Put(key1, orig1) 148 require.Nil(t, err) 149 err = b.Put(key2, orig2) 150 require.Nil(t, err) 151 err = b.Put(key3, orig3) 152 require.Nil(t, err) 153 154 res, err := b.Get(key1) 155 require.Nil(t, err) 156 assert.Equal(t, res, orig1) 157 res, err = b.Get(key2) 158 require.Nil(t, err) 159 assert.Equal(t, res, orig2) 160 res, err = b.Get(key3) 161 require.Nil(t, err) 162 assert.Equal(t, res, orig3) 163 }) 164 165 t.Run("flush memtable to disk", func(t *testing.T) { 166 require.Nil(t, b.FlushAndSwitch()) 167 }) 168 169 t.Run("count only objects on disk segment", func(t *testing.T) { 170 assert.Equal(t, 3, b.Count()) 171 assert.Equal(t, 3, b.CountAsync()) 172 }) 173 174 t.Run("replace some, keep one", func(t *testing.T) { 175 key1 := []byte("key-1") 176 key2 := []byte("key-2") 177 key3 := []byte("key-3") 178 orig1 := []byte("original value for key1") 179 replaced2 := []byte("updated value for key2") 180 replaced3 := []byte("updated value for key3") 181 182 err = b.Put(key2, replaced2) 183 require.Nil(t, err) 184 err = b.Put(key3, replaced3) 185 require.Nil(t, err) 186 187 // make sure that the updates aren't counted as additions 188 assert.Equal(t, 3, b.Count()) 189 190 // happens to be the same value, but that's just a coincidence, async 191 // ignores the memtable 192 assert.Equal(t, 3, b.CountAsync()) 193 194 res, err := b.Get(key1) 195 require.Nil(t, err) 196 assert.Equal(t, orig1, res) 197 res, err = b.Get(key2) 198 require.Nil(t, err) 199 assert.Equal(t, replaced2, res) 200 res, err = b.Get(key3) 201 require.Nil(t, err) 202 assert.Equal(t, replaced3, res) 203 }) 204 }) 205 206 t.Run("with a flush after the initial write and after the update", func(t *testing.T) { 207 b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, 208 cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) 209 require.Nil(t, err) 210 211 // so big it effectively never triggers as part of this test 212 b.SetMemtableThreshold(1e9) 213 214 t.Run("set original values and verify", func(t *testing.T) { 215 key1 := []byte("key-1") 216 key2 := []byte("key-2") 217 key3 := []byte("key-3") 218 orig1 := []byte("original value for key1") 219 orig2 := []byte("original value for key2") 220 orig3 := []byte("original value for key3") 221 222 err = b.Put(key1, orig1) 223 require.Nil(t, err) 224 err = b.Put(key2, orig2) 225 require.Nil(t, err) 226 err = b.Put(key3, orig3) 227 require.Nil(t, err) 228 229 res, err := b.Get(key1) 230 require.Nil(t, err) 231 assert.Equal(t, res, orig1) 232 res, err = b.Get(key2) 233 require.Nil(t, err) 234 assert.Equal(t, res, orig2) 235 res, err = b.Get(key3) 236 require.Nil(t, err) 237 assert.Equal(t, res, orig3) 238 }) 239 240 t.Run("flush memtable to disk", func(t *testing.T) { 241 require.Nil(t, b.FlushAndSwitch()) 242 }) 243 244 t.Run("replace some, keep one", func(t *testing.T) { 245 key1 := []byte("key-1") 246 key2 := []byte("key-2") 247 key3 := []byte("key-3") 248 orig1 := []byte("original value for key1") 249 replaced2 := []byte("updated value for key2") 250 replaced3 := []byte("updated value for key3") 251 252 err = b.Put(key2, replaced2) 253 require.Nil(t, err) 254 err = b.Put(key3, replaced3) 255 require.Nil(t, err) 256 257 // Flush before verifying! 258 require.Nil(t, b.FlushAndSwitch()) 259 260 res, err := b.Get(key1) 261 require.Nil(t, err) 262 assert.Equal(t, res, orig1) 263 res, err = b.Get(key2) 264 require.Nil(t, err) 265 assert.Equal(t, res, replaced2) 266 res, err = b.Get(key3) 267 require.Nil(t, err) 268 assert.Equal(t, res, replaced3) 269 }) 270 271 t.Run("count objects over several segments", func(t *testing.T) { 272 assert.Equal(t, 3, b.Count()) 273 assert.Equal(t, 3, b.CountAsync()) 274 }) 275 }) 276 277 t.Run("update in memtable, then do an orderly shutdown, and re-init", func(t *testing.T) { 278 b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, 279 cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) 280 require.Nil(t, err) 281 282 // so big it effectively never triggers as part of this test 283 b.SetMemtableThreshold(1e9) 284 285 t.Run("set original values and verify", func(t *testing.T) { 286 key1 := []byte("key-1") 287 key2 := []byte("key-2") 288 key3 := []byte("key-3") 289 orig1 := []byte("original value for key1") 290 orig2 := []byte("original value for key2") 291 orig3 := []byte("original value for key3") 292 293 err = b.Put(key1, orig1) 294 require.Nil(t, err) 295 err = b.Put(key2, orig2) 296 require.Nil(t, err) 297 err = b.Put(key3, orig3) 298 require.Nil(t, err) 299 }) 300 301 t.Run("replace some, keep one", func(t *testing.T) { 302 key1 := []byte("key-1") 303 key2 := []byte("key-2") 304 key3 := []byte("key-3") 305 orig1 := []byte("original value for key1") 306 replaced2 := []byte("updated value for key2") 307 replaced3 := []byte("updated value for key3") 308 309 err = b.Put(key2, replaced2) 310 require.Nil(t, err) 311 err = b.Put(key3, replaced3) 312 require.Nil(t, err) 313 314 res, err := b.Get(key1) 315 require.Nil(t, err) 316 assert.Equal(t, res, orig1) 317 res, err = b.Get(key2) 318 require.Nil(t, err) 319 assert.Equal(t, res, replaced2) 320 res, err = b.Get(key3) 321 require.Nil(t, err) 322 assert.Equal(t, res, replaced3) 323 }) 324 325 t.Run("orderly shutdown", func(t *testing.T) { 326 b.Shutdown(context.Background()) 327 }) 328 329 t.Run("init another bucket on the same files", func(t *testing.T) { 330 b2, err := NewBucket(ctx, dirName, "", nullLogger(), nil, 331 cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) 332 require.Nil(t, err) 333 334 key1 := []byte("key-1") 335 key2 := []byte("key-2") 336 key3 := []byte("key-3") 337 orig1 := []byte("original value for key1") 338 replaced2 := []byte("updated value for key2") 339 replaced3 := []byte("updated value for key3") 340 341 res, err := b2.Get(key1) 342 require.Nil(t, err) 343 assert.Equal(t, res, orig1) 344 res, err = b2.Get(key2) 345 require.Nil(t, err) 346 assert.Equal(t, res, replaced2) 347 res, err = b2.Get(key3) 348 require.Nil(t, err) 349 assert.Equal(t, res, replaced3) 350 351 // count objects over several segments after disk read 352 assert.Equal(t, 3, b2.Count()) 353 assert.Equal(t, 3, b2.CountAsync()) 354 }) 355 }) 356 } 357 358 func replaceInsertAndUpdate_WithSecondaryKeys(ctx context.Context, t *testing.T, opts []BucketOption) { 359 dirName := t.TempDir() 360 361 t.Run("memtable-only", func(t *testing.T) { 362 b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, 363 cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) 364 require.Nil(t, err) 365 366 // so big it effectively never triggers as part of this test 367 b.SetMemtableThreshold(1e9) 368 369 t.Run("set original values and verify", func(t *testing.T) { 370 key1 := []byte("key-1") 371 key2 := []byte("key-2") 372 key3 := []byte("key-3") 373 secondaryKey1 := []byte("secondary-key-1") 374 secondaryKey2 := []byte("secondary-key-2") 375 secondaryKey3 := []byte("secondary-key-3") 376 orig1 := []byte("original value for key1") 377 orig2 := []byte("original value for key2") 378 orig3 := []byte("original value for key3") 379 380 err = b.Put(key1, orig1, WithSecondaryKey(0, secondaryKey1)) 381 require.Nil(t, err) 382 err = b.Put(key2, orig2, WithSecondaryKey(0, secondaryKey2)) 383 require.Nil(t, err) 384 err = b.Put(key3, orig3, WithSecondaryKey(0, secondaryKey3)) 385 require.Nil(t, err) 386 387 res, err := b.GetBySecondary(0, secondaryKey1) 388 require.Nil(t, err) 389 assert.Equal(t, res, orig1) 390 res, err = b.GetBySecondary(0, secondaryKey2) 391 require.Nil(t, err) 392 assert.Equal(t, res, orig2) 393 res, err = b.GetBySecondary(0, secondaryKey3) 394 require.Nil(t, err) 395 assert.Equal(t, res, orig3) 396 }) 397 398 t.Run("replace some values, keep one - secondary keys not changed", func(t *testing.T) { 399 key2 := []byte("key-2") 400 key3 := []byte("key-3") 401 secondaryKey1 := []byte("secondary-key-1") 402 secondaryKey2 := []byte("secondary-key-2") 403 secondaryKey3 := []byte("secondary-key-3") 404 orig1 := []byte("original value for key1") 405 replaced2 := []byte("updated value for key2") 406 replaced3 := []byte("updated value for key3") 407 408 err = b.Put(key2, replaced2, WithSecondaryKey(0, secondaryKey2)) 409 require.Nil(t, err) 410 err = b.Put(key3, replaced3, WithSecondaryKey(0, secondaryKey3)) 411 require.Nil(t, err) 412 413 res, err := b.GetBySecondary(0, secondaryKey1) 414 require.Nil(t, err) 415 assert.Equal(t, res, orig1) 416 res, err = b.GetBySecondary(0, secondaryKey2) 417 require.Nil(t, err) 418 assert.Equal(t, res, replaced2) 419 res, err = b.GetBySecondary(0, secondaryKey3) 420 require.Nil(t, err) 421 assert.Equal(t, res, replaced3) 422 }) 423 424 t.Run("replace the secondary keys on an update", func(t *testing.T) { 425 key2 := []byte("key-2") 426 key3 := []byte("key-3") 427 secondaryKey1 := []byte("secondary-key-1") 428 secondaryKey2 := []byte("secondary-key-2-updated") 429 secondaryKey3 := []byte("secondary-key-3-updated") 430 orig1 := []byte("original value for key1") 431 replaced2 := []byte("twice updated value for key2") 432 replaced3 := []byte("twice updated value for key3") 433 434 err = b.Put(key2, replaced2, WithSecondaryKey(0, secondaryKey2)) 435 require.Nil(t, err) 436 err = b.Put(key3, replaced3, WithSecondaryKey(0, secondaryKey3)) 437 require.Nil(t, err) 438 439 // verify you can find by updated secondary keys 440 res, err := b.GetBySecondary(0, secondaryKey1) 441 require.Nil(t, err) 442 assert.Equal(t, res, orig1) 443 res, err = b.GetBySecondary(0, secondaryKey2) 444 require.Nil(t, err) 445 assert.Equal(t, res, replaced2) 446 res, err = b.GetBySecondary(0, secondaryKey3) 447 require.Nil(t, err) 448 assert.Equal(t, res, replaced3) 449 }) 450 }) 451 452 t.Run("with single flush in between updates", func(t *testing.T) { 453 b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, 454 cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) 455 require.Nil(t, err) 456 457 // so big it effectively never triggers as part of this test 458 b.SetMemtableThreshold(1e9) 459 460 t.Run("set original values", func(t *testing.T) { 461 key1 := []byte("key-1") 462 key2 := []byte("key-2") 463 key3 := []byte("key-3") 464 secondaryKey1 := []byte("secondary-key-1") 465 secondaryKey2 := []byte("secondary-key-2") 466 secondaryKey3 := []byte("secondary-key-3") 467 orig1 := []byte("original value for key1") 468 orig2 := []byte("original value for key2") 469 orig3 := []byte("original value for key3") 470 471 err = b.Put(key1, orig1, WithSecondaryKey(0, secondaryKey1)) 472 require.Nil(t, err) 473 err = b.Put(key2, orig2, WithSecondaryKey(0, secondaryKey2)) 474 require.Nil(t, err) 475 err = b.Put(key3, orig3, WithSecondaryKey(0, secondaryKey3)) 476 require.Nil(t, err) 477 }) 478 479 t.Run("flush memtable to disk", func(t *testing.T) { 480 require.Nil(t, b.FlushAndSwitch()) 481 }) 482 483 t.Run("replace the secondary keys on an update", func(t *testing.T) { 484 key2 := []byte("key-2") 485 key3 := []byte("key-3") 486 secondaryKey1 := []byte("secondary-key-1") 487 secondaryKey2 := []byte("secondary-key-2-updated") 488 secondaryKey3 := []byte("secondary-key-3-updated") 489 orig1 := []byte("original value for key1") 490 replaced2 := []byte("twice updated value for key2") 491 replaced3 := []byte("twice updated value for key3") 492 493 err = b.Put(key2, replaced2, WithSecondaryKey(0, secondaryKey2)) 494 require.Nil(t, err) 495 err = b.Put(key3, replaced3, WithSecondaryKey(0, secondaryKey3)) 496 require.Nil(t, err) 497 498 // verify you can find by updated secondary keys 499 res, err := b.GetBySecondary(0, secondaryKey1) 500 require.Nil(t, err) 501 assert.Equal(t, res, orig1) 502 res, err = b.GetBySecondary(0, secondaryKey2) 503 require.Nil(t, err) 504 assert.Equal(t, res, replaced2) 505 res, err = b.GetBySecondary(0, secondaryKey3) 506 require.Nil(t, err) 507 assert.Equal(t, res, replaced3) 508 }) 509 }) 510 511 t.Run("with a flush after initial write and update", func(t *testing.T) { 512 b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, 513 cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) 514 require.Nil(t, err) 515 516 // so big it effectively never triggers as part of this test 517 b.SetMemtableThreshold(1e9) 518 519 t.Run("set original values", func(t *testing.T) { 520 key1 := []byte("key-1") 521 key2 := []byte("key-2") 522 key3 := []byte("key-3") 523 secondaryKey1 := []byte("secondary-key-1") 524 secondaryKey2 := []byte("secondary-key-2") 525 secondaryKey3 := []byte("secondary-key-3") 526 orig1 := []byte("original value for key1") 527 orig2 := []byte("original value for key2") 528 orig3 := []byte("original value for key3") 529 530 err = b.Put(key1, orig1, WithSecondaryKey(0, secondaryKey1)) 531 require.Nil(t, err) 532 err = b.Put(key2, orig2, WithSecondaryKey(0, secondaryKey2)) 533 require.Nil(t, err) 534 err = b.Put(key3, orig3, WithSecondaryKey(0, secondaryKey3)) 535 require.Nil(t, err) 536 }) 537 538 t.Run("flush memtable to disk", func(t *testing.T) { 539 require.Nil(t, b.FlushAndSwitch()) 540 }) 541 542 t.Run("replace the secondary keys on an update", func(t *testing.T) { 543 key2 := []byte("key-2") 544 key3 := []byte("key-3") 545 secondaryKey2 := []byte("secondary-key-2-updated") 546 secondaryKey3 := []byte("secondary-key-3-updated") 547 replaced2 := []byte("twice updated value for key2") 548 replaced3 := []byte("twice updated value for key3") 549 550 err = b.Put(key2, replaced2, WithSecondaryKey(0, secondaryKey2)) 551 require.Nil(t, err) 552 err = b.Put(key3, replaced3, WithSecondaryKey(0, secondaryKey3)) 553 require.Nil(t, err) 554 }) 555 556 t.Run("flush memtable to disk", func(t *testing.T) { 557 require.Nil(t, b.FlushAndSwitch()) 558 }) 559 560 t.Run("verify again", func(t *testing.T) { 561 secondaryKey1 := []byte("secondary-key-1") 562 secondaryKey2 := []byte("secondary-key-2-updated") 563 secondaryKey3 := []byte("secondary-key-3-updated") 564 orig1 := []byte("original value for key1") 565 replaced2 := []byte("twice updated value for key2") 566 replaced3 := []byte("twice updated value for key3") 567 568 // verify you can find by updated secondary keys 569 res, err := b.GetBySecondary(0, secondaryKey1) 570 require.Nil(t, err) 571 assert.Equal(t, res, orig1) 572 res, err = b.GetBySecondary(0, secondaryKey2) 573 require.Nil(t, err) 574 assert.Equal(t, res, replaced2) 575 res, err = b.GetBySecondary(0, secondaryKey3) 576 require.Nil(t, err) 577 assert.Equal(t, res, replaced3) 578 }) 579 }) 580 581 t.Run("update in memtable then do an orderly shutdown and reinit", func(t *testing.T) { 582 b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, 583 cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) 584 require.Nil(t, err) 585 586 // so big it effectively never triggers as part of this test 587 b.SetMemtableThreshold(1e9) 588 589 t.Run("set original values", func(t *testing.T) { 590 key1 := []byte("key-1") 591 key2 := []byte("key-2") 592 key3 := []byte("key-3") 593 secondaryKey1 := []byte("secondary-key-1") 594 secondaryKey2 := []byte("secondary-key-2") 595 secondaryKey3 := []byte("secondary-key-3") 596 orig1 := []byte("original value for key1") 597 orig2 := []byte("original value for key2") 598 orig3 := []byte("original value for key3") 599 600 err = b.Put(key1, orig1, WithSecondaryKey(0, secondaryKey1)) 601 require.Nil(t, err) 602 err = b.Put(key2, orig2, WithSecondaryKey(0, secondaryKey2)) 603 require.Nil(t, err) 604 err = b.Put(key3, orig3, WithSecondaryKey(0, secondaryKey3)) 605 require.Nil(t, err) 606 }) 607 608 t.Run("replace the secondary keys on an update", func(t *testing.T) { 609 key2 := []byte("key-2") 610 key3 := []byte("key-3") 611 secondaryKey2 := []byte("secondary-key-2-updated") 612 secondaryKey3 := []byte("secondary-key-3-updated") 613 replaced2 := []byte("twice updated value for key2") 614 replaced3 := []byte("twice updated value for key3") 615 616 err = b.Put(key2, replaced2, WithSecondaryKey(0, secondaryKey2)) 617 require.Nil(t, err) 618 err = b.Put(key3, replaced3, WithSecondaryKey(0, secondaryKey3)) 619 require.Nil(t, err) 620 }) 621 622 t.Run("flush memtable to disk", func(t *testing.T) { 623 require.Nil(t, b.Shutdown(context.Background())) 624 }) 625 626 t.Run("init a new one and verify", func(t *testing.T) { 627 b2, err := NewBucket(ctx, dirName, "", nullLogger(), nil, 628 cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) 629 require.Nil(t, err) 630 631 secondaryKey1 := []byte("secondary-key-1") 632 secondaryKey2 := []byte("secondary-key-2-updated") 633 secondaryKey3 := []byte("secondary-key-3-updated") 634 orig1 := []byte("original value for key1") 635 replaced2 := []byte("twice updated value for key2") 636 replaced3 := []byte("twice updated value for key3") 637 638 // verify you can find by updated secondary keys 639 res, err := b2.GetBySecondary(0, secondaryKey1) 640 require.Nil(t, err) 641 assert.Equal(t, res, orig1) 642 res, err = b2.GetBySecondary(0, secondaryKey2) 643 require.Nil(t, err) 644 assert.Equal(t, res, replaced2) 645 res, err = b2.GetBySecondary(0, secondaryKey3) 646 require.Nil(t, err) 647 assert.Equal(t, res, replaced3) 648 }) 649 }) 650 } 651 652 func replaceInsertAndDelete(ctx context.Context, t *testing.T, opts []BucketOption) { 653 dirName := t.TempDir() 654 655 t.Run("memtable-only", func(t *testing.T) { 656 b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, 657 cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) 658 require.Nil(t, err) 659 660 // so big it effectively never triggers as part of this test 661 b.SetMemtableThreshold(1e9) 662 663 t.Run("set original values", func(t *testing.T) { 664 key1 := []byte("key-1") 665 key2 := []byte("key-2") 666 key3 := []byte("key-3") 667 orig1 := []byte("original value for key1") 668 orig2 := []byte("original value for key2") 669 orig3 := []byte("original value for key3") 670 671 err = b.Put(key1, orig1) 672 require.Nil(t, err) 673 err = b.Put(key2, orig2) 674 require.Nil(t, err) 675 err = b.Put(key3, orig3) 676 require.Nil(t, err) 677 }) 678 679 t.Run("delete some, keep one", func(t *testing.T) { 680 key1 := []byte("key-1") 681 key2 := []byte("key-2") 682 key3 := []byte("key-3") 683 orig1 := []byte("original value for key1") 684 685 err = b.Delete(key2) 686 require.Nil(t, err) 687 err = b.Delete(key3) 688 require.Nil(t, err) 689 690 res, err := b.Get(key1) 691 require.Nil(t, err) 692 assert.Equal(t, res, orig1) 693 res, err = b.Get(key2) 694 require.Nil(t, err) 695 assert.Nil(t, res) 696 res, err = b.Get(key3) 697 require.Nil(t, err) 698 assert.Nil(t, res) 699 }) 700 701 t.Run("count objects", func(t *testing.T) { 702 assert.Equal(t, 1, b.Count()) 703 // all happenin in the memtable so far, async does not know of any 704 // objects yet 705 assert.Equal(t, 0, b.CountAsync()) 706 }) 707 }) 708 709 t.Run("with single flush in between updates", func(t *testing.T) { 710 b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, 711 cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) 712 require.Nil(t, err) 713 714 // so big it effectively never triggers as part of this test 715 b.SetMemtableThreshold(1e9) 716 717 t.Run("set original values", func(t *testing.T) { 718 key1 := []byte("key-1") 719 key2 := []byte("key-2") 720 key3 := []byte("key-3") 721 orig1 := []byte("original value for key1") 722 orig2 := []byte("original value for key2") 723 orig3 := []byte("original value for key3") 724 725 err = b.Put(key1, orig1) 726 require.Nil(t, err) 727 err = b.Put(key2, orig2) 728 require.Nil(t, err) 729 err = b.Put(key3, orig3) 730 require.Nil(t, err) 731 }) 732 733 t.Run("flush to disk", func(t *testing.T) { 734 require.Nil(t, b.FlushAndSwitch()) 735 }) 736 737 t.Run("delete some, keep one", func(t *testing.T) { 738 key1 := []byte("key-1") 739 key2 := []byte("key-2") 740 key3 := []byte("key-3") 741 orig1 := []byte("original value for key1") 742 743 err = b.Delete(key2) 744 require.Nil(t, err) 745 err = b.Delete(key3) 746 require.Nil(t, err) 747 748 res, err := b.Get(key1) 749 require.Nil(t, err) 750 assert.Equal(t, res, orig1) 751 res, err = b.Get(key2) 752 require.Nil(t, err) 753 assert.Nil(t, res) 754 res, err = b.Get(key3) 755 require.Nil(t, err) 756 assert.Nil(t, res) 757 }) 758 759 t.Run("count objects", func(t *testing.T) { 760 assert.Equal(t, 1, b.Count()) 761 // async still looks at the objects in the segment, ignores deletes in 762 // the memtable 763 assert.Equal(t, 3, b.CountAsync()) 764 }) 765 }) 766 767 t.Run("with flushes after initial write and delete", func(t *testing.T) { 768 b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, 769 cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) 770 require.Nil(t, err) 771 772 // so big it effectively never triggers as part of this test 773 b.SetMemtableThreshold(1e9) 774 775 t.Run("set original values", func(t *testing.T) { 776 key1 := []byte("key-1") 777 key2 := []byte("key-2") 778 key3 := []byte("key-3") 779 orig1 := []byte("original value for key1") 780 orig2 := []byte("original value for key2") 781 orig3 := []byte("original value for key3") 782 783 err = b.Put(key1, orig1) 784 require.Nil(t, err) 785 err = b.Put(key2, orig2) 786 require.Nil(t, err) 787 err = b.Put(key3, orig3) 788 require.Nil(t, err) 789 }) 790 791 t.Run("flush to disk", func(t *testing.T) { 792 require.Nil(t, b.FlushAndSwitch()) 793 }) 794 795 t.Run("delete some, keep one", func(t *testing.T) { 796 key1 := []byte("key-1") 797 key2 := []byte("key-2") 798 key3 := []byte("key-3") 799 orig1 := []byte("original value for key1") 800 801 err = b.Delete(key2) 802 require.Nil(t, err) 803 err = b.Delete(key3) 804 require.Nil(t, err) 805 806 // Flush again! 807 require.Nil(t, b.FlushAndSwitch()) 808 809 res, err := b.Get(key1) 810 require.Nil(t, err) 811 assert.Equal(t, res, orig1) 812 res, err = b.Get(key2) 813 require.Nil(t, err) 814 assert.Nil(t, res) 815 res, err = b.Get(key3) 816 require.Nil(t, err) 817 assert.Nil(t, res) 818 }) 819 820 t.Run("count objects", func(t *testing.T) { 821 assert.Equal(t, 1, b.Count()) 822 assert.Equal(t, 1, b.CountAsync()) 823 }) 824 }) 825 } 826 827 func replaceCursors(ctx context.Context, t *testing.T, opts []BucketOption) { 828 t.Run("memtable-only", func(t *testing.T) { 829 r := getRandomSeed() 830 dirName := t.TempDir() 831 832 b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, 833 cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) 834 require.Nil(t, err) 835 836 // so big it effectively never triggers as part of this test 837 b.SetMemtableThreshold(1e9) 838 839 t.Run("set original values", func(t *testing.T) { 840 pairs := 20 841 keys := make([][]byte, pairs) 842 values := make([][]byte, pairs) 843 844 for i := range keys { 845 keys[i] = []byte(fmt.Sprintf("key-%03d", i)) 846 values[i] = []byte(fmt.Sprintf("value-%03d", i)) 847 } 848 849 // shuffle to make sure the BST isn't accidentally in order 850 r.Shuffle(len(keys), func(i, j int) { 851 keys[i], keys[j] = keys[j], keys[i] 852 values[i], values[j] = values[j], values[i] 853 }) 854 855 for i := range keys { 856 err = b.Put(keys[i], values[i]) 857 require.Nil(t, err) 858 } 859 }) 860 861 t.Run("seek from somewhere in the middle", func(t *testing.T) { 862 expectedKeys := [][]byte{ 863 []byte("key-016"), 864 []byte("key-017"), 865 []byte("key-018"), 866 []byte("key-019"), 867 } 868 expectedValues := [][]byte{ 869 []byte("value-016"), 870 []byte("value-017"), 871 []byte("value-018"), 872 []byte("value-019"), 873 } 874 875 var retrievedKeys [][]byte 876 var retrievedValues [][]byte 877 c := b.Cursor() 878 defer c.Close() 879 for k, v := c.Seek([]byte("key-016")); k != nil; k, v = c.Next() { 880 retrievedKeys = copyAndAppend(retrievedKeys, k) 881 retrievedValues = copyAndAppend(retrievedValues, v) 882 } 883 884 assert.Equal(t, expectedKeys, retrievedKeys) 885 assert.Equal(t, expectedValues, retrievedValues) 886 }) 887 888 t.Run("start from the beginning", func(t *testing.T) { 889 expectedKeys := [][]byte{ 890 []byte("key-000"), 891 []byte("key-001"), 892 []byte("key-002"), 893 } 894 expectedValues := [][]byte{ 895 []byte("value-000"), 896 []byte("value-001"), 897 []byte("value-002"), 898 } 899 900 var retrievedKeys [][]byte 901 var retrievedValues [][]byte 902 c := b.Cursor() 903 defer c.Close() 904 retrieved := 0 905 for k, v := c.First(); k != nil && retrieved < 3; k, v = c.Next() { 906 retrieved++ 907 retrievedKeys = copyAndAppend(retrievedKeys, k) 908 retrievedValues = copyAndAppend(retrievedValues, v) 909 } 910 911 assert.Equal(t, expectedKeys, retrievedKeys) 912 assert.Equal(t, expectedValues, retrievedValues) 913 }) 914 915 t.Run("replace a key", func(t *testing.T) { 916 key := []byte("key-002") 917 value := []byte("value-002-updated") 918 919 err = b.Put(key, value) 920 require.Nil(t, err) 921 922 expectedKeys := [][]byte{ 923 []byte("key-001"), 924 []byte("key-002"), 925 } 926 expectedValues := [][]byte{ 927 []byte("value-001"), 928 []byte("value-002-updated"), 929 } 930 931 var retrievedKeys [][]byte 932 var retrievedValues [][]byte 933 c := b.Cursor() 934 defer c.Close() 935 retrieved := 0 936 for k, v := c.Seek([]byte("key-001")); k != nil && retrieved < 2; k, v = c.Next() { 937 retrieved++ 938 retrievedKeys = copyAndAppend(retrievedKeys, k) 939 retrievedValues = copyAndAppend(retrievedValues, v) 940 } 941 942 assert.Equal(t, expectedKeys, retrievedKeys) 943 assert.Equal(t, expectedValues, retrievedValues) 944 }) 945 946 t.Run("delete a key", func(t *testing.T) { 947 key := []byte("key-002") 948 949 err = b.Delete(key) 950 require.Nil(t, err) 951 952 t.Run("seek to a specific key", func(t *testing.T) { 953 expectedKeys := [][]byte{ 954 []byte("key-001"), 955 []byte("key-003"), 956 } 957 expectedValues := [][]byte{ 958 []byte("value-001"), 959 []byte("value-003"), 960 } 961 var retrievedKeys [][]byte 962 var retrievedValues [][]byte 963 c := b.Cursor() 964 defer c.Close() 965 retrieved := 0 966 for k, v := c.Seek([]byte("key-001")); k != nil && retrieved < 2; k, v = c.Next() { 967 retrieved++ 968 retrievedKeys = copyAndAppend(retrievedKeys, k) 969 retrievedValues = copyAndAppend(retrievedValues, v) 970 } 971 972 assert.Equal(t, expectedKeys, retrievedKeys) 973 assert.Equal(t, expectedValues, retrievedValues) 974 }) 975 976 t.Run("seek to first key", func(t *testing.T) { 977 expectedKeys := [][]byte{ 978 []byte("key-000"), 979 []byte("key-001"), 980 []byte("key-003"), 981 } 982 expectedValues := [][]byte{ 983 []byte("value-000"), 984 []byte("value-001"), 985 []byte("value-003"), 986 } 987 988 var retrievedKeys [][]byte 989 var retrievedValues [][]byte 990 c := b.Cursor() 991 defer c.Close() 992 retrieved := 0 993 for k, v := c.First(); k != nil && retrieved < 3; k, v = c.Next() { 994 retrieved++ 995 retrievedKeys = copyAndAppend(retrievedKeys, k) 996 retrievedValues = copyAndAppend(retrievedValues, v) 997 } 998 999 assert.Equal(t, expectedKeys, retrievedKeys) 1000 assert.Equal(t, expectedValues, retrievedValues) 1001 }) 1002 }) 1003 1004 t.Run("delete the first key", func(t *testing.T) { 1005 key := []byte("key-000") 1006 1007 err = b.Delete(key) 1008 require.Nil(t, err) 1009 1010 t.Run("seek to a specific key", func(t *testing.T) { 1011 expectedKeys := [][]byte{ 1012 []byte("key-001"), 1013 []byte("key-003"), 1014 } 1015 expectedValues := [][]byte{ 1016 []byte("value-001"), 1017 []byte("value-003"), 1018 } 1019 var retrievedKeys [][]byte 1020 var retrievedValues [][]byte 1021 c := b.Cursor() 1022 defer c.Close() 1023 retrieved := 0 1024 for k, v := c.Seek([]byte("key-000")); k != nil && retrieved < 2; k, v = c.Next() { 1025 retrieved++ 1026 retrievedKeys = copyAndAppend(retrievedKeys, k) 1027 retrievedValues = copyAndAppend(retrievedValues, v) 1028 } 1029 1030 assert.Equal(t, expectedKeys, retrievedKeys) 1031 assert.Equal(t, expectedValues, retrievedValues) 1032 }) 1033 1034 t.Run("seek to first key", func(t *testing.T) { 1035 expectedKeys := [][]byte{ 1036 []byte("key-001"), 1037 []byte("key-003"), 1038 } 1039 expectedValues := [][]byte{ 1040 []byte("value-001"), 1041 []byte("value-003"), 1042 } 1043 1044 var retrievedKeys [][]byte 1045 var retrievedValues [][]byte 1046 c := b.Cursor() 1047 defer c.Close() 1048 retrieved := 0 1049 for k, v := c.First(); k != nil && retrieved < 2; k, v = c.Next() { 1050 retrieved++ 1051 retrievedKeys = copyAndAppend(retrievedKeys, k) 1052 retrievedValues = copyAndAppend(retrievedValues, v) 1053 } 1054 1055 assert.Equal(t, expectedKeys, retrievedKeys) 1056 assert.Equal(t, expectedValues, retrievedValues) 1057 }) 1058 }) 1059 }) 1060 1061 t.Run("with a single flush", func(t *testing.T) { 1062 r := getRandomSeed() 1063 dirName := t.TempDir() 1064 1065 b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, 1066 cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) 1067 require.Nil(t, err) 1068 1069 // so big it effectively never triggers as part of this test 1070 b.SetMemtableThreshold(1e9) 1071 1072 t.Run("set original values", func(t *testing.T) { 1073 pairs := 20 1074 keys := make([][]byte, pairs) 1075 values := make([][]byte, pairs) 1076 1077 for i := range keys { 1078 keys[i] = []byte(fmt.Sprintf("key-%03d", i)) 1079 values[i] = []byte(fmt.Sprintf("value-%03d", i)) 1080 } 1081 1082 // shuffle to make sure the BST isn't accidentally in order 1083 r.Shuffle(len(keys), func(i, j int) { 1084 keys[i], keys[j] = keys[j], keys[i] 1085 values[i], values[j] = values[j], values[i] 1086 }) 1087 1088 for i := range keys { 1089 err = b.Put(keys[i], values[i]) 1090 require.Nil(t, err) 1091 } 1092 }) 1093 1094 t.Run("flush to disk", func(t *testing.T) { 1095 require.Nil(t, b.FlushAndSwitch()) 1096 }) 1097 1098 t.Run("seek from somewhere in the middle", func(t *testing.T) { 1099 expectedKeys := [][]byte{ 1100 []byte("key-016"), 1101 []byte("key-017"), 1102 []byte("key-018"), 1103 []byte("key-019"), 1104 } 1105 expectedValues := [][]byte{ 1106 []byte("value-016"), 1107 []byte("value-017"), 1108 []byte("value-018"), 1109 []byte("value-019"), 1110 } 1111 1112 var retrievedKeys [][]byte 1113 var retrievedValues [][]byte 1114 c := b.Cursor() 1115 defer c.Close() 1116 for k, v := c.Seek([]byte("key-016")); k != nil; k, v = c.Next() { 1117 retrievedKeys = copyAndAppend(retrievedKeys, k) 1118 retrievedValues = copyAndAppend(retrievedValues, v) 1119 } 1120 1121 assert.Equal(t, expectedKeys, retrievedKeys) 1122 assert.Equal(t, expectedValues, retrievedValues) 1123 }) 1124 1125 t.Run("start from the beginning", func(t *testing.T) { 1126 expectedKeys := [][]byte{ 1127 []byte("key-000"), 1128 []byte("key-001"), 1129 []byte("key-002"), 1130 } 1131 expectedValues := [][]byte{ 1132 []byte("value-000"), 1133 []byte("value-001"), 1134 []byte("value-002"), 1135 } 1136 1137 var retrievedKeys [][]byte 1138 var retrievedValues [][]byte 1139 c := b.Cursor() 1140 defer c.Close() 1141 retrieved := 0 1142 for k, v := c.First(); k != nil && retrieved < 3; k, v = c.Next() { 1143 retrieved++ 1144 retrievedKeys = copyAndAppend(retrievedKeys, k) 1145 retrievedValues = copyAndAppend(retrievedValues, v) 1146 } 1147 1148 assert.Equal(t, expectedKeys, retrievedKeys) 1149 assert.Equal(t, expectedValues, retrievedValues) 1150 }) 1151 }) 1152 1153 t.Run("mixing several disk segments and memtable - with updates", func(t *testing.T) { 1154 r := getRandomSeed() 1155 dirName := t.TempDir() 1156 1157 b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, 1158 cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) 1159 require.Nil(t, err) 1160 1161 // so big it effectively never triggers as part of this test 1162 b.SetMemtableThreshold(1e9) 1163 1164 t.Run("first third (%3==0)", func(t *testing.T) { 1165 pairs := 20 1166 var keys [][]byte 1167 var values [][]byte 1168 1169 for i := 0; i < pairs; i++ { 1170 if i%3 == 0 { 1171 keys = copyAndAppend(keys, []byte(fmt.Sprintf("key-%03d", i))) 1172 values = copyAndAppend(values, []byte(fmt.Sprintf("value-%03d", i))) 1173 } 1174 } 1175 1176 // shuffle to make sure the BST isn't accidentally in order 1177 r.Shuffle(len(keys), func(i, j int) { 1178 keys[i], keys[j] = keys[j], keys[i] 1179 values[i], values[j] = values[j], values[i] 1180 }) 1181 1182 for i := range keys { 1183 err = b.Put(keys[i], values[i]) 1184 require.Nil(t, err) 1185 } 1186 }) 1187 1188 t.Run("flush to disk", func(t *testing.T) { 1189 require.Nil(t, b.FlushAndSwitch()) 1190 }) 1191 1192 t.Run("second third (%3==1)", func(t *testing.T) { 1193 pairs := 20 1194 var keys [][]byte 1195 var values [][]byte 1196 1197 for i := 0; i < pairs; i++ { 1198 if i%3 == 1 { 1199 keys = copyAndAppend(keys, []byte(fmt.Sprintf("key-%03d", i))) 1200 values = copyAndAppend(values, []byte(fmt.Sprintf("value-%03d", i))) 1201 } 1202 } 1203 1204 // shuffle to make sure the BST isn't accidentally in order 1205 r.Shuffle(len(keys), func(i, j int) { 1206 keys[i], keys[j] = keys[j], keys[i] 1207 values[i], values[j] = values[j], values[i] 1208 }) 1209 1210 for i := range keys { 1211 err = b.Put(keys[i], values[i]) 1212 require.Nil(t, err) 1213 } 1214 }) 1215 1216 t.Run("update something that was already written in segment 1", func(t *testing.T) { 1217 require.Nil(t, b.Put([]byte("key-000"), []byte("updated-value-000"))) 1218 require.Nil(t, b.Delete([]byte("key-003"))) 1219 }) 1220 1221 t.Run("flush to disk", func(t *testing.T) { 1222 require.Nil(t, b.FlushAndSwitch()) 1223 }) 1224 1225 t.Run("third third (%3==2) memtable only", func(t *testing.T) { 1226 pairs := 20 1227 var keys [][]byte 1228 var values [][]byte 1229 1230 for i := 0; i < pairs; i++ { 1231 if i%3 == 2 { 1232 keys = copyAndAppend(keys, []byte(fmt.Sprintf("key-%03d", i))) 1233 values = copyAndAppend(values, []byte(fmt.Sprintf("value-%03d", i))) 1234 } 1235 } 1236 1237 // shuffle to make sure the BST isn't accidentally in order 1238 r.Shuffle(len(keys), func(i, j int) { 1239 keys[i], keys[j] = keys[j], keys[i] 1240 values[i], values[j] = values[j], values[i] 1241 }) 1242 1243 for i := range keys { 1244 err = b.Put(keys[i], values[i]) 1245 require.Nil(t, err) 1246 } 1247 1248 // no flush for this one, so this segment stays in the memtable 1249 }) 1250 1251 t.Run("update something that was already written previously", func(t *testing.T) { 1252 require.Nil(t, b.Put([]byte("key-000"), []byte("twice-updated-value-000"))) 1253 require.Nil(t, b.Put([]byte("key-001"), []byte("once-updated-value-001"))) 1254 require.Nil(t, b.Put([]byte("key-019"), []byte("once-updated-value-019"))) 1255 require.Nil(t, b.Delete([]byte("key-018"))) 1256 }) 1257 1258 t.Run("seek from somewhere in the middle", func(t *testing.T) { 1259 expectedKeys := [][]byte{ 1260 []byte("key-016"), 1261 []byte("key-017"), 1262 // key-018 deleted 1263 []byte("key-019"), 1264 } 1265 expectedValues := [][]byte{ 1266 []byte("value-016"), 1267 []byte("value-017"), 1268 []byte("once-updated-value-019"), 1269 } 1270 1271 var retrievedKeys [][]byte 1272 var retrievedValues [][]byte 1273 c := b.Cursor() 1274 defer c.Close() 1275 for k, v := c.Seek([]byte("key-016")); k != nil; k, v = c.Next() { 1276 retrievedKeys = copyAndAppend(retrievedKeys, k) 1277 retrievedValues = copyAndAppend(retrievedValues, v) 1278 } 1279 1280 assert.Equal(t, expectedKeys, retrievedKeys) 1281 assert.Equal(t, expectedValues, retrievedValues) 1282 }) 1283 1284 t.Run("start from the beginning", func(t *testing.T) { 1285 expectedKeys := [][]byte{ 1286 []byte("key-000"), 1287 []byte("key-001"), 1288 []byte("key-002"), 1289 // key-003 was deleted 1290 []byte("key-004"), 1291 } 1292 expectedValues := [][]byte{ 1293 []byte("twice-updated-value-000"), 1294 []byte("once-updated-value-001"), 1295 []byte("value-002"), 1296 []byte("value-004"), 1297 } 1298 1299 var retrievedKeys [][]byte 1300 var retrievedValues [][]byte 1301 c := b.Cursor() 1302 defer c.Close() 1303 retrieved := 0 1304 for k, v := c.First(); k != nil && retrieved < 4; k, v = c.Next() { 1305 retrieved++ 1306 retrievedKeys = copyAndAppend(retrievedKeys, k) 1307 retrievedValues = copyAndAppend(retrievedValues, v) 1308 } 1309 1310 assert.Equal(t, expectedKeys, retrievedKeys) 1311 assert.Equal(t, expectedValues, retrievedValues) 1312 }) 1313 1314 t.Run("re-add the deleted keys", func(t *testing.T) { 1315 require.Nil(t, b.Put([]byte("key-003"), []byte("readded-003"))) 1316 require.Nil(t, b.Put([]byte("key-018"), []byte("readded-018"))) 1317 // tombstones are now only in memtable 1318 }) 1319 1320 t.Run("seek from somewhere in the middle", func(t *testing.T) { 1321 expectedKeys := [][]byte{ 1322 []byte("key-016"), 1323 []byte("key-017"), 1324 []byte("key-018"), 1325 []byte("key-019"), 1326 } 1327 expectedValues := [][]byte{ 1328 []byte("value-016"), 1329 []byte("value-017"), 1330 []byte("readded-018"), 1331 []byte("once-updated-value-019"), 1332 } 1333 1334 var retrievedKeys [][]byte 1335 var retrievedValues [][]byte 1336 c := b.Cursor() 1337 defer c.Close() 1338 for k, v := c.Seek([]byte("key-016")); k != nil; k, v = c.Next() { 1339 retrievedKeys = copyAndAppend(retrievedKeys, k) 1340 retrievedValues = copyAndAppend(retrievedValues, v) 1341 } 1342 1343 assert.Equal(t, expectedKeys, retrievedKeys) 1344 assert.Equal(t, expectedValues, retrievedValues) 1345 }) 1346 1347 t.Run("start from the beginning", func(t *testing.T) { 1348 expectedKeys := [][]byte{ 1349 []byte("key-000"), 1350 []byte("key-001"), 1351 []byte("key-002"), 1352 []byte("key-003"), 1353 } 1354 expectedValues := [][]byte{ 1355 []byte("twice-updated-value-000"), 1356 []byte("once-updated-value-001"), 1357 []byte("value-002"), 1358 []byte("readded-003"), 1359 } 1360 1361 var retrievedKeys [][]byte 1362 var retrievedValues [][]byte 1363 c := b.Cursor() 1364 defer c.Close() 1365 retrieved := 0 1366 for k, v := c.First(); k != nil && retrieved < 4; k, v = c.Next() { 1367 retrieved++ 1368 retrievedKeys = copyAndAppend(retrievedKeys, k) 1369 retrievedValues = copyAndAppend(retrievedValues, v) 1370 } 1371 1372 assert.Equal(t, expectedKeys, retrievedKeys) 1373 assert.Equal(t, expectedValues, retrievedValues) 1374 }) 1375 1376 t.Run("perform a final flush to disk", func(t *testing.T) { 1377 require.Nil(t, b.FlushAndSwitch()) 1378 }) 1379 1380 t.Run("seek from somewhere in the middle", func(t *testing.T) { 1381 expectedKeys := [][]byte{ 1382 []byte("key-016"), 1383 []byte("key-017"), 1384 []byte("key-018"), 1385 []byte("key-019"), 1386 } 1387 expectedValues := [][]byte{ 1388 []byte("value-016"), 1389 []byte("value-017"), 1390 []byte("readded-018"), 1391 []byte("once-updated-value-019"), 1392 } 1393 1394 var retrievedKeys [][]byte 1395 var retrievedValues [][]byte 1396 c := b.Cursor() 1397 defer c.Close() 1398 for k, v := c.Seek([]byte("key-016")); k != nil; k, v = c.Next() { 1399 retrievedKeys = copyAndAppend(retrievedKeys, k) 1400 retrievedValues = copyAndAppend(retrievedValues, v) 1401 } 1402 1403 assert.Equal(t, expectedKeys, retrievedKeys) 1404 assert.Equal(t, expectedValues, retrievedValues) 1405 }) 1406 1407 t.Run("start from the beginning", func(t *testing.T) { 1408 expectedKeys := [][]byte{ 1409 []byte("key-000"), 1410 []byte("key-001"), 1411 []byte("key-002"), 1412 []byte("key-003"), 1413 } 1414 expectedValues := [][]byte{ 1415 []byte("twice-updated-value-000"), 1416 []byte("once-updated-value-001"), 1417 []byte("value-002"), 1418 []byte("readded-003"), 1419 } 1420 1421 var retrievedKeys [][]byte 1422 var retrievedValues [][]byte 1423 c := b.Cursor() 1424 defer c.Close() 1425 retrieved := 0 1426 for k, v := c.First(); k != nil && retrieved < 4; k, v = c.Next() { 1427 retrieved++ 1428 retrievedKeys = copyAndAppend(retrievedKeys, k) 1429 retrievedValues = copyAndAppend(retrievedValues, v) 1430 } 1431 1432 assert.Equal(t, expectedKeys, retrievedKeys) 1433 assert.Equal(t, expectedValues, retrievedValues) 1434 }) 1435 }) 1436 1437 // This test is inspired by unusual behavior encountered as part of the 1438 // evaluation of gh-1569 where a delete could sometimes lead to no data after 1439 // a restart which was caused by the disk segment cursor's .first() method 1440 // not returning the correct key. Thus we'd have a null-key with a tombstone 1441 // which would override whatever is the real "first" key, since null is 1442 // always smaller 1443 t.Run("with deletes as latest in some segments", func(t *testing.T) { 1444 dirName := t.TempDir() 1445 1446 b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, 1447 cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) 1448 require.Nil(t, err) 1449 1450 // so big it effectively never triggers as part of this test 1451 b.SetMemtableThreshold(1e9) 1452 1453 t.Run("add new datapoint", func(t *testing.T) { 1454 err := b.Put([]byte("key-1"), []byte("value-1")) 1455 require.Nil(t, err) 1456 }) 1457 1458 t.Run("add datapoint and flush", func(t *testing.T) { 1459 err := b.Put([]byte("key-8"), []byte("value-8")) 1460 require.Nil(t, err) 1461 1462 require.Nil(t, b.FlushAndSwitch()) 1463 }) 1464 1465 t.Run("delete datapoint and flush", func(t *testing.T) { 1466 err := b.Delete([]byte("key-8")) 1467 // note that we are deleting the key with the 'higher' key, so a missing 1468 // key on the delete would definitely be mismatched. If we had instead 1469 // the deleted the first key, the incorrect tombstone would have been 1470 // correct by coincidence 1471 require.Nil(t, err) 1472 1473 require.Nil(t, b.FlushAndSwitch()) 1474 }) 1475 1476 t.Run("verify", func(t *testing.T) { 1477 expectedKeys := [][]byte{ 1478 []byte("key-1"), 1479 } 1480 expectedValues := [][]byte{ 1481 []byte("value-1"), 1482 } 1483 1484 var retrievedKeys [][]byte 1485 var retrievedValues [][]byte 1486 c := b.Cursor() 1487 defer c.Close() 1488 retrieved := 0 1489 for k, v := c.First(); k != nil && retrieved < 4; k, v = c.Next() { 1490 retrieved++ 1491 retrievedKeys = copyAndAppend(retrievedKeys, k) 1492 retrievedValues = copyAndAppend(retrievedValues, v) 1493 } 1494 1495 assert.Equal(t, expectedKeys, retrievedKeys) 1496 assert.Equal(t, expectedValues, retrievedValues) 1497 }) 1498 }) 1499 } 1500 1501 func copyAndAppend(list [][]byte, elem []byte) [][]byte { 1502 elemCopy := make([]byte, len(elem)) 1503 copy(elemCopy, elem) 1504 return append(list, elemCopy) 1505 }