github.com/cockroachdb/pebble@v1.1.1-0.20240513155919-3622ade60459/testdata/manual_compaction_set_with_del_sstable_Pebblev4 (about) 1 batch 2 set a 1 3 set b 2 4 ---- 5 6 compact a-b 7 ---- 8 6: 9 000005:[a#10,SET-b#11,SET] 10 11 batch 12 set c 3 13 set d 4 14 ---- 15 16 compact c-d 17 ---- 18 6: 19 000005:[a#10,SET-b#11,SET] 20 000007:[c#12,SET-d#13,SET] 21 22 batch 23 set b 5 24 set c 6 25 ---- 26 27 compact a-d 28 ---- 29 6: 30 000010:[a#0,SET-d#0,SET] 31 32 # This also tests flushing a memtable that only contains range 33 # deletions. 34 35 batch 36 del-range a e 37 ---- 38 39 compact a-d 40 ---- 41 42 # Test that a multi-output-file compaction generates non-overlapping files. 43 44 define target-file-sizes=(100, 1) 45 L0 46 b.SET.1:v 47 L0 48 a.SET.2:v 49 ---- 50 0.0: 51 000005:[a#2,SET-a#2,SET] 52 000004:[b#1,SET-b#1,SET] 53 54 compact a-b 55 ---- 56 1: 57 000006:[a#0,SET-a#0,SET] 58 000007:[b#0,SET-b#0,SET] 59 60 # A range tombstone extends past the grandparent file boundary used to limit the 61 # size of future compactions. Verify the range tombstone is split at that file 62 # boundary. 63 64 define target-file-sizes=(1, 1, 1, 1) 65 L1 66 a.SET.3:v 67 L2 68 a.RANGEDEL.2:e 69 L3 70 a.SET.0:v 71 b.SET.0:v 72 L3 73 c.SET.0:v 74 d.SET.0:v 75 ---- 76 1: 77 000004:[a#3,SET-a#3,SET] 78 2: 79 000005:[a#2,RANGEDEL-e#inf,RANGEDEL] 80 3: 81 000006:[a#0,SET-b#0,SET] 82 000007:[c#0,SET-d#0,SET] 83 84 wait-pending-table-stats 85 000005 86 ---- 87 num-entries: 1 88 num-deletions: 1 89 num-range-key-sets: 0 90 point-deletions-bytes-estimate: 0 91 range-deletions-bytes-estimate: 1334 92 93 compact a-e L1 94 ---- 95 2: 96 000008:[a#3,SETWITHDEL-c#inf,RANGEDEL] 97 000009:[c#2,RANGEDEL-e#inf,RANGEDEL] 98 3: 99 000006:[a#0,SET-b#0,SET] 100 000007:[c#0,SET-d#0,SET] 101 102 wait-pending-table-stats 103 000008 104 ---- 105 num-entries: 2 106 num-deletions: 1 107 num-range-key-sets: 0 108 point-deletions-bytes-estimate: 0 109 range-deletions-bytes-estimate: 667 110 111 # Same as above, except range tombstone covers multiple grandparent file boundaries. 112 113 define target-file-sizes=(1, 1, 1, 1) 114 L1 115 a.SET.3:v 116 L2 117 a.RANGEDEL.2:g 118 L3 119 a.SET.0:v 120 b.SET.0:v 121 L3 122 c.SET.0:v 123 d.SET.0:v 124 L3 125 e.SET.0:v 126 f.SET.1:v 127 L3 128 g.SET.1:v 129 g.SET.0:v 130 ---- 131 1: 132 000004:[a#3,SET-a#3,SET] 133 2: 134 000005:[a#2,RANGEDEL-g#inf,RANGEDEL] 135 3: 136 000006:[a#0,SET-b#0,SET] 137 000007:[c#0,SET-d#0,SET] 138 000008:[e#0,SET-f#1,SET] 139 000009:[g#1,SET-g#1,SET] 140 141 compact a-e L1 142 ---- 143 2: 144 000010:[a#3,SETWITHDEL-c#inf,RANGEDEL] 145 000011:[c#2,RANGEDEL-e#inf,RANGEDEL] 146 000012:[e#2,RANGEDEL-g#inf,RANGEDEL] 147 3: 148 000006:[a#0,SET-b#0,SET] 149 000007:[c#0,SET-d#0,SET] 150 000008:[e#0,SET-f#1,SET] 151 000009:[g#1,SET-g#1,SET] 152 153 # A range tombstone covers multiple grandparent file boundaries between point keys, 154 # rather than after all point keys. 155 156 define target-file-sizes=(1, 1, 1, 1) 157 L1 158 a.SET.3:v 159 h.SET.3:v 160 L2 161 a.RANGEDEL.2:g 162 L3 163 a.SET.0:v 164 b.SET.0:v 165 L3 166 c.SET.0:v 167 d.SET.0:v 168 L3 169 e.SET.0:v 170 f.SET.1:v 171 ---- 172 1: 173 000004:[a#3,SET-h#3,SET] 174 2: 175 000005:[a#2,RANGEDEL-g#inf,RANGEDEL] 176 3: 177 000006:[a#0,SET-b#0,SET] 178 000007:[c#0,SET-d#0,SET] 179 000008:[e#0,SET-f#1,SET] 180 181 compact a-e L1 182 ---- 183 2: 184 000009:[a#3,SETWITHDEL-c#inf,RANGEDEL] 185 000010:[c#2,RANGEDEL-h#3,SET] 186 3: 187 000006:[a#0,SET-b#0,SET] 188 000007:[c#0,SET-d#0,SET] 189 000008:[e#0,SET-f#1,SET] 190 191 # A range tombstone is the first and only item output by a compaction, and it 192 # extends past the grandparent file boundary used to limit the size of future 193 # compactions. Verify the range tombstone is split at that file boundary. 194 195 define target-file-sizes=(1, 1, 1, 1) 196 L1 197 a.RANGEDEL.3:e 198 L2 199 a.SET.2:v 200 L3 201 a.SET.0:v 202 b.SET.0:v 203 L3 204 c.SET.0:v 205 d.SET.0:v 206 ---- 207 1: 208 000004:[a#3,RANGEDEL-e#inf,RANGEDEL] 209 2: 210 000005:[a#2,SET-a#2,SET] 211 3: 212 000006:[a#0,SET-b#0,SET] 213 000007:[c#0,SET-d#0,SET] 214 215 compact a-e L1 216 ---- 217 2: 218 000008:[a#3,RANGEDEL-c#inf,RANGEDEL] 219 000009:[c#3,RANGEDEL-e#inf,RANGEDEL] 220 3: 221 000006:[a#0,SET-b#0,SET] 222 000007:[c#0,SET-d#0,SET] 223 224 # An elided range tombstone is the first item encountered by a compaction, 225 # and the grandparent limit set by it extends to the next item, also a range 226 # tombstone. The first item should be elided, and the second item should 227 # reset the grandparent limit. 228 229 define target-file-sizes=(100, 100, 100, 100) 230 L1 231 a.RANGEDEL.4:d 232 L1 233 grandparent.RANGEDEL.2:z 234 h.SET.3:v 235 L2 236 grandparent.SET.1:v 237 L3 238 grandparent.SET.0:v 239 L3 240 m.SET.0:v 241 ---- 242 1: 243 000004:[a#4,RANGEDEL-d#inf,RANGEDEL] 244 000005:[grandparent#2,RANGEDEL-z#inf,RANGEDEL] 245 2: 246 000006:[grandparent#1,SET-grandparent#1,SET] 247 3: 248 000007:[grandparent#0,SET-grandparent#0,SET] 249 000008:[m#0,SET-m#0,SET] 250 251 compact a-h L1 252 ---- 253 2: 254 000009:[grandparent#2,RANGEDEL-m#inf,RANGEDEL] 255 000010:[m#2,RANGEDEL-z#inf,RANGEDEL] 256 3: 257 000007:[grandparent#0,SET-grandparent#0,SET] 258 000008:[m#0,SET-m#0,SET] 259 260 # Regression test for a bug where compaction would stop process range 261 # tombstones for an input level upon finding an sstable in the input 262 # level with no range tombstones. In the scenario below, sstable 6 263 # does not contain any range tombstones while sstable 7 does. Both are 264 # compacted together with sstable 5. 265 266 reset 267 ---- 268 269 batch 270 set a 1 271 set b 1 272 set c 1 273 set d 1 274 set z 1 275 ---- 276 277 compact a-z 278 ---- 279 6: 280 000005:[a#10,SET-z#14,SET] 281 282 build ext1 283 set a 2 284 ---- 285 286 build ext2 287 set b 2 288 del-range c z 289 ---- 290 291 ingest ext1 ext2 292 ---- 293 0.0: 294 000006:[a#15,SET-a#15,SET] 295 000007:[b#16,SET-z#inf,RANGEDEL] 296 6: 297 000005:[a#10,SET-z#14,SET] 298 299 iter 300 first 301 next 302 next 303 next 304 ---- 305 a: (2, .) 306 b: (2, .) 307 z: (1, .) 308 . 309 310 compact a-z 311 ---- 312 6: 313 000008:[a#0,SET-z#0,SET] 314 315 iter 316 first 317 next 318 next 319 next 320 ---- 321 a: (2, .) 322 b: (2, .) 323 z: (1, .) 324 . 325 326 # Regresion test for a bug in sstable smallest boundary generation 327 # where the smallest key for an sstable was set to a key "larger" than 328 # the start key of the first range tombstone. This in turn fouled up 329 # the processing logic of range tombstones used by mergingIter which 330 # allowed stepping out of an sstable even though it contained a range 331 # tombstone that covered keys in lower levels. 332 333 define target-file-sizes=(1, 1, 1, 1) 334 L0 335 c.SET.4:4 336 L1 337 a.SET.3:3 338 L2 339 a.RANGEDEL.2:e 340 L3 341 b.SET.1:1 342 ---- 343 0.0: 344 000004:[c#4,SET-c#4,SET] 345 1: 346 000005:[a#3,SET-a#3,SET] 347 2: 348 000006:[a#2,RANGEDEL-e#inf,RANGEDEL] 349 3: 350 000007:[b#1,SET-b#1,SET] 351 352 compact a-e L1 353 ---- 354 0.0: 355 000004:[c#4,SET-c#4,SET] 356 2: 357 000008:[a#3,SETWITHDEL-b#inf,RANGEDEL] 358 000009:[b#2,RANGEDEL-e#inf,RANGEDEL] 359 3: 360 000007:[b#1,SET-b#1,SET] 361 362 # We should only see a:3 and c:4 at this point. 363 364 iter 365 first 366 next 367 next 368 ---- 369 a: (3, .) 370 c: (4, .) 371 . 372 373 # The bug allowed seeing b:1 during reverse iteration. 374 375 iter 376 last 377 prev 378 prev 379 ---- 380 c: (4, .) 381 a: (3, .) 382 . 383 384 # This is a similar scenario to the one above. In older versions of Pebble this 385 # case necessitated adjusting the seqnum of the range tombstone to 386 # prev.LargestKey.SeqNum-1. We no longer allow user keys to be split across 387 # sstables, and the seqnum adjustment is no longer necessary. 388 # 389 # Note the target-file-size of 26 is specially tailored to get the 390 # desired compaction output. 391 392 define target-file-sizes=(26, 26, 26, 26) snapshots=(1, 2, 3) 393 L1 394 a.SET.4:4 395 L1 396 b.SET.2:2 397 b.RANGEDEL.3:e 398 L3 399 b.SET.1:1 400 ---- 401 1: 402 000004:[a#4,SET-a#4,SET] 403 000005:[b#3,RANGEDEL-e#inf,RANGEDEL] 404 3: 405 000006:[b#1,SET-b#1,SET] 406 407 compact a-e L1 408 ---- 409 2: 410 000007:[a#4,SET-a#4,SET] 411 000008:[b#3,RANGEDEL-e#inf,RANGEDEL] 412 3: 413 000006:[b#1,SET-b#1,SET] 414 415 iter 416 first 417 next 418 last 419 prev 420 ---- 421 a: (4, .) 422 . 423 a: (4, .) 424 . 425 426 # Similar to the preceding scenario, except the range tombstone has 427 # the same seqnum as the largest key in the preceding file. 428 429 define target-file-sizes=(26, 26, 26, 26) snapshots=(1, 2, 3) 430 L1 431 a.SET.4:4 432 L1 433 b.SET.3:3 434 b.RANGEDEL.3:e 435 L3 436 b.SET.1:1 437 ---- 438 1: 439 000004:[a#4,SET-a#4,SET] 440 000005:[b#3,RANGEDEL-e#inf,RANGEDEL] 441 3: 442 000006:[b#1,SET-b#1,SET] 443 444 compact a-e L1 445 ---- 446 2: 447 000007:[a#4,SET-a#4,SET] 448 000008:[b#3,RANGEDEL-e#inf,RANGEDEL] 449 3: 450 000006:[b#1,SET-b#1,SET] 451 452 iter 453 first 454 next 455 next 456 last 457 prev 458 prev 459 ---- 460 a: (4, .) 461 b: (3, .) 462 . 463 b: (3, .) 464 a: (4, .) 465 . 466 467 # Similar to the preceding scenario, except the range tombstone has 468 # a smaller seqnum than the largest key in the preceding file. 469 470 define target-file-sizes=(26, 26, 26, 26) snapshots=(1, 2, 3) 471 L1 472 a.SET.4:4 473 L1 474 b.SET.4:4 475 b.RANGEDEL.2:e 476 L3 477 b.SET.1:1 478 ---- 479 1: 480 000004:[a#4,SET-a#4,SET] 481 000005:[b#4,SET-e#inf,RANGEDEL] 482 3: 483 000006:[b#1,SET-b#1,SET] 484 485 compact a-e L1 486 ---- 487 2: 488 000007:[a#4,SET-a#4,SET] 489 000008:[b#4,SET-e#inf,RANGEDEL] 490 3: 491 000006:[b#1,SET-b#1,SET] 492 493 iter 494 first 495 next 496 next 497 last 498 prev 499 prev 500 ---- 501 a: (4, .) 502 b: (4, .) 503 . 504 b: (4, .) 505 a: (4, .) 506 . 507 508 # Test a scenario where the last point key in an sstable has a seqnum 509 # of 0. 510 511 define target-file-sizes=(1, 1, 26) snapshots=(2) 512 L1 513 a.SET.3:3 514 b.RANGEDEL.3:e 515 b.SET.0:0 516 L3 517 a.RANGEDEL.2:b 518 L3 519 c.SET.0:0 520 d.SET.0:0 521 ---- 522 1: 523 000004:[a#3,SET-e#inf,RANGEDEL] 524 3: 525 000005:[a#2,RANGEDEL-b#inf,RANGEDEL] 526 000006:[c#0,SET-d#0,SET] 527 528 iter 529 last 530 prev 531 ---- 532 a: (3, .) 533 . 534 535 compact a-e L1 536 ---- 537 2: 538 000007:[a#3,SET-c#inf,RANGEDEL] 539 000008:[c#3,RANGEDEL-e#inf,RANGEDEL] 540 3: 541 000005:[a#2,RANGEDEL-b#inf,RANGEDEL] 542 000006:[c#0,SET-d#0,SET] 543 544 iter 545 last 546 prev 547 ---- 548 a: (3, .) 549 . 550 551 # Test a scenario where the last point key in an sstable before the 552 # grandparent limit is reached has a seqnum of 0. We want to cut the 553 # sstable after the next point key is added, rather than continuing to 554 # add keys indefinitely (or till the size limit is reached). 555 556 define target-file-sizes=(100, 1, 52) snapshots=(2) 557 L1 558 a.SET.3:3 559 b.RANGEDEL.3:e 560 b.SET.0:0 561 c.SET.3:1 562 d.SET.1:1 563 L3 564 c.RANGEDEL.2:d 565 ---- 566 1: 567 000004:[a#3,SET-e#inf,RANGEDEL] 568 3: 569 000005:[c#2,RANGEDEL-d#inf,RANGEDEL] 570 571 compact a-f L1 572 ---- 573 2: 574 000006:[a#3,SET-c#inf,RANGEDEL] 575 000007:[c#3,RANGEDEL-e#inf,RANGEDEL] 576 3: 577 000005:[c#2,RANGEDEL-d#inf,RANGEDEL] 578 579 580 # Test a scenario where we the last point key in an sstable has a 581 # seqnum of 0, but there is another range tombstone later in the 582 # compaction. This scenario was previously triggering an assertion due 583 # to the rangedel.Fragmenter being finished prematurely. 584 585 define target-file-sizes=(1, 1, 1) 586 L1 587 a.SET.0:0 588 c.RANGEDEL.1:d 589 L3 590 b.SET.0:0 591 ---- 592 1: 593 000004:[a#0,SET-d#inf,RANGEDEL] 594 3: 595 000005:[b#0,SET-b#0,SET] 596 597 compact a-e L1 598 ---- 599 2: 600 000006:[a#0,SET-a#0,SET] 601 3: 602 000005:[b#0,SET-b#0,SET] 603 604 define target-file-sizes=(1, 1, 1, 1) 605 L0 606 b.SET.1:v 607 L0 608 a.SET.2:v 609 ---- 610 0.0: 611 000005:[a#2,SET-a#2,SET] 612 000004:[b#1,SET-b#1,SET] 613 614 add-ongoing-compaction startLevel=0 outputLevel=1 start=a end=z 615 ---- 616 617 async-compact a-b L0 618 ---- 619 manual compaction blocked until ongoing finished 620 1: 621 000006:[a#0,SET-a#0,SET] 622 000007:[b#0,SET-b#0,SET] 623 624 compact a-b L1 625 ---- 626 2: 627 000008:[a#0,SET-a#0,SET] 628 000009:[b#0,SET-b#0,SET] 629 630 add-ongoing-compaction startLevel=0 outputLevel=1 start=a end=z 631 ---- 632 633 async-compact a-b L2 634 ---- 635 manual compaction blocked until ongoing finished 636 3: 637 000010:[a#0,SET-a#0,SET] 638 000011:[b#0,SET-b#0,SET] 639 640 add-ongoing-compaction startLevel=0 outputLevel=1 start=a end=z 641 ---- 642 643 set-concurrent-compactions num=2 644 ---- 645 646 async-compact a-b L3 647 ---- 648 manual compaction did not block for ongoing 649 4: 650 000012:[a#0,SET-a#0,SET] 651 000013:[b#0,SET-b#0,SET] 652 653 remove-ongoing-compaction 654 ---- 655 656 add-ongoing-compaction startLevel=4 outputLevel=5 start=a end=b 657 ---- 658 659 async-compact a-b L4 660 ---- 661 manual compaction blocked until ongoing finished 662 5: 663 000014:[a#0,SET-a#0,SET] 664 000015:[b#0,SET-b#0,SET] 665 666 # Test of a scenario where consecutive elided range tombstones and grandparent 667 # boundaries could result in an invariant violation in the rangedel fragmenter. 668 669 define target-file-sizes=(1, 1, 1, 1) 670 L1 671 a.RANGEDEL.4:b 672 c.RANGEDEL.4:d 673 e.RANGEDEL.4:f 674 L1 675 g.RANGEDEL.6:h 676 i.RANGEDEL.4:j 677 L1 678 k.RANGEDEL.5:q 679 m.RANGEDEL.4:q 680 L2 681 a.SET.2:foo 682 L3 683 a.SET.1:foo 684 c.SET.1:foo 685 L3 686 ff.SET.1:v 687 L3 688 k.SET.1:foo 689 ---- 690 1: 691 000004:[a#4,RANGEDEL-f#inf,RANGEDEL] 692 000005:[g#6,RANGEDEL-j#inf,RANGEDEL] 693 000006:[k#5,RANGEDEL-q#inf,RANGEDEL] 694 2: 695 000007:[a#2,SET-a#2,SET] 696 3: 697 000008:[a#1,SET-c#1,SET] 698 000009:[ff#1,SET-ff#1,SET] 699 000010:[k#1,SET-k#1,SET] 700 701 compact a-q L1 702 ---- 703 2: 704 000011:[a#4,RANGEDEL-d#inf,RANGEDEL] 705 000012:[k#5,RANGEDEL-m#inf,RANGEDEL] 706 3: 707 000008:[a#1,SET-c#1,SET] 708 000009:[ff#1,SET-ff#1,SET] 709 000010:[k#1,SET-k#1,SET] 710 711 # Test a case where a new output file is started, there are no previous output 712 # files, there are no additional keys (key = nil) and the rangedel fragmenter 713 # is non-empty. 714 define target-file-sizes=(1, 1, 1) 715 L1 716 a.RANGEDEL.10:b 717 d.RANGEDEL.9:e 718 q.RANGEDEL.8:r 719 L2 720 g.RANGEDEL.7:h 721 L3 722 q.SET.6:6 723 ---- 724 1: 725 000004:[a#10,RANGEDEL-r#inf,RANGEDEL] 726 2: 727 000005:[g#7,RANGEDEL-h#inf,RANGEDEL] 728 3: 729 000006:[q#6,SET-q#6,SET] 730 731 compact a-r L1 732 ---- 733 2: 734 000007:[q#8,RANGEDEL-r#inf,RANGEDEL] 735 3: 736 000006:[q#6,SET-q#6,SET] 737 738 # Test a snapshot that separates a range deletion from all the data that it 739 # deletes. Ensure that we respect the target-file-size and split into multiple 740 # outputs. 741 742 define target-file-sizes=(1, 1, 1) snapshots=(14) 743 L1 744 a.RANGEDEL.15:z 745 b.SET.11:foo 746 c.SET.11:foo 747 L2 748 c.SET.0:foo 749 d.SET.0:foo 750 ---- 751 1: 752 000004:[a#15,RANGEDEL-z#inf,RANGEDEL] 753 2: 754 000005:[c#0,SET-d#0,SET] 755 756 compact a-z L1 757 ---- 758 2: 759 000006:[a#15,RANGEDEL-c#inf,RANGEDEL] 760 000007:[c#15,RANGEDEL-d#inf,RANGEDEL] 761 000008:[d#15,RANGEDEL-z#inf,RANGEDEL] 762 763 # Test an interaction between a range deletion that will be elided with 764 # output splitting. Ensure that the output is still split (previous versions 765 # of the code did not, because of intricacies around preventing a zero 766 # sequence number in an output's largest key). 767 768 define target-file-sizes=(1, 1, 1) 769 L1 770 a.RANGEDEL.10:z 771 b.SET.11:foo 772 c.SET.11:foo 773 L2 774 c.SET.0:foo 775 d.SET.0:foo 776 ---- 777 1: 778 000004:[a#10,RANGEDEL-z#inf,RANGEDEL] 779 2: 780 000005:[c#0,SET-d#0,SET] 781 782 compact a-z L1 783 ---- 784 2: 785 000006:[b#0,SET-b#0,SET] 786 000007:[c#0,SET-c#0,SET] 787 788 define snapshots=(10) 789 L1 790 a.MERGE.15:a15 791 L2 792 a.SET.5:a5 793 ---- 794 1: 795 000004:[a#15,MERGE-a#15,MERGE] 796 2: 797 000005:[a#5,SET-a#5,SET] 798 799 compact a-z 800 ---- 801 3: 802 000006:[a#15,MERGE-a#0,SET] 803 804 # Fix for #2705. When snapshotPinned was used to set force obsolete, the 805 # merged value would be a15 since the SET was incorrectly ignored. 806 iter 807 first 808 next 809 ---- 810 a: (a5a15, .) 811 .