github.com/cockroachdb/pebble@v1.1.1-0.20240513155919-3622ade60459/testdata/compaction_delete_only_hints (about) 1 # The first few cases are adapted from this ASCII example. The y-axis is 2 # sequence numbers and the x-axis is the user key space. LSM levels are 3 # omitted from the visualization. 4 # 5 # 250 6 # +--------00004 (fragmented)------+ 7 # V | 8 # |-b...230:h-| | 9 # _______________________________________V_____________ snapshot #210 10 # 200 |--h.RANGEDEL.200:r--| 11 # 12 # _____________________________________________________ snapshot #180 13 # 14 # 150 +--------+ 15 # +---------+ | 000006 | 16 # | 000005 | | | 17 # +_________+ | | 18 # 100_____________________|________|___________________ snapshot #100 19 # +--------+ 20 # _____________________________________________________ snapshot #70 21 # +---------------+ 22 # 50 | 000007 | 23 # | | 24 # +---------------+ 25 # ______________________________________________________________ 26 # a b c d e f g h i j k l m n o p q r s t u v w x y z 27 28 define snapshots=(70, 100, 180, 210) 29 L0 30 b.RANGEDEL.230:h h.RANGEDEL.200:r 31 L2 32 d.SET.110:d i.SET.140:i 33 L3 34 k.SET.90:k o.SET.150:o 35 L4 36 m.SET.30:m u.SET.60:u 37 ---- 38 0.0: 39 000004:[b#230,RANGEDEL-r#inf,RANGEDEL] 40 2: 41 000005:[d#110,SET-i#140,SET] 42 3: 43 000006:[k#90,SET-o#150,SET] 44 4: 45 000007:[m#30,SET-u#60,SET] 46 47 # Test a hint that is blocked by open snapshots. No compaction should occur 48 # and the hint should not be removed. 49 50 get-hints 51 ---- 52 L0.000004 b-r seqnums(tombstone=200-230, file-smallest=90, type=point-key-only) 53 54 maybe-compact 55 ---- 56 Deletion hints: 57 L0.000004 b-r seqnums(tombstone=200-230, file-smallest=90, type=point-key-only) 58 Compactions: 59 (none) 60 61 # Adopt the same LSM but without snapshots 100, 180 and 210. 62 63 define snapshots=(70) 64 L0 65 b.RANGEDEL.230:h h.RANGEDEL.200:r 66 L2 67 d.SET.110:d i.SET.140:i 68 L3 69 k.SET.90:k o.SET.150:o 70 L4 71 m.SET.30:m u.SET.60:u 72 ---- 73 0.0: 74 000004:[b#230,RANGEDEL-r#inf,RANGEDEL] 75 2: 76 000005:[d#110,SET-i#140,SET] 77 3: 78 000006:[k#90,SET-o#150,SET] 79 4: 80 000007:[m#30,SET-u#60,SET] 81 82 get-hints 83 ---- 84 L0.000004 b-r seqnums(tombstone=200-230, file-smallest=90, type=point-key-only) 85 86 maybe-compact 87 ---- 88 Deletion hints: 89 (none) 90 Compactions: 91 [JOB 100] compacted(delete-only) L2 [000005] (677B) Score=0.00 + L3 [000006] (677B) Score=0.00 -> L6 [] (0B), in 1.0s (2.0s total), output rate 0B/s 92 93 # Verify that compaction correctly handles the presence of multiple 94 # overlapping hints which might delete a file multiple times. All of the 95 # resolvable hints should be removed. 96 97 define snapshots=(70) 98 L0 99 a.RANGEDEL.300:k 100 L1 101 b.RANGEDEL.230:h h.RANGEDEL.200:r 102 L2 103 d.SET.110:d i.SET.140:i 104 L3 105 k.SET.90:k o.SET.150:o 106 L4 107 m.SET.30:m u.SET.60:u 108 ---- 109 0.0: 110 000004:[a#300,RANGEDEL-k#inf,RANGEDEL] 111 1: 112 000005:[b#230,RANGEDEL-r#inf,RANGEDEL] 113 2: 114 000006:[d#110,SET-i#140,SET] 115 3: 116 000007:[k#90,SET-o#150,SET] 117 4: 118 000008:[m#30,SET-u#60,SET] 119 120 get-hints 121 ---- 122 L0.000004 a-k seqnums(tombstone=300-300, file-smallest=110, type=point-key-only) 123 L1.000005 b-r seqnums(tombstone=200-230, file-smallest=90, type=point-key-only) 124 125 maybe-compact 126 ---- 127 Deletion hints: 128 (none) 129 Compactions: 130 [JOB 100] compacted(delete-only) L2 [000006] (677B) Score=0.00 + L3 [000007] (677B) Score=0.00 -> L6 [] (0B), in 1.0s (2.0s total), output rate 0B/s 131 132 # Test a range tombstone that is already compacted into L6. 133 134 define snapshots=(70) 135 L0 136 m.SET.300:m b.RANGEDEL.230:h h.RANGEDEL.200:r 137 L2 138 d.SET.110:d i.SET.140:i 139 L3 140 k.SET.90:k o.SET.150:o 141 L4 142 m.SET.30:m u.SET.60:u 143 ---- 144 0.0: 145 000004:[b#230,RANGEDEL-r#inf,RANGEDEL] 146 2: 147 000005:[d#110,SET-i#140,SET] 148 3: 149 000006:[k#90,SET-o#150,SET] 150 4: 151 000007:[m#30,SET-u#60,SET] 152 153 get-hints 154 ---- 155 L0.000004 b-r seqnums(tombstone=200-230, file-smallest=90, type=point-key-only) 156 157 compact a-z 158 ---- 159 5: 160 000008:[b#230,RANGEDEL-u#0,SET] 161 162 maybe-compact 163 ---- 164 Deletion hints: 165 (none) 166 Compactions: 167 (none) 168 169 # The same test case, without snapshots, with a table (000008) that exists 170 # within the range del user key bounds, but above it in the LSM. 171 172 define 173 L1 174 b.RANGEDEL.230:h h.RANGEDEL.200:r 175 L2 176 d.SET.110:d i.SET.140:i 177 L3 178 k.SET.90:k o.SET.150:o 179 L4 180 m.SET.30:m u.SET.60:u 181 L0 182 e.SET.240:e m.SET.260:m 183 ---- 184 0.0: 185 000008:[e#240,SET-m#260,SET] 186 1: 187 000004:[b#230,RANGEDEL-r#inf,RANGEDEL] 188 2: 189 000005:[d#110,SET-i#140,SET] 190 3: 191 000006:[k#90,SET-o#150,SET] 192 4: 193 000007:[m#30,SET-u#60,SET] 194 195 get-hints 196 ---- 197 L1.000004 b-r seqnums(tombstone=200-230, file-smallest=90, type=point-key-only) 198 199 # Tables 000005 and 000006 can be deleted as their largest sequence numbers fall 200 # below the smallest sequence number of the range del. Table 000007 falls 201 # outside the user key bounds, and table 000008 exists at a sequence number 202 # above the range del, so neither are deleted. 203 204 maybe-compact 205 ---- 206 Deletion hints: 207 (none) 208 Compactions: 209 [JOB 100] compacted(delete-only) L2 [000005] (677B) Score=0.00 + L3 [000006] (677B) Score=0.00 -> L6 [] (0B), in 1.0s (2.0s total), output rate 0B/s 210 211 # A deletion hint present on an sstable in a higher level should NOT result in a 212 # deletion-only compaction incorrectly removing an sstable in L6 following an 213 # elision-only compaction that zeroes the sequence numbers in an L6 table. 214 # 215 # This is a regression test for pebble#1285. 216 217 # Create an sstable at L6. We expect that the SET survives the following 218 # sequence of compactions. 219 define snapshots=(10, 25) 220 L6 221 a.SET.20:b a.RANGEDEL.15:z 222 ---- 223 6: 224 000004:[a#20,SETWITHDEL-z#inf,RANGEDEL] 225 226 # Note that this test depends on stats being present on the sstables, so we 227 # collect hints here. We expect none, as the table is in L6. 228 get-hints 229 ---- 230 (none) 231 232 # Place a compaction hint on a non-existent table in a higher level in the LSM. 233 # 234 # The selection of the sequence numbers for the hints is nuanced, and warrants 235 # some explanation. The largest tombstone sequence number (27) and file smallest 236 # sequence number (0) were chosen such that they fall into different snapshot 237 # stripes, which ensures the hint is not resolved and dropped. The deletion 238 # range 5-27 is also chosen such that it covers the sequence number range from 239 # the table, i.e. 15-20, which *appears* to make the keys eligible for deletion. 240 force-set-hints 241 L0.000001 a-z 0 5-27 point_key_only 242 ---- 243 L0.000001 a-z seqnums(tombstone=5-27, file-smallest=0, type=point-key-only) 244 245 # Hints on the table are unchanged, as the new sstable is at L6, and hints are 246 # not generated on tables at this level. 247 get-hints 248 ---- 249 L0.000001 a-z seqnums(tombstone=5-27, file-smallest=0, type=point-key-only) 250 251 # Closing snapshot 10 triggers an elision-only compaction in L6 rather than a 252 # deletion-only compaction, as the earliest snapshot that remains open is 25, 253 # preventing the delete compaction hint from being resolved as it does not exist 254 # in the same snapshot stripe as the table in L6. 255 close-snapshot 256 10 257 ---- 258 [JOB 100] compacted(elision-only) L6 [000004] (741B) Score=0.00 + L6 [] (0B) Score=0.00 -> L6 [000005] (662B), in 1.0s (2.0s total), output rate 662B/s 259 260 # The deletion hint was removed by the elision-only compaction. 261 get-hints 262 ---- 263 (none) 264 265 # The LSM contains the key, as expected. 266 iter 267 first 268 next 269 ---- 270 a: (b, .) 271 . 272 273 # Closing the next snapshot should NOT trigger another compaction, as the 274 # deletion hint was removed in the elision-only compaction. 275 close-snapshot 276 25 277 ---- 278 (none) 279 280 # The key remains in the LSM. 281 iter 282 first 283 next 284 ---- 285 a: (b, .) 286 . 287 288 # Construct a scenario with tables containing a mixture of range dels and range 289 # key dels that sit within different types of hints. 290 # 291 # +------- 000013 (internally fragmented spans) ----| 292 # | V 293 # | |-------------------------| m.RANGEKEYDEL:z 294 # | |-------| i.RANGEKEYDEL:m 295 # V |-----------------------| f.RANGEDEL:r 296 # |---------| a.RANGEDEL:f 297 # +-+ +---+ +---+ 298 # | | 000006 | | 000009 | | 000012 <- Point keys only. 299 # +-+ +---+ +---+ 300 # +---+ +---+ +---+ 301 # | | 000005 | | 000008 | | 000011 <- Range keys only. 302 # +---+ +---+ +---+ 303 # +---+ +---+ +---+ 304 # | | 000004 | | 000007 | | 000010 <- Point and range keys. 305 # +---+ +---+ +---+ 306 # __________________________________________________________ 307 # a b c d e f g h i j k l m n o p q r s t u v w x y z 308 # 309 # Note that table 000013 contains both range dels and range key dels that have 310 # been internally fragmented. After defragmentation there are three hints 311 # created: 312 # - [a, i) - a point-key-only hint 313 # - [i, r) - a point-and-range-key hint 314 # - [r, z) - a range-key-only hint 315 # 316 # Based on the defragmented hints, the following tables can be deleted: 317 # - 000006: covered by range del hint [a, i), table contains only point keys. 318 # - 000007: covered by mixed hint [i, r), table contains point and range keys. 319 # - 000008: covered by mixed hint [i, r), table contains only range keys. 320 # - 000009: covered by mixed hint [i, r), table contains only point keys. 321 # - 000011: covered by range key hint [r, z), table contains only range keys. 322 # 323 324 # NOTE: the LSM shown in the example above is created bottom-up via ingestions. 325 326 reset 327 ---- 328 329 ingest ext 330 set a a 331 range-key-set a c @1 foo 332 set c c 333 ---- 334 OK 335 336 ingest ext 337 range-key-set d f @2 bar 338 ---- 339 OK 340 341 ingest ext 342 set g g 343 set h h 344 ---- 345 OK 346 347 ingest ext 348 set i i 349 range-key-set i k @1 v1 350 set k k 351 ---- 352 OK 353 354 ingest ext 355 range-key-set l n @2 bar 356 ---- 357 OK 358 359 ingest ext 360 set o o 361 set q q 362 ---- 363 OK 364 365 ingest ext 366 set r r 367 range-key-set r t @1 v1 368 set t t 369 ---- 370 OK 371 372 ingest ext 373 range-key-set u w @2 bar 374 ---- 375 OK 376 377 ingest ext 378 set x x 379 set z z 380 ---- 381 OK 382 383 ingest ext 384 del-range a f 385 del-range f r 386 range-key-del i m 387 range-key-del m z 388 ---- 389 OK 390 391 describe-lsm 392 ---- 393 0.0: 394 000013:[a#19,RANGEDEL-z#inf,RANGEKEYDEL] 395 6: 396 000004:[a#10,RANGEKEYSET-c#10,SET] 397 000005:[d#11,RANGEKEYSET-f#inf,RANGEKEYSET] 398 000006:[g#12,SET-h#12,SET] 399 000007:[i#13,RANGEKEYSET-k#13,SET] 400 000008:[l#14,RANGEKEYSET-n#inf,RANGEKEYSET] 401 000009:[o#15,SET-q#15,SET] 402 000010:[r#16,RANGEKEYSET-t#16,SET] 403 000011:[u#17,RANGEKEYSET-w#inf,RANGEKEYSET] 404 000012:[x#18,SET-z#18,SET] 405 406 get-hints 407 ---- 408 L0.000013 a-i seqnums(tombstone=19-19, file-smallest=12, type=point-key-only) 409 L0.000013 i-r seqnums(tombstone=19-19, file-smallest=13, type=point-and-range-key) 410 L0.000013 r-z seqnums(tombstone=19-19, file-smallest=17, type=range-key-only) 411 412 maybe-compact 413 ---- 414 Deletion hints: 415 (none) 416 Compactions: 417 [JOB 100] compacted(delete-only) L6 [000006 000007 000008 000009 000011] (3.9KB) Score=0.00 -> L6 [] (0B), in 1.0s (2.0s total), output rate 0B/s