github.com/cockroachdb/pebble@v1.1.1-0.20240513155919-3622ade60459/testdata/compaction_tombstones (about)

     1  # Test an L6 file that contains range tombstones, but whose keys are not in
     2  # the last snapshot stripe. The tombstones wouldn't be elided, so no
     3  # compaction is pursued.
     4  define snapshots=(70, 100, 180, 210)
     5  L6
     6  b.RANGEDEL.230:h h.RANGEDEL.200:r
     7  ----
     8  6:
     9    000004:[b#230,RANGEDEL-r#inf,RANGEDEL]
    10  
    11  wait-pending-table-stats
    12  000004
    13  ----
    14  num-entries: 2
    15  num-deletions: 2
    16  num-range-key-sets: 0
    17  point-deletions-bytes-estimate: 0
    18  range-deletions-bytes-estimate: 0
    19  
    20  maybe-compact
    21  ----
    22  (none)
    23  
    24  # Test the same scenario, but the file is in the last stripe. Since the file
    25  # only contains deletes, no new sstable is written.
    26  define snapshots=(270, 300, 380, 410)
    27  L6
    28  b.RANGEDEL.230:h h.RANGEDEL.200:r
    29  ----
    30  6:
    31    000004:[b#230,RANGEDEL-r#inf,RANGEDEL]
    32  
    33  wait-pending-table-stats
    34  000004
    35  ----
    36  num-entries: 2
    37  num-deletions: 2
    38  num-range-key-sets: 0
    39  point-deletions-bytes-estimate: 0
    40  range-deletions-bytes-estimate: 0
    41  
    42  maybe-compact
    43  ----
    44  [JOB 100] compacted(elision-only) L6 [000004] (748B) Score=0.00 + L6 [] (0B) Score=0.00 -> L6 [] (0B), in 1.0s (2.0s total), output rate 0B/s
    45  
    46  # Test a table that straddles a snapshot. It should not be compacted.
    47  define snapshots=(50) auto-compactions=off
    48  L6
    49  a.SET.55:a b.RANGEDEL.5:h
    50  ----
    51  6:
    52    000004:[a#55,SET-h#inf,RANGEDEL]
    53  
    54  wait-pending-table-stats
    55  000004
    56  ----
    57  num-entries: 2
    58  num-deletions: 1
    59  num-range-key-sets: 0
    60  point-deletions-bytes-estimate: 0
    61  range-deletions-bytes-estimate: 0
    62  
    63  maybe-compact
    64  ----
    65  (none)
    66  
    67  # Test a table with a point deletion and a non-deletion entry. The table
    68  # should be compacted, and a new table with the point tombstone should be
    69  # written.
    70  define auto-compactions=off
    71  L6
    72  a.SET.55:a b.DEL.5:
    73  ----
    74  6:
    75    000004:[a#55,SET-b#5,DEL]
    76  
    77  wait-pending-table-stats
    78  000004
    79  ----
    80  num-entries: 2
    81  num-deletions: 1
    82  num-range-key-sets: 0
    83  point-deletions-bytes-estimate: 93
    84  range-deletions-bytes-estimate: 0
    85  
    86  maybe-compact
    87  ----
    88  [JOB 100] compacted(elision-only) L6 [000004] (714B) Score=0.00 + L6 [] (0B) Score=0.00 -> L6 [000005] (663B), in 1.0s (2.0s total), output rate 663B/s
    89  
    90  version
    91  ----
    92  6:
    93    000005:[a#0,SET-a#0,SET]
    94  
    95  # Checking for a compaction again should not trigger a compaction, because
    96  # 000005 does not contain deletions.
    97  
    98  maybe-compact
    99  ----
   100  (none)
   101  
   102  maybe-compact
   103  ----
   104  (none)
   105  
   106  # Test a table that contains both deletions and non-deletions, but whose
   107  # deletions remove the non-deletions. The compaction should not create a new
   108  # table, but shouldn't happen until the snapshots are removed.
   109  define snapshots=(59, 103) auto-compactions=off
   110  L6
   111  a.DEL.60: a.SET.55:a b.SET.100:b c.SET.101:c d.SET.102:d b.RANGEDEL.103:z
   112  ----
   113  6:
   114    000004:[a#60,DEL-z#inf,RANGEDEL]
   115  
   116  wait-pending-table-stats
   117  000004
   118  ----
   119  num-entries: 6
   120  num-deletions: 2
   121  num-range-key-sets: 0
   122  point-deletions-bytes-estimate: 42
   123  range-deletions-bytes-estimate: 66
   124  
   125  maybe-compact
   126  ----
   127  (none)
   128  
   129  close-snapshot
   130  59
   131  ----
   132  (none)
   133  
   134  close-snapshot
   135  103
   136  ----
   137  [JOB 100] compacted(elision-only) L6 [000004] (892B) Score=0.00 + L6 [] (0B) Score=0.00 -> L6 [] (0B), in 1.0s (2.0s total), output rate 0B/s
   138  
   139  # Test a table that contains both deletions and non-deletions, but whose
   140  # non-deletions well outnumber its deletions. The table should not be
   141  # compacted because it falls beneath the threshold.
   142  define snapshots=(15) auto-compactions=off
   143  L6
   144  a.DEL.20: a.SET.1:a b.SET.2:b c.SET.3:c d.SET.4:d e.SET.5:e f.SET.6:f g.SET.7:g h.SET.8:h i.SET.9:i j.SET.10:j
   145  ----
   146  6:
   147    000004:[a#20,DEL-j#10,SET]
   148  
   149  wait-pending-table-stats
   150  000004
   151  ----
   152  num-entries: 11
   153  num-deletions: 1
   154  num-range-key-sets: 0
   155  point-deletions-bytes-estimate: 23
   156  range-deletions-bytes-estimate: 0
   157  
   158  close-snapshot
   159  15
   160  ----
   161  (none)
   162  
   163  # Test a table that contains both deletions and non-deletions, but whose
   164  # deletions remove the non-deletions. Set L5's max bytes low so that an
   165  # automatic compaction will be pursued when we call maybe-compact.
   166  # Automatic compactions need to be disabled to prevent a race where an
   167  # automatic compaction compacts before we've closen the snapshot.
   168  define snapshots=(103) level-max-bytes=(L5 : 1000) auto-compactions=off
   169  L5
   170  b.SET.200:<rand-bytes=4096> bb.SET.203:<rand-bytes=4096> cc.SET.204:<rand-bytes=4096>
   171  L5
   172  d.SET.302:<rand-bytes=4096> dd.SET.303:<rand-bytes=4096> de.SET.303:<rand-bytes=4096>
   173  L5
   174  m.SET.320:<rand-bytes=4096> n.SET.330:<rand-bytes=4096> o.SET.340:<rand-bytes=4096>
   175  L6
   176  a.SET.55:<rand-bytes=4096> b.SET.100:<rand-bytes=4096> c.SET.101:<rand-bytes=4096> d.SET.102:<rand-bytes=4096> a.RANGEDEL.103:e
   177  L6
   178  f.SET.30:<rand-bytes=4096> z.SET.31:<rand-bytes=4096>
   179  ----
   180  5:
   181    000004:[b#200,SET-cc#204,SET]
   182    000005:[d#302,SET-de#303,SET]
   183    000006:[m#320,SET-o#340,SET]
   184  6:
   185    000007:[a#103,RANGEDEL-e#inf,RANGEDEL]
   186    000008:[f#30,SET-z#31,SET]
   187  
   188  close-snapshot
   189  103
   190  ----
   191  (none)
   192  
   193  wait-pending-table-stats
   194  000007
   195  ----
   196  num-entries: 5
   197  num-deletions: 1
   198  num-range-key-sets: 0
   199  point-deletions-bytes-estimate: 0
   200  range-deletions-bytes-estimate: 16492
   201  
   202  # Because we set max bytes low, maybe-compact will trigger an automatic
   203  # compaction in preference over an elision-only compaction.
   204  # By plain file size, 000006 should be picked because it overlaps
   205  # significantly less data in L6. However, 000007 has significant obsolete
   206  # data. The compaction picker should recognize that it's more efficient to
   207  # compact (000004 + 000005) into 000007.
   208  
   209  maybe-compact
   210  ----
   211  [JOB 100] compacted(default) L5 [000004 000005] (26KB) Score=87.72 + L6 [000007] (17KB) Score=0.73 -> L6 [000009] (25KB), in 1.0s (2.0s total), output rate 25KB/s
   212  
   213  define level-max-bytes=(L5 : 1000) auto-compactions=off
   214  L5
   215  a.DEL.101: b.DEL.102: c.DEL.103:
   216  L5
   217  m.SET.107:<rand-bytes=4096>
   218  L6
   219  a.SET.001:<rand-bytes=4096> b.SET.002:<rand-bytes=4096> c.SET.003:<rand-bytes=4096>
   220  L6
   221  f.SET.007:<rand-bytes=4096> x.SET.008:<rand-bytes=4096> z.SET.009:<rand-bytes=4096>
   222  ----
   223  5:
   224    000004:[a#101,DEL-c#103,DEL]
   225    000005:[m#107,SET-m#107,SET]
   226  6:
   227    000006:[a#1,SET-c#3,SET]
   228    000007:[f#7,SET-z#9,SET]
   229  
   230  wait-pending-table-stats
   231  000004
   232  ----
   233  num-entries: 3
   234  num-deletions: 3
   235  num-range-key-sets: 0
   236  point-deletions-bytes-estimate: 6858
   237  range-deletions-bytes-estimate: 0
   238  
   239  # By plain file size, 000005 should be picked because it is larger and
   240  # overlaps the same amount of data in L6. However, 000004 has a high
   241  # point-deletions-bytes-estimate, and the compaction picker should pick 000004
   242  # instead.
   243  
   244  maybe-compact
   245  ----
   246  [JOB 100] compacted(default) L5 [000004] (724B) Score=13.45 + L6 [000006] (13KB) Score=0.92 -> L6 [] (0B), in 1.0s (2.0s total), output rate 0B/s
   247  
   248  # A table containing only range keys is not eligible for elision.
   249  # RANGEKEYDEL or RANGEKEYUNSET.
   250  
   251  define auto-compactions=off
   252  L6
   253    rangekey:a-b:{(#1,RANGEKEYDEL)}
   254  L6
   255    rangekey:b-c:{(#2,RANGEKEYUNSET,@1)}
   256  L6
   257    rangekey:c-d:{(#3,RANGEKEYSET,@1)}
   258  ----
   259  6:
   260    000004:[a#1,RANGEKEYDEL-b#inf,RANGEKEYDEL]
   261    000005:[b#2,RANGEKEYUNSET-c#inf,RANGEKEYUNSET]
   262    000006:[c#3,RANGEKEYSET-d#inf,RANGEKEYSET]
   263  
   264  wait-pending-table-stats
   265  000004
   266  ----
   267  num-entries: 0
   268  num-deletions: 0
   269  num-range-key-sets: 0
   270  point-deletions-bytes-estimate: 0
   271  range-deletions-bytes-estimate: 0
   272  
   273  wait-pending-table-stats
   274  000005
   275  ----
   276  num-entries: 0
   277  num-deletions: 0
   278  num-range-key-sets: 0
   279  point-deletions-bytes-estimate: 0
   280  range-deletions-bytes-estimate: 0
   281  
   282  wait-pending-table-stats
   283  000006
   284  ----
   285  num-entries: 0
   286  num-deletions: 0
   287  num-range-key-sets: 1
   288  point-deletions-bytes-estimate: 0
   289  range-deletions-bytes-estimate: 0
   290  
   291  maybe-compact
   292  ----
   293  (none)
   294  
   295  # Regression test for cockroachdb/cockroach#90149, exercising reference counting
   296  # on tables that contain a mixture of point, range dels and range keys.
   297  #
   298  # Place a table in L6 that contains a RANGEKEYDEL. Because this table is in the
   299  # bottom of the LSM, and there are no range keys below it, the RANGEKEYDEL is
   300  # eligible for elision (the RANGEDEL too). After the elision, the input table
   301  # should be deleted. In #90149, the table still had a reference count, and
   302  # therefore could not be deleted from the filesystem.
   303  
   304  define auto-compactions=off
   305  L6
   306    rangekey:a-b:{(#1,RANGEKEYDEL)}
   307    a.SET.2:a
   308    b.SET.3:b
   309    c.SET.4:c
   310    c.RANGEDEL.5:z
   311  ----
   312  6:
   313    000004:[a#2,SET-z#inf,RANGEDEL]
   314  
   315  wait-pending-table-stats
   316  000004
   317  ----
   318  num-entries: 3
   319  num-deletions: 1
   320  num-range-key-sets: 0
   321  point-deletions-bytes-estimate: 0
   322  range-deletions-bytes-estimate: 41
   323  
   324  maybe-compact
   325  ----
   326  [JOB 100] compacted(elision-only) L6 [000004] (894B) Score=0.00 + L6 [] (0B) Score=0.00 -> L6 [000005] (669B), in 1.0s (2.0s total), output rate 669B/s
   327  
   328  # Close the DB, asserting that the reference counts balance.
   329  close
   330  ----
   331  
   332  # Demonstration of point tombstone weighting.
   333  #
   334  # Construct an LSM with two tables in L6, with a table above each in L5. The
   335  # layout of the tables is such that the range deletion bytes estimate for table
   336  # 000005 is greater than the point deletion bytes estimate for table 000004.
   337  # Without weighting, table 000005 will be selected.
   338  
   339  define auto-compactions=off level-max-bytes=(L5 : 1000)
   340  L5
   341  a.DEL.101: b.SET.102:
   342  L5
   343  e.RANGEDEL.107:f f.SET.108:
   344  L6
   345  a.SET.001:<rand-bytes=4096> b.SET.002:<rand-bytes=4096> c.SET.003:<rand-bytes=4096>
   346  L6
   347  e.SET.007:<rand-bytes=4096> f.SET.008:<rand-bytes=4096> g.SET.009:<rand-bytes=4096>
   348  ----
   349  5:
   350    000004:[a#101,DEL-b#102,SET]
   351    000005:[e#107,RANGEDEL-f#108,SET]
   352  6:
   353    000006:[a#1,SET-c#3,SET]
   354    000007:[e#7,SET-g#9,SET]
   355  
   356  wait-pending-table-stats
   357  000004
   358  ----
   359  num-entries: 2
   360  num-deletions: 1
   361  num-range-key-sets: 0
   362  point-deletions-bytes-estimate: 2742
   363  range-deletions-bytes-estimate: 0
   364  
   365  wait-pending-table-stats
   366  000005
   367  ----
   368  num-entries: 2
   369  num-deletions: 1
   370  num-range-key-sets: 0
   371  point-deletions-bytes-estimate: 0
   372  range-deletions-bytes-estimate: 8246
   373  
   374  maybe-compact
   375  ----
   376  [JOB 100] compacted(default) L5 [000005] (741B) Score=11.82 + L6 [000007] (13KB) Score=1.05 -> L6 [000008] (4.7KB), in 1.0s (2.0s total), output rate 4.7KB/s
   377  
   378  # The same LSM as above. However, this time, with point tombstone weighting at
   379  # 2x, the table with the point tombstone (000004) will be selected as the
   380  # compaction input.
   381  
   382  define auto-compactions=off level-max-bytes=(L5 : 1000) point-tombstone-weight=2
   383  L5
   384  a.DEL.101: b.SET.102:
   385  L5
   386  e.RANGEDEL.107:f f.SET.108:
   387  L6
   388  a.SET.001:<rand-bytes=4096> b.SET.002:<rand-bytes=4096> c.SET.003:<rand-bytes=4096>
   389  L6
   390  e.SET.007:<rand-bytes=4096> f.SET.008:<rand-bytes=4096> g.SET.009:<rand-bytes=4096>
   391  ----
   392  5:
   393    000004:[a#101,DEL-b#102,SET]
   394    000005:[e#107,RANGEDEL-f#108,SET]
   395  6:
   396    000006:[a#1,SET-c#3,SET]
   397    000007:[e#7,SET-g#9,SET]
   398  
   399  wait-pending-table-stats
   400  000004
   401  ----
   402  num-entries: 2
   403  num-deletions: 1
   404  num-range-key-sets: 0
   405  point-deletions-bytes-estimate: 2742
   406  range-deletions-bytes-estimate: 0
   407  
   408  wait-pending-table-stats
   409  000005
   410  ----
   411  num-entries: 2
   412  num-deletions: 1
   413  num-range-key-sets: 0
   414  point-deletions-bytes-estimate: 0
   415  range-deletions-bytes-estimate: 8246
   416  
   417  maybe-compact
   418  ----
   419  [JOB 100] compacted(default) L5 [000005] (741B) Score=11.82 + L6 [000007] (13KB) Score=1.05 -> L6 [000008] (4.7KB), in 1.0s (2.0s total), output rate 4.7KB/s