github.com/cockroachdb/pebble@v1.1.1-0.20240513155919-3622ade60459/options_test.go (about) 1 // Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 // of this source code is governed by a BSD-style license that can be found in 3 // the LICENSE file. 4 5 package pebble 6 7 import ( 8 "fmt" 9 "math/rand" 10 "runtime" 11 "testing" 12 "time" 13 14 "github.com/cockroachdb/errors" 15 "github.com/cockroachdb/pebble/internal/base" 16 "github.com/cockroachdb/pebble/vfs" 17 "github.com/stretchr/testify/require" 18 ) 19 20 // testingRandomized randomizes some default options. Currently, it's 21 // used for testing under a random format major version in some tests. 22 func (o *Options) testingRandomized(t testing.TB) *Options { 23 if o == nil { 24 o = &Options{} 25 } 26 if o.Logger == nil { 27 o.Logger = testLogger{t: t} 28 } 29 if o.FormatMajorVersion == FormatDefault { 30 // Pick a random format major version from the range 31 // [MostCompatible, FormatNewest]. 32 o.FormatMajorVersion = FormatMajorVersion(rand.Intn(int(internalFormatNewest)) + 1) 33 t.Logf("Running %s with format major version %s", t.Name(), o.FormatMajorVersion.String()) 34 } 35 return o 36 } 37 38 func testingRandomized(t testing.TB, o *Options) *Options { 39 o.testingRandomized(t) 40 return o 41 } 42 43 func TestLevelOptions(t *testing.T) { 44 var opts *Options 45 opts = opts.EnsureDefaults() 46 47 testCases := []struct { 48 level int 49 targetFileSize int64 50 }{ 51 {0, 2 << 20}, 52 {1, (2 * 2) << 20}, 53 {2, (4 * 2) << 20}, 54 {3, (8 * 2) << 20}, 55 {4, (16 * 2) << 20}, 56 {5, (32 * 2) << 20}, 57 {6, (64 * 2) << 20}, 58 } 59 for _, c := range testCases { 60 l := opts.Level(c.level) 61 if c.targetFileSize != l.TargetFileSize { 62 t.Fatalf("%d: expected target-file-size %d, but found %d", 63 c.level, c.targetFileSize, l.TargetFileSize) 64 } 65 } 66 } 67 68 func TestOptionsString(t *testing.T) { 69 n := runtime.GOMAXPROCS(8) 70 defer runtime.GOMAXPROCS(n) 71 72 const expected = `[Version] 73 pebble_version=0.1 74 75 [Options] 76 bytes_per_sync=524288 77 cache_size=8388608 78 cleaner=delete 79 compaction_debt_concurrency=1073741824 80 comparer=leveldb.BytewiseComparator 81 disable_wal=false 82 flush_delay_delete_range=0s 83 flush_delay_range_key=0s 84 flush_split_bytes=4194304 85 format_major_version=1 86 l0_compaction_concurrency=10 87 l0_compaction_file_threshold=500 88 l0_compaction_threshold=4 89 l0_stop_writes_threshold=12 90 lbase_max_bytes=67108864 91 max_concurrent_compactions=1 92 max_manifest_file_size=134217728 93 max_open_files=1000 94 mem_table_size=4194304 95 mem_table_stop_writes_threshold=2 96 min_deletion_rate=0 97 merger=pebble.concatenate 98 read_compaction_rate=16000 99 read_sampling_multiplier=16 100 strict_wal_tail=true 101 table_cache_shards=8 102 table_property_collectors=[] 103 validate_on_ingest=false 104 wal_dir= 105 wal_bytes_per_sync=0 106 max_writer_concurrency=0 107 force_writer_parallelism=false 108 secondary_cache_size_bytes=0 109 create_on_shared=0 110 111 [Level "0"] 112 block_restart_interval=16 113 block_size=4096 114 block_size_threshold=90 115 compression=Snappy 116 filter_policy=none 117 filter_type=table 118 index_block_size=4096 119 target_file_size=2097152 120 ` 121 122 var opts *Options 123 opts = opts.EnsureDefaults() 124 if v := opts.String(); expected != v { 125 t.Fatalf("expected\n%s\nbut found\n%s", expected, v) 126 } 127 } 128 129 func TestOptionsCheck(t *testing.T) { 130 var opts *Options 131 opts = opts.EnsureDefaults() 132 s := opts.String() 133 require.NoError(t, opts.Check(s)) 134 require.Regexp(t, `invalid key=value syntax`, opts.Check("foo\n")) 135 136 tmp := *opts 137 tmp.Comparer = &Comparer{Name: "foo"} 138 require.Regexp(t, `comparer name from file.*!=.*`, tmp.Check(s)) 139 140 tmp = *opts 141 tmp.Merger = &Merger{Name: "foo"} 142 require.Regexp(t, `merger name from file.*!=.*`, tmp.Check(s)) 143 144 // RocksDB uses a similar (INI-style) syntax for the OPTIONS file, but 145 // different section names and keys. 146 s = ` 147 [CFOptions "default"] 148 comparator=rocksdb-comparer 149 merge_operator=rocksdb-merger 150 ` 151 tmp = *opts 152 tmp.Comparer = &Comparer{Name: "foo"} 153 require.Regexp(t, `comparer name from file.*!=.*`, tmp.Check(s)) 154 155 tmp.Comparer = &Comparer{Name: "rocksdb-comparer"} 156 tmp.Merger = &Merger{Name: "foo"} 157 require.Regexp(t, `merger name from file.*!=.*`, tmp.Check(s)) 158 159 tmp.Merger = &Merger{Name: "rocksdb-merger"} 160 require.NoError(t, tmp.Check(s)) 161 162 // RocksDB allows the merge operator to be unspecified, in which case it 163 // shows up as "nullptr". 164 s = ` 165 [CFOptions "default"] 166 merge_operator=nullptr 167 ` 168 tmp = *opts 169 require.NoError(t, tmp.Check(s)) 170 } 171 172 type testCleaner struct{} 173 174 func (testCleaner) Clean(fs vfs.FS, fileType base.FileType, path string) error { 175 return nil 176 } 177 178 func (testCleaner) String() string { 179 return "test-cleaner" 180 } 181 182 func TestOptionsParse(t *testing.T) { 183 testComparer := *DefaultComparer 184 testComparer.Name = "test-comparer" 185 testMerger := *DefaultMerger 186 testMerger.Name = "test-merger" 187 var newCacheSize int64 188 189 hooks := &ParseHooks{ 190 NewCache: func(size int64) *Cache { 191 newCacheSize = size 192 return nil 193 }, 194 NewCleaner: func(name string) (Cleaner, error) { 195 if name == (testCleaner{}).String() { 196 return testCleaner{}, nil 197 } 198 return nil, errors.Errorf("unknown cleaner: %q", name) 199 }, 200 NewComparer: func(name string) (*Comparer, error) { 201 if name == testComparer.Name { 202 return &testComparer, nil 203 } 204 return nil, errors.Errorf("unknown comparer: %q", name) 205 }, 206 NewMerger: func(name string) (*Merger, error) { 207 if name == testMerger.Name { 208 return &testMerger, nil 209 } 210 return nil, errors.Errorf("unknown merger: %q", name) 211 }, 212 } 213 214 testCases := []struct { 215 cleaner Cleaner 216 comparer *Comparer 217 merger *Merger 218 }{ 219 {testCleaner{}, nil, nil}, 220 {nil, &testComparer, nil}, 221 {nil, nil, &testMerger}, 222 } 223 for _, c := range testCases { 224 t.Run("", func(t *testing.T) { 225 var opts Options 226 opts.Comparer = c.comparer 227 opts.Merger = c.merger 228 opts.WALDir = "wal" 229 opts.Levels = make([]LevelOptions, 3) 230 opts.Levels[0].BlockSize = 1024 231 opts.Levels[1].BlockSize = 2048 232 opts.Levels[2].BlockSize = 4096 233 opts.Experimental.CompactionDebtConcurrency = 100 234 opts.FlushDelayDeleteRange = 10 * time.Second 235 opts.FlushDelayRangeKey = 11 * time.Second 236 opts.Experimental.LevelMultiplier = 5 237 opts.TargetByteDeletionRate = 200 238 opts.Experimental.ReadCompactionRate = 300 239 opts.Experimental.ReadSamplingMultiplier = 400 240 opts.Experimental.TableCacheShards = 500 241 opts.Experimental.MaxWriterConcurrency = 1 242 opts.Experimental.ForceWriterParallelism = true 243 opts.Experimental.SecondaryCacheSizeBytes = 1024 244 opts.EnsureDefaults() 245 str := opts.String() 246 247 newCacheSize = 0 248 var parsedOptions Options 249 require.NoError(t, parsedOptions.Parse(str, hooks)) 250 parsedStr := parsedOptions.String() 251 if str != parsedStr { 252 t.Fatalf("expected\n%s\nbut found\n%s", str, parsedStr) 253 } 254 require.Nil(t, parsedOptions.Cache) 255 require.NotEqual(t, newCacheSize, 0) 256 }) 257 } 258 } 259 260 func TestOptionsValidate(t *testing.T) { 261 testCases := []struct { 262 options string 263 expected string 264 }{ 265 {``, ``}, 266 {` 267 [Options] 268 l0_compaction_concurrency=0 269 `, 270 `L0CompactionConcurrency \(0\) must be >= 1`, 271 }, 272 {` 273 [Options] 274 l0_compaction_threshold=2 275 l0_stop_writes_threshold=1 276 `, 277 `L0StopWritesThreshold .* must be >= L0CompactionThreshold .*`, 278 }, 279 {` 280 [Options] 281 mem_table_size=4294967296 282 `, 283 `MemTableSize \(4\.0GB\) must be < [2|4]\.0GB`, 284 }, 285 {` 286 [Options] 287 mem_table_stop_writes_threshold=1 288 `, 289 `MemTableStopWritesThreshold .* must be >= 2`, 290 }, 291 } 292 293 for _, c := range testCases { 294 t.Run("", func(t *testing.T) { 295 var opts Options 296 opts.EnsureDefaults() 297 require.NoError(t, opts.Parse(c.options, nil)) 298 err := opts.Validate() 299 if c.expected == "" { 300 require.NoError(t, err) 301 } else { 302 require.Error(t, err) 303 require.Regexp(t, c.expected, err.Error()) 304 } 305 }) 306 } 307 } 308 309 // This test isn't being done in TestOptionsValidate 310 // cause it doesn't support setting pointers. 311 func TestOptionsValidateCache(t *testing.T) { 312 var opts Options 313 opts.EnsureDefaults() 314 opts.Cache = NewCache(8 << 20) 315 defer opts.Cache.Unref() 316 opts.TableCache = NewTableCache(NewCache(8<<20), 10, 1) 317 defer opts.TableCache.cache.Unref() 318 defer opts.TableCache.Unref() 319 320 err := opts.Validate() 321 require.Error(t, err) 322 if fmt.Sprint(err) != "underlying cache in the TableCache and the Cache dont match" { 323 t.Errorf("Unexpected error message") 324 } 325 }