github.com/zuoyebang/bitalostable@v1.0.1-0.20240229032404-e3b99a834294/options_test.go (about)

     1  // Copyright 2018 The LevelDB-Go and Pebble and Bitalostored Authors. All rights reserved. Use
     2  // of this source code is governed by a BSD-style license that can be found in
     3  // the LICENSE file.
     4  
     5  package bitalostable
     6  
     7  import (
     8  	"fmt"
     9  	"math/rand"
    10  	"runtime"
    11  	"testing"
    12  	"time"
    13  
    14  	"github.com/cockroachdb/errors"
    15  	"github.com/stretchr/testify/require"
    16  	"github.com/zuoyebang/bitalostable/internal/base"
    17  	"github.com/zuoyebang/bitalostable/vfs"
    18  )
    19  
    20  // testingRandomized randomizes some default options. Currently, it's
    21  // used for testing under a random format major version in some tests.
    22  func (o *Options) testingRandomized() *Options {
    23  	if o == nil {
    24  		o = &Options{}
    25  	}
    26  	if o.FormatMajorVersion == FormatDefault {
    27  		// Pick a random format major version from the range
    28  		// [MostCompatible, FormatNewest].
    29  		o.FormatMajorVersion = FormatMajorVersion(rand.Intn(int(FormatNewest)) + 1)
    30  	}
    31  	return o
    32  }
    33  
    34  func testingRandomized(o *Options) *Options {
    35  	o.testingRandomized()
    36  	return o
    37  }
    38  
    39  func TestLevelOptions(t *testing.T) {
    40  	var opts *Options
    41  	opts = opts.EnsureDefaults()
    42  
    43  	testCases := []struct {
    44  		level          int
    45  		targetFileSize int64
    46  	}{
    47  		{0, 2 << 20},
    48  		{1, (2 * 2) << 20},
    49  		{2, (4 * 2) << 20},
    50  		{3, (8 * 2) << 20},
    51  		{4, (16 * 2) << 20},
    52  		{5, (32 * 2) << 20},
    53  		{6, (64 * 2) << 20},
    54  	}
    55  	for _, c := range testCases {
    56  		l := opts.Level(c.level)
    57  		if c.targetFileSize != l.TargetFileSize {
    58  			t.Fatalf("%d: expected target-file-size %d, but found %d",
    59  				c.level, c.targetFileSize, l.TargetFileSize)
    60  		}
    61  	}
    62  }
    63  
    64  func TestOptionsString(t *testing.T) {
    65  	n := runtime.GOMAXPROCS(8)
    66  	defer runtime.GOMAXPROCS(n)
    67  
    68  	const expected = `[Version]
    69    bitalostable_version=0.1
    70  
    71  [Options]
    72    bytes_per_sync=524288
    73    cache_size=8388608
    74    cleaner=delete
    75    compaction_debt_concurrency=1073741824
    76    comparer=leveldb.BytewiseComparator
    77    disable_wal=false
    78    flush_delay_delete_range=0s
    79    flush_delay_range_key=0s
    80    flush_split_bytes=4194304
    81    format_major_version=1
    82    l0_compaction_concurrency=10
    83    l0_compaction_file_threshold=500
    84    l0_compaction_threshold=4
    85    l0_stop_writes_threshold=12
    86    lbase_max_bytes=67108864
    87    max_concurrent_compactions=1
    88    max_manifest_file_size=134217728
    89    max_open_files=1000
    90    mem_table_size=4194304
    91    mem_table_stop_writes_threshold=2
    92    min_deletion_rate=0
    93    merger=bitalostable.concatenate
    94    read_compaction_rate=16000
    95    read_sampling_multiplier=16
    96    strict_wal_tail=true
    97    table_cache_shards=8
    98    table_property_collectors=[]
    99    validate_on_ingest=false
   100    wal_dir=
   101    wal_bytes_per_sync=0
   102    max_writer_concurrency=0
   103    force_writer_parallelism=false
   104  
   105  [Level "0"]
   106    block_restart_interval=16
   107    block_size=4096
   108    compression=Snappy
   109    filter_policy=none
   110    filter_type=table
   111    index_block_size=4096
   112    target_file_size=2097152
   113  `
   114  
   115  	var opts *Options
   116  	opts = opts.EnsureDefaults()
   117  	if v := opts.String(); expected != v {
   118  		t.Fatalf("expected\n%s\nbut found\n%s", expected, v)
   119  	}
   120  }
   121  
   122  func TestOptionsCheck(t *testing.T) {
   123  	var opts *Options
   124  	opts = opts.EnsureDefaults()
   125  	s := opts.String()
   126  	require.NoError(t, opts.Check(s))
   127  	require.Regexp(t, `invalid key=value syntax`, opts.Check("foo\n"))
   128  
   129  	tmp := *opts
   130  	tmp.Comparer = &Comparer{Name: "foo"}
   131  	require.Regexp(t, `comparer name from file.*!=.*`, tmp.Check(s))
   132  
   133  	tmp = *opts
   134  	tmp.Merger = &Merger{Name: "foo"}
   135  	require.Regexp(t, `merger name from file.*!=.*`, tmp.Check(s))
   136  
   137  	// RocksDB uses a similar (INI-style) syntax for the OPTIONS file, but
   138  	// different section names and keys.
   139  	s = `
   140  [CFOptions "default"]
   141    comparator=rocksdb-comparer
   142    merge_operator=rocksdb-merger
   143  `
   144  	tmp = *opts
   145  	tmp.Comparer = &Comparer{Name: "foo"}
   146  	require.Regexp(t, `comparer name from file.*!=.*`, tmp.Check(s))
   147  
   148  	tmp.Comparer = &Comparer{Name: "rocksdb-comparer"}
   149  	tmp.Merger = &Merger{Name: "foo"}
   150  	require.Regexp(t, `merger name from file.*!=.*`, tmp.Check(s))
   151  
   152  	tmp.Merger = &Merger{Name: "rocksdb-merger"}
   153  	require.NoError(t, tmp.Check(s))
   154  
   155  	// RocksDB allows the merge operator to be unspecified, in which case it
   156  	// shows up as "nullptr".
   157  	s = `
   158  [CFOptions "default"]
   159    merge_operator=nullptr
   160  `
   161  	tmp = *opts
   162  	require.NoError(t, tmp.Check(s))
   163  }
   164  
   165  type testCleaner struct{}
   166  
   167  func (testCleaner) Clean(fs vfs.FS, fileType base.FileType, path string) error {
   168  	return nil
   169  }
   170  
   171  func (testCleaner) String() string {
   172  	return "test-cleaner"
   173  }
   174  
   175  func TestOptionsParse(t *testing.T) {
   176  	testComparer := *DefaultComparer
   177  	testComparer.Name = "test-comparer"
   178  	testMerger := *DefaultMerger
   179  	testMerger.Name = "test-merger"
   180  	var newCacheSize int64
   181  
   182  	hooks := &ParseHooks{
   183  		NewCache: func(size int64) *Cache {
   184  			newCacheSize = size
   185  			return nil
   186  		},
   187  		NewCleaner: func(name string) (Cleaner, error) {
   188  			if name == (testCleaner{}).String() {
   189  				return testCleaner{}, nil
   190  			}
   191  			return nil, errors.Errorf("unknown cleaner: %q", name)
   192  		},
   193  		NewComparer: func(name string) (*Comparer, error) {
   194  			if name == testComparer.Name {
   195  				return &testComparer, nil
   196  			}
   197  			return nil, errors.Errorf("unknown comparer: %q", name)
   198  		},
   199  		NewMerger: func(name string) (*Merger, error) {
   200  			if name == testMerger.Name {
   201  				return &testMerger, nil
   202  			}
   203  			return nil, errors.Errorf("unknown merger: %q", name)
   204  		},
   205  	}
   206  
   207  	testCases := []struct {
   208  		cleaner  Cleaner
   209  		comparer *Comparer
   210  		merger   *Merger
   211  	}{
   212  		{testCleaner{}, nil, nil},
   213  		{nil, &testComparer, nil},
   214  		{nil, nil, &testMerger},
   215  	}
   216  	for _, c := range testCases {
   217  		t.Run("", func(t *testing.T) {
   218  			var opts Options
   219  			opts.Comparer = c.comparer
   220  			opts.Merger = c.merger
   221  			opts.WALDir = "wal"
   222  			opts.Levels = make([]LevelOptions, 3)
   223  			opts.Levels[0].BlockSize = 1024
   224  			opts.Levels[1].BlockSize = 2048
   225  			opts.Levels[2].BlockSize = 4096
   226  			opts.Experimental.CompactionDebtConcurrency = 100
   227  			opts.FlushDelayDeleteRange = 10 * time.Second
   228  			opts.FlushDelayRangeKey = 11 * time.Second
   229  			opts.Experimental.MinDeletionRate = 200
   230  			opts.Experimental.ReadCompactionRate = 300
   231  			opts.Experimental.ReadSamplingMultiplier = 400
   232  			opts.Experimental.TableCacheShards = 500
   233  			opts.Experimental.MaxWriterConcurrency = 1
   234  			opts.Experimental.ForceWriterParallelism = true
   235  			opts.EnsureDefaults()
   236  			str := opts.String()
   237  
   238  			newCacheSize = 0
   239  			var parsedOptions Options
   240  			require.NoError(t, parsedOptions.Parse(str, hooks))
   241  			parsedStr := parsedOptions.String()
   242  			if str != parsedStr {
   243  				t.Fatalf("expected\n%s\nbut found\n%s", str, parsedStr)
   244  			}
   245  			require.Nil(t, parsedOptions.Cache)
   246  			require.NotEqual(t, newCacheSize, 0)
   247  		})
   248  	}
   249  }
   250  
   251  func TestOptionsValidate(t *testing.T) {
   252  	testCases := []struct {
   253  		options  string
   254  		expected string
   255  	}{
   256  		{``, ``},
   257  		{`
   258  [Options]
   259    l0_compaction_concurrency=0
   260  `,
   261  			`L0CompactionConcurrency \(0\) must be >= 1`,
   262  		},
   263  		{`
   264  [Options]
   265    l0_compaction_threshold=2
   266    l0_stop_writes_threshold=1
   267  `,
   268  			`L0StopWritesThreshold .* must be >= L0CompactionThreshold .*`,
   269  		},
   270  		{`
   271  [Options]
   272    mem_table_size=4294967296
   273  `,
   274  			`MemTableSize \(4\.0 G\) must be < 4\.0 G`,
   275  		},
   276  		{`
   277  [Options]
   278    mem_table_stop_writes_threshold=1
   279  `,
   280  			`MemTableStopWritesThreshold .* must be >= 2`,
   281  		},
   282  	}
   283  
   284  	for _, c := range testCases {
   285  		t.Run("", func(t *testing.T) {
   286  			var opts Options
   287  			opts.EnsureDefaults()
   288  			require.NoError(t, opts.Parse(c.options, nil))
   289  			err := opts.Validate()
   290  			if c.expected == "" {
   291  				require.NoError(t, err)
   292  			} else {
   293  				require.Error(t, err)
   294  				require.Regexp(t, c.expected, err.Error())
   295  			}
   296  		})
   297  	}
   298  }
   299  
   300  // This test isn't being done in TestOptionsValidate
   301  // cause it doesn't support setting pointers.
   302  func TestOptionsValidateCache(t *testing.T) {
   303  	var opts Options
   304  	opts.EnsureDefaults()
   305  	opts.Cache = NewCache(8 << 20)
   306  	defer opts.Cache.Unref()
   307  	opts.TableCache = NewTableCache(NewCache(8<<20), 10, 1)
   308  	defer opts.TableCache.cache.Unref()
   309  	defer opts.TableCache.Unref()
   310  
   311  	err := opts.Validate()
   312  	require.Error(t, err)
   313  	if fmt.Sprint(err) != "underlying cache in the TableCache and the Cache dont match" {
   314  		t.Errorf("Unexpected error message")
   315  	}
   316  }