github.com/zuoyebang/bitalostable@v1.0.1-0.20240229032404-e3b99a834294/format_major_version_test.go (about)

     1  // Copyright 2021 The LevelDB-Go and Pebble and Bitalostored Authors. All rights reserved. Use
     2  // of this source code is governed by a BSD-style license that can be found in
     3  // the LICENSE file.
     4  
     5  package bitalostable
     6  
     7  import (
     8  	"bytes"
     9  	"fmt"
    10  	"strconv"
    11  	"testing"
    12  	"time"
    13  
    14  	"github.com/stretchr/testify/require"
    15  	"github.com/zuoyebang/bitalostable/internal/base"
    16  	"github.com/zuoyebang/bitalostable/internal/datadriven"
    17  	"github.com/zuoyebang/bitalostable/sstable"
    18  	"github.com/zuoyebang/bitalostable/vfs"
    19  	"github.com/zuoyebang/bitalostable/vfs/atomicfs"
    20  )
    21  
    22  func TestFormatMajorVersion_MigrationDefined(t *testing.T) {
    23  	for v := FormatMostCompatible; v <= FormatNewest; v++ {
    24  		if _, ok := formatMajorVersionMigrations[v]; !ok {
    25  			t.Errorf("format major version %d has no migration defined", v)
    26  		}
    27  	}
    28  }
    29  
    30  func TestRatchetFormat(t *testing.T) {
    31  	fs := vfs.NewMem()
    32  	d, err := Open("", &Options{FS: fs})
    33  	require.NoError(t, err)
    34  	require.NoError(t, d.Set([]byte("foo"), []byte("bar"), Sync))
    35  	require.Equal(t, FormatMostCompatible, d.FormatMajorVersion())
    36  	require.NoError(t, d.RatchetFormatMajorVersion(FormatVersioned))
    37  	require.Equal(t, FormatVersioned, d.FormatMajorVersion())
    38  	require.NoError(t, d.RatchetFormatMajorVersion(FormatVersioned))
    39  	require.Equal(t, FormatVersioned, d.FormatMajorVersion())
    40  	require.NoError(t, d.RatchetFormatMajorVersion(FormatSetWithDelete))
    41  	require.Equal(t, FormatSetWithDelete, d.FormatMajorVersion())
    42  	require.NoError(t, d.RatchetFormatMajorVersion(FormatBlockPropertyCollector))
    43  	require.Equal(t, FormatBlockPropertyCollector, d.FormatMajorVersion())
    44  	require.NoError(t, d.RatchetFormatMajorVersion(FormatSplitUserKeysMarked))
    45  	require.Equal(t, FormatSplitUserKeysMarked, d.FormatMajorVersion())
    46  	require.NoError(t, d.RatchetFormatMajorVersion(FormatSplitUserKeysMarkedCompacted))
    47  	require.Equal(t, FormatSplitUserKeysMarkedCompacted, d.FormatMajorVersion())
    48  	require.NoError(t, d.RatchetFormatMajorVersion(FormatRangeKeys))
    49  	require.Equal(t, FormatRangeKeys, d.FormatMajorVersion())
    50  	require.NoError(t, d.RatchetFormatMajorVersion(FormatMinTableFormatPebblev1))
    51  	require.Equal(t, FormatMinTableFormatPebblev1, d.FormatMajorVersion())
    52  	require.NoError(t, d.RatchetFormatMajorVersion(FormatPrePebblev1Marked))
    53  	require.Equal(t, FormatPrePebblev1Marked, d.FormatMajorVersion())
    54  	require.NoError(t, d.RatchetFormatMajorVersion(FormatPrePebblev1MarkedCompacted))
    55  	require.Equal(t, FormatPrePebblev1MarkedCompacted, d.FormatMajorVersion())
    56  	require.NoError(t, d.Close())
    57  
    58  	// If we Open the database again, leaving the default format, the
    59  	// database should Open using the persisted FormatNewest.
    60  	d, err = Open("", &Options{FS: fs})
    61  	require.NoError(t, err)
    62  	require.Equal(t, FormatNewest, d.FormatMajorVersion())
    63  	require.NoError(t, d.Close())
    64  
    65  	// Move the marker to a version that does not exist.
    66  	m, _, err := atomicfs.LocateMarker(fs, "", formatVersionMarkerName)
    67  	require.NoError(t, err)
    68  	require.NoError(t, m.Move("999999"))
    69  	require.NoError(t, m.Close())
    70  
    71  	_, err = Open("", &Options{
    72  		FS:                 fs,
    73  		FormatMajorVersion: FormatVersioned,
    74  	})
    75  	require.Error(t, err)
    76  	require.EqualError(t, err, `bitalostable: database "" written in format major version 999999`)
    77  }
    78  
    79  func testBasicDB(d *DB) error {
    80  	key := []byte("a")
    81  	value := []byte("b")
    82  	if err := d.Set(key, value, nil); err != nil {
    83  		return err
    84  	}
    85  	if err := d.Flush(); err != nil {
    86  		return err
    87  	}
    88  	if err := d.Compact(nil, []byte("\xff"), false); err != nil {
    89  		return err
    90  	}
    91  
    92  	iter := d.NewIter(nil)
    93  	for valid := iter.First(); valid; valid = iter.Next() {
    94  	}
    95  	if err := iter.Close(); err != nil {
    96  		return err
    97  	}
    98  	return nil
    99  }
   100  
   101  func TestFormatMajorVersions(t *testing.T) {
   102  	for vers := FormatMostCompatible; vers <= FormatNewest; vers++ {
   103  		t.Run(fmt.Sprintf("vers=%03d", vers), func(t *testing.T) {
   104  			fs := vfs.NewStrictMem()
   105  			opts := &Options{
   106  				FS:                 fs,
   107  				FormatMajorVersion: vers,
   108  			}
   109  
   110  			// Create a database at this format major version and perform
   111  			// some very basic operations.
   112  			d, err := Open("", opts)
   113  			require.NoError(t, err)
   114  			require.NoError(t, testBasicDB(d))
   115  			require.NoError(t, d.Close())
   116  
   117  			// Re-open the database at this format major version, and again
   118  			// perform some basic operations.
   119  			d, err = Open("", opts)
   120  			require.NoError(t, err)
   121  			require.NoError(t, testBasicDB(d))
   122  			require.NoError(t, d.Close())
   123  
   124  			t.Run("upgrade-at-open", func(t *testing.T) {
   125  				for upgradeVers := vers + 1; upgradeVers <= FormatNewest; upgradeVers++ {
   126  					t.Run(fmt.Sprintf("upgrade-vers=%03d", upgradeVers), func(t *testing.T) {
   127  						// We use vfs.MemFS's option to ignore syncs so
   128  						// that we can perform an upgrade on the current
   129  						// database state in fs, and revert it when this
   130  						// subtest is complete.
   131  						fs.SetIgnoreSyncs(true)
   132  						defer fs.ResetToSyncedState()
   133  
   134  						// Re-open the database, passing a higher format
   135  						// major version in the Options to automatically
   136  						// ratchet the format major version. Ensure some
   137  						// basic operations pass.
   138  						opts := opts.Clone()
   139  						opts.FormatMajorVersion = upgradeVers
   140  						d, err = Open("", opts)
   141  						require.NoError(t, err)
   142  						require.Equal(t, upgradeVers, d.FormatMajorVersion())
   143  						require.NoError(t, testBasicDB(d))
   144  						require.NoError(t, d.Close())
   145  
   146  						// Re-open to ensure the upgrade persisted.
   147  						d, err = Open("", opts)
   148  						require.NoError(t, err)
   149  						require.Equal(t, upgradeVers, d.FormatMajorVersion())
   150  						require.NoError(t, testBasicDB(d))
   151  						require.NoError(t, d.Close())
   152  					})
   153  				}
   154  			})
   155  
   156  			t.Run("upgrade-while-open", func(t *testing.T) {
   157  				for upgradeVers := vers + 1; upgradeVers <= FormatNewest; upgradeVers++ {
   158  					t.Run(fmt.Sprintf("upgrade-vers=%03d", upgradeVers), func(t *testing.T) {
   159  						// Ensure the previous tests don't overwrite our
   160  						// options.
   161  						require.Equal(t, vers, opts.FormatMajorVersion)
   162  
   163  						// We use vfs.MemFS's option to ignore syncs so
   164  						// that we can perform an upgrade on the current
   165  						// database state in fs, and revert it when this
   166  						// subtest is complete.
   167  						fs.SetIgnoreSyncs(true)
   168  						defer fs.ResetToSyncedState()
   169  
   170  						// Re-open the database, still at the current format
   171  						// major version. Perform some basic operations,
   172  						// ratchet the format version up, and perform
   173  						// additional basic operations.
   174  						d, err = Open("", opts)
   175  						require.NoError(t, err)
   176  						require.NoError(t, testBasicDB(d))
   177  						require.Equal(t, vers, d.FormatMajorVersion())
   178  						require.NoError(t, d.RatchetFormatMajorVersion(upgradeVers))
   179  						require.Equal(t, upgradeVers, d.FormatMajorVersion())
   180  						require.NoError(t, testBasicDB(d))
   181  						require.NoError(t, d.Close())
   182  
   183  						// Re-open to ensure the upgrade persisted.
   184  						d, err = Open("", opts)
   185  						require.NoError(t, err)
   186  						require.Equal(t, upgradeVers, d.FormatMajorVersion())
   187  						require.NoError(t, testBasicDB(d))
   188  						require.NoError(t, d.Close())
   189  					})
   190  				}
   191  			})
   192  		})
   193  	}
   194  }
   195  
   196  func TestFormatMajorVersions_TableFormat(t *testing.T) {
   197  	// NB: This test is intended to validate the mapping between every
   198  	// FormatMajorVersion and sstable.TableFormat exhaustively. This serves as a
   199  	// sanity check that new versions have a corresponding mapping. The test
   200  	// fixture is intentionally verbose.
   201  
   202  	m := map[FormatMajorVersion][2]sstable.TableFormat{
   203  		FormatDefault:                      {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2},
   204  		FormatMostCompatible:               {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2},
   205  		formatVersionedManifestMarker:      {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2},
   206  		FormatVersioned:                    {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2},
   207  		FormatSetWithDelete:                {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2},
   208  		FormatBlockPropertyCollector:       {sstable.TableFormatLevelDB, sstable.TableFormatPebblev1},
   209  		FormatSplitUserKeysMarked:          {sstable.TableFormatLevelDB, sstable.TableFormatPebblev1},
   210  		FormatSplitUserKeysMarkedCompacted: {sstable.TableFormatLevelDB, sstable.TableFormatPebblev1},
   211  		FormatRangeKeys:                    {sstable.TableFormatLevelDB, sstable.TableFormatPebblev2},
   212  		FormatMinTableFormatPebblev1:       {sstable.TableFormatPebblev1, sstable.TableFormatPebblev2},
   213  		FormatPrePebblev1Marked:            {sstable.TableFormatPebblev1, sstable.TableFormatPebblev2},
   214  		FormatPrePebblev1MarkedCompacted:   {sstable.TableFormatPebblev1, sstable.TableFormatPebblev2},
   215  	}
   216  
   217  	// Valid versions.
   218  	for fmv := FormatMostCompatible; fmv <= FormatNewest; fmv++ {
   219  		got := [2]sstable.TableFormat{fmv.MinTableFormat(), fmv.MaxTableFormat()}
   220  		require.Equalf(t, m[fmv], got, "got %s; want %s", got, m[fmv])
   221  		require.True(t, got[0] <= got[1] /* min <= max */)
   222  	}
   223  
   224  	// Invalid versions.
   225  	fmv := FormatNewest + 1
   226  	require.Panics(t, func() { _ = fmv.MaxTableFormat() })
   227  	require.Panics(t, func() { _ = fmv.MinTableFormat() })
   228  }
   229  
   230  func TestSplitUserKeyMigration(t *testing.T) {
   231  	var d *DB
   232  	var opts *Options
   233  	var fs vfs.FS
   234  	var buf bytes.Buffer
   235  	defer func() {
   236  		if d != nil {
   237  			require.NoError(t, d.Close())
   238  		}
   239  	}()
   240  
   241  	datadriven.RunTest(t, "testdata/format_major_version_split_user_key_migration",
   242  		func(td *datadriven.TestData) string {
   243  			switch td.Cmd {
   244  			case "define":
   245  				if d != nil {
   246  					if err := d.Close(); err != nil {
   247  						return err.Error()
   248  					}
   249  					buf.Reset()
   250  				}
   251  				opts = &Options{
   252  					FormatMajorVersion: FormatBlockPropertyCollector,
   253  					EventListener: EventListener{
   254  						CompactionEnd: func(info CompactionInfo) {
   255  							// Fix the job ID and durations for determinism.
   256  							info.JobID = 100
   257  							info.Duration = time.Second
   258  							info.TotalDuration = 2 * time.Second
   259  							fmt.Fprintln(&buf, info)
   260  						},
   261  					},
   262  					DisableAutomaticCompactions: true,
   263  				}
   264  				var err error
   265  				if d, err = runDBDefineCmd(td, opts); err != nil {
   266  					return err.Error()
   267  				}
   268  
   269  				fs = d.opts.FS
   270  				d.mu.Lock()
   271  				defer d.mu.Unlock()
   272  				return d.mu.versions.currentVersion().DebugString(base.DefaultFormatter)
   273  			case "reopen":
   274  				if d != nil {
   275  					if err := d.Close(); err != nil {
   276  						return err.Error()
   277  					}
   278  					buf.Reset()
   279  				}
   280  				opts.FS = fs
   281  				opts.DisableAutomaticCompactions = true
   282  				var err error
   283  				d, err = Open("", opts)
   284  				if err != nil {
   285  					return err.Error()
   286  				}
   287  				return "OK"
   288  			case "build":
   289  				if err := runBuildCmd(td, d, fs); err != nil {
   290  					return err.Error()
   291  				}
   292  				return ""
   293  			case "force-ingest":
   294  				if err := runForceIngestCmd(td, d); err != nil {
   295  					return err.Error()
   296  				}
   297  				d.mu.Lock()
   298  				defer d.mu.Unlock()
   299  				return d.mu.versions.currentVersion().DebugString(base.DefaultFormatter)
   300  			case "format-major-version":
   301  				return d.FormatMajorVersion().String()
   302  			case "ratchet-format-major-version":
   303  				v, err := strconv.Atoi(td.CmdArgs[0].String())
   304  				if err != nil {
   305  					return err.Error()
   306  				}
   307  				if err := d.RatchetFormatMajorVersion(FormatMajorVersion(v)); err != nil {
   308  					return err.Error()
   309  				}
   310  				return buf.String()
   311  			case "lsm":
   312  				return runLSMCmd(td, d)
   313  			case "marked-file-count":
   314  				m := d.Metrics()
   315  				return fmt.Sprintf("%d files marked for compaction", m.Compact.MarkedFiles)
   316  			case "disable-automatic-compactions":
   317  				d.mu.Lock()
   318  				defer d.mu.Unlock()
   319  				switch v := td.CmdArgs[0].String(); v {
   320  				case "true":
   321  					d.opts.DisableAutomaticCompactions = true
   322  				case "false":
   323  					d.opts.DisableAutomaticCompactions = false
   324  				default:
   325  					return fmt.Sprintf("unknown value %q", v)
   326  				}
   327  				return ""
   328  			default:
   329  				return fmt.Sprintf("unrecognized command %q", td.Cmd)
   330  			}
   331  		})
   332  }
   333  
   334  func TestPebblev1Migration(t *testing.T) {
   335  	var d *DB
   336  	defer func() {
   337  		if d != nil {
   338  			require.NoError(t, d.Close())
   339  		}
   340  	}()
   341  
   342  	datadriven.RunTest(t, "testdata/format_major_version_bitalostablev1_migration",
   343  		func(td *datadriven.TestData) string {
   344  			switch cmd := td.Cmd; cmd {
   345  			case "open":
   346  				var version int
   347  				var err error
   348  				for _, cmdArg := range td.CmdArgs {
   349  					switch cmd := cmdArg.Key; cmd {
   350  					case "version":
   351  						version, err = strconv.Atoi(cmdArg.Vals[0])
   352  						if err != nil {
   353  							return err.Error()
   354  						}
   355  					default:
   356  						return fmt.Sprintf("unknown argument: %s", cmd)
   357  					}
   358  				}
   359  				opts := &Options{
   360  					FS:                 vfs.NewMem(),
   361  					FormatMajorVersion: FormatMajorVersion(version),
   362  				}
   363  				d, err = Open("", opts)
   364  				if err != nil {
   365  					return err.Error()
   366  				}
   367  				return ""
   368  
   369  			case "format-major-version":
   370  				return d.FormatMajorVersion().String()
   371  
   372  			case "min-table-format":
   373  				return d.FormatMajorVersion().MinTableFormat().String()
   374  
   375  			case "max-table-format":
   376  				return d.FormatMajorVersion().MaxTableFormat().String()
   377  
   378  			case "disable-automatic-compactions":
   379  				d.mu.Lock()
   380  				defer d.mu.Unlock()
   381  				switch v := td.CmdArgs[0].String(); v {
   382  				case "true":
   383  					d.opts.DisableAutomaticCompactions = true
   384  				case "false":
   385  					d.opts.DisableAutomaticCompactions = false
   386  				default:
   387  					return fmt.Sprintf("unknown value %q", v)
   388  				}
   389  				return ""
   390  
   391  			case "batch":
   392  				b := d.NewIndexedBatch()
   393  				if err := runBatchDefineCmd(td, b); err != nil {
   394  					return err.Error()
   395  				}
   396  				if err := b.Commit(nil); err != nil {
   397  					return err.Error()
   398  				}
   399  				return ""
   400  
   401  			case "flush":
   402  				if err := d.Flush(); err != nil {
   403  					return err.Error()
   404  				}
   405  				return ""
   406  
   407  			case "ingest":
   408  				if err := runBuildCmd(td, d, d.opts.FS); err != nil {
   409  					return err.Error()
   410  				}
   411  				if err := runIngestCmd(td, d, d.opts.FS); err != nil {
   412  					return err.Error()
   413  				}
   414  				return ""
   415  
   416  			case "lsm":
   417  				return runLSMCmd(td, d)
   418  
   419  			case "tally-table-formats":
   420  				d.mu.Lock()
   421  				defer d.mu.Unlock()
   422  				v := d.mu.versions.currentVersion()
   423  				tally := make([]int, sstable.TableFormatMax+1)
   424  				for _, l := range v.Levels {
   425  					iter := l.Iter()
   426  					for m := iter.First(); m != nil; m = iter.Next() {
   427  						err := d.tableCache.withReader(m, func(r *sstable.Reader) error {
   428  							f, err := r.TableFormat()
   429  							if err != nil {
   430  								return err
   431  							}
   432  							tally[f]++
   433  							return nil
   434  						})
   435  						if err != nil {
   436  							return err.Error()
   437  						}
   438  					}
   439  				}
   440  				var b bytes.Buffer
   441  				for i := 1; i <= int(sstable.TableFormatMax); i++ {
   442  					_, _ = fmt.Fprintf(&b, "%s: %d\n", sstable.TableFormat(i), tally[i])
   443  				}
   444  				return b.String()
   445  
   446  			case "ratchet-format-major-version":
   447  				v, err := strconv.Atoi(td.CmdArgs[0].String())
   448  				if err != nil {
   449  					return err.Error()
   450  				}
   451  				if err = d.RatchetFormatMajorVersion(FormatMajorVersion(v)); err != nil {
   452  					return err.Error()
   453  				}
   454  				return ""
   455  
   456  			case "marked-file-count":
   457  				m := d.Metrics()
   458  				return fmt.Sprintf("%d files marked for compaction", m.Compact.MarkedFiles)
   459  
   460  			default:
   461  				return fmt.Sprintf("unknown command: %s", cmd)
   462  			}
   463  		},
   464  	)
   465  }