github.com/cockroachdb/pebble@v1.1.1-0.20240513155919-3622ade60459/format_major_version_test.go (about)

     1  // Copyright 2021 The LevelDB-Go and Pebble Authors. All rights reserved. Use
     2  // of this source code is governed by a BSD-style license that can be found in
     3  // the LICENSE file.
     4  
     5  package pebble
     6  
     7  import (
     8  	"bytes"
     9  	"fmt"
    10  	"strconv"
    11  	"sync"
    12  	"testing"
    13  	"time"
    14  
    15  	"github.com/cockroachdb/datadriven"
    16  	"github.com/cockroachdb/pebble/bloom"
    17  	"github.com/cockroachdb/pebble/internal/base"
    18  	"github.com/cockroachdb/pebble/internal/testkeys"
    19  	"github.com/cockroachdb/pebble/sstable"
    20  	"github.com/cockroachdb/pebble/vfs"
    21  	"github.com/cockroachdb/pebble/vfs/atomicfs"
    22  	"github.com/stretchr/testify/require"
    23  )
    24  
    25  func TestFormatMajorVersion_MigrationDefined(t *testing.T) {
    26  	for v := FormatMostCompatible; v <= FormatNewest; v++ {
    27  		if _, ok := formatMajorVersionMigrations[v]; !ok {
    28  			t.Errorf("format major version %d has no migration defined", v)
    29  		}
    30  	}
    31  }
    32  
    33  func TestRatchetFormat(t *testing.T) {
    34  	fs := vfs.NewMem()
    35  	d, err := Open("", (&Options{FS: fs}).WithFSDefaults())
    36  	require.NoError(t, err)
    37  	require.NoError(t, d.Set([]byte("foo"), []byte("bar"), Sync))
    38  	require.Equal(t, FormatMostCompatible, d.FormatMajorVersion())
    39  	require.NoError(t, d.RatchetFormatMajorVersion(FormatVersioned))
    40  	require.Equal(t, FormatVersioned, d.FormatMajorVersion())
    41  	require.NoError(t, d.RatchetFormatMajorVersion(FormatVersioned))
    42  	require.Equal(t, FormatVersioned, d.FormatMajorVersion())
    43  	require.NoError(t, d.RatchetFormatMajorVersion(FormatSetWithDelete))
    44  	require.Equal(t, FormatSetWithDelete, d.FormatMajorVersion())
    45  	require.NoError(t, d.RatchetFormatMajorVersion(FormatBlockPropertyCollector))
    46  	require.Equal(t, FormatBlockPropertyCollector, d.FormatMajorVersion())
    47  	require.NoError(t, d.RatchetFormatMajorVersion(FormatSplitUserKeysMarked))
    48  	require.Equal(t, FormatSplitUserKeysMarked, d.FormatMajorVersion())
    49  	require.NoError(t, d.RatchetFormatMajorVersion(FormatSplitUserKeysMarkedCompacted))
    50  	require.Equal(t, FormatSplitUserKeysMarkedCompacted, d.FormatMajorVersion())
    51  	require.NoError(t, d.RatchetFormatMajorVersion(FormatRangeKeys))
    52  	require.Equal(t, FormatRangeKeys, d.FormatMajorVersion())
    53  	require.NoError(t, d.RatchetFormatMajorVersion(FormatMinTableFormatPebblev1))
    54  	require.Equal(t, FormatMinTableFormatPebblev1, d.FormatMajorVersion())
    55  	require.NoError(t, d.RatchetFormatMajorVersion(FormatPrePebblev1Marked))
    56  	require.Equal(t, FormatPrePebblev1Marked, d.FormatMajorVersion())
    57  	require.NoError(t, d.RatchetFormatMajorVersion(formatUnusedPrePebblev1MarkedCompacted))
    58  	require.Equal(t, formatUnusedPrePebblev1MarkedCompacted, d.FormatMajorVersion())
    59  	require.NoError(t, d.RatchetFormatMajorVersion(FormatSSTableValueBlocks))
    60  	require.Equal(t, FormatSSTableValueBlocks, d.FormatMajorVersion())
    61  	require.NoError(t, d.RatchetFormatMajorVersion(FormatFlushableIngest))
    62  	require.Equal(t, FormatFlushableIngest, d.FormatMajorVersion())
    63  	require.NoError(t, d.RatchetFormatMajorVersion(FormatPrePebblev1MarkedCompacted))
    64  	require.Equal(t, FormatPrePebblev1MarkedCompacted, d.FormatMajorVersion())
    65  	require.NoError(t, d.RatchetFormatMajorVersion(FormatDeleteSizedAndObsolete))
    66  	require.Equal(t, FormatDeleteSizedAndObsolete, d.FormatMajorVersion())
    67  	require.NoError(t, d.RatchetFormatMajorVersion(FormatVirtualSSTables))
    68  	require.Equal(t, FormatVirtualSSTables, d.FormatMajorVersion())
    69  
    70  	require.NoError(t, d.Close())
    71  
    72  	// If we Open the database again, leaving the default format, the
    73  	// database should Open using the persisted FormatNewest.
    74  	d, err = Open("", (&Options{FS: fs}).WithFSDefaults())
    75  	require.NoError(t, err)
    76  	require.Equal(t, internalFormatNewest, d.FormatMajorVersion())
    77  	require.NoError(t, d.Close())
    78  
    79  	// Move the marker to a version that does not exist.
    80  	m, _, err := atomicfs.LocateMarker(fs, "", formatVersionMarkerName)
    81  	require.NoError(t, err)
    82  	require.NoError(t, m.Move("999999"))
    83  	require.NoError(t, m.Close())
    84  
    85  	_, err = Open("", (&Options{
    86  		FS:                 fs,
    87  		FormatMajorVersion: FormatVersioned,
    88  	}).WithFSDefaults())
    89  	require.Error(t, err)
    90  	require.EqualError(t, err, `pebble: database "" written in format major version 999999`)
    91  }
    92  
    93  func testBasicDB(d *DB) error {
    94  	key := []byte("a")
    95  	value := []byte("b")
    96  	if err := d.Set(key, value, nil); err != nil {
    97  		return err
    98  	}
    99  	if err := d.Flush(); err != nil {
   100  		return err
   101  	}
   102  	if err := d.Compact(nil, []byte("\xff"), false); err != nil {
   103  		return err
   104  	}
   105  
   106  	iter, _ := d.NewIter(nil)
   107  	for valid := iter.First(); valid; valid = iter.Next() {
   108  	}
   109  	if err := iter.Close(); err != nil {
   110  		return err
   111  	}
   112  	return nil
   113  }
   114  
   115  func TestFormatMajorVersions(t *testing.T) {
   116  	for vers := FormatMostCompatible; vers <= FormatNewest; vers++ {
   117  		t.Run(fmt.Sprintf("vers=%03d", vers), func(t *testing.T) {
   118  			fs := vfs.NewStrictMem()
   119  			opts := (&Options{
   120  				FS:                 fs,
   121  				FormatMajorVersion: vers,
   122  			}).WithFSDefaults()
   123  
   124  			// Create a database at this format major version and perform
   125  			// some very basic operations.
   126  			d, err := Open("", opts)
   127  			require.NoError(t, err)
   128  			require.NoError(t, testBasicDB(d))
   129  			require.NoError(t, d.Close())
   130  
   131  			// Re-open the database at this format major version, and again
   132  			// perform some basic operations.
   133  			d, err = Open("", opts)
   134  			require.NoError(t, err)
   135  			require.NoError(t, testBasicDB(d))
   136  			require.NoError(t, d.Close())
   137  
   138  			t.Run("upgrade-at-open", func(t *testing.T) {
   139  				for upgradeVers := vers + 1; upgradeVers <= FormatNewest; upgradeVers++ {
   140  					t.Run(fmt.Sprintf("upgrade-vers=%03d", upgradeVers), func(t *testing.T) {
   141  						// We use vfs.MemFS's option to ignore syncs so
   142  						// that we can perform an upgrade on the current
   143  						// database state in fs, and revert it when this
   144  						// subtest is complete.
   145  						fs.SetIgnoreSyncs(true)
   146  						defer fs.ResetToSyncedState()
   147  
   148  						// Re-open the database, passing a higher format
   149  						// major version in the Options to automatically
   150  						// ratchet the format major version. Ensure some
   151  						// basic operations pass.
   152  						opts := opts.Clone()
   153  						opts.FormatMajorVersion = upgradeVers
   154  						d, err = Open("", opts)
   155  						require.NoError(t, err)
   156  						require.Equal(t, upgradeVers, d.FormatMajorVersion())
   157  						require.NoError(t, testBasicDB(d))
   158  						require.NoError(t, d.Close())
   159  
   160  						// Re-open to ensure the upgrade persisted.
   161  						d, err = Open("", opts)
   162  						require.NoError(t, err)
   163  						require.Equal(t, upgradeVers, d.FormatMajorVersion())
   164  						require.NoError(t, testBasicDB(d))
   165  						require.NoError(t, d.Close())
   166  					})
   167  				}
   168  			})
   169  
   170  			t.Run("upgrade-while-open", func(t *testing.T) {
   171  				for upgradeVers := vers + 1; upgradeVers <= FormatNewest; upgradeVers++ {
   172  					t.Run(fmt.Sprintf("upgrade-vers=%03d", upgradeVers), func(t *testing.T) {
   173  						// Ensure the previous tests don't overwrite our
   174  						// options.
   175  						require.Equal(t, vers, opts.FormatMajorVersion)
   176  
   177  						// We use vfs.MemFS's option to ignore syncs so
   178  						// that we can perform an upgrade on the current
   179  						// database state in fs, and revert it when this
   180  						// subtest is complete.
   181  						fs.SetIgnoreSyncs(true)
   182  						defer fs.ResetToSyncedState()
   183  
   184  						// Re-open the database, still at the current format
   185  						// major version. Perform some basic operations,
   186  						// ratchet the format version up, and perform
   187  						// additional basic operations.
   188  						d, err = Open("", opts)
   189  						require.NoError(t, err)
   190  						require.NoError(t, testBasicDB(d))
   191  						require.Equal(t, vers, d.FormatMajorVersion())
   192  						require.NoError(t, d.RatchetFormatMajorVersion(upgradeVers))
   193  						require.Equal(t, upgradeVers, d.FormatMajorVersion())
   194  						require.NoError(t, testBasicDB(d))
   195  						require.NoError(t, d.Close())
   196  
   197  						// Re-open to ensure the upgrade persisted.
   198  						d, err = Open("", opts)
   199  						require.NoError(t, err)
   200  						require.Equal(t, upgradeVers, d.FormatMajorVersion())
   201  						require.NoError(t, testBasicDB(d))
   202  						require.NoError(t, d.Close())
   203  					})
   204  				}
   205  			})
   206  		})
   207  	}
   208  }
   209  
   210  func TestFormatMajorVersions_TableFormat(t *testing.T) {
   211  	// NB: This test is intended to validate the mapping between every
   212  	// FormatMajorVersion and sstable.TableFormat exhaustively. This serves as a
   213  	// sanity check that new versions have a corresponding mapping. The test
   214  	// fixture is intentionally verbose.
   215  
   216  	m := map[FormatMajorVersion][2]sstable.TableFormat{
   217  		FormatDefault:                          {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2},
   218  		FormatMostCompatible:                   {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2},
   219  		formatVersionedManifestMarker:          {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2},
   220  		FormatVersioned:                        {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2},
   221  		FormatSetWithDelete:                    {sstable.TableFormatLevelDB, sstable.TableFormatRocksDBv2},
   222  		FormatBlockPropertyCollector:           {sstable.TableFormatLevelDB, sstable.TableFormatPebblev1},
   223  		FormatSplitUserKeysMarked:              {sstable.TableFormatLevelDB, sstable.TableFormatPebblev1},
   224  		FormatSplitUserKeysMarkedCompacted:     {sstable.TableFormatLevelDB, sstable.TableFormatPebblev1},
   225  		FormatRangeKeys:                        {sstable.TableFormatLevelDB, sstable.TableFormatPebblev2},
   226  		FormatMinTableFormatPebblev1:           {sstable.TableFormatPebblev1, sstable.TableFormatPebblev2},
   227  		FormatPrePebblev1Marked:                {sstable.TableFormatPebblev1, sstable.TableFormatPebblev2},
   228  		formatUnusedPrePebblev1MarkedCompacted: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev2},
   229  		FormatSSTableValueBlocks:               {sstable.TableFormatPebblev1, sstable.TableFormatPebblev3},
   230  		FormatFlushableIngest:                  {sstable.TableFormatPebblev1, sstable.TableFormatPebblev3},
   231  		FormatPrePebblev1MarkedCompacted:       {sstable.TableFormatPebblev1, sstable.TableFormatPebblev3},
   232  		FormatDeleteSizedAndObsolete:           {sstable.TableFormatPebblev1, sstable.TableFormatPebblev4},
   233  		FormatVirtualSSTables:                  {sstable.TableFormatPebblev1, sstable.TableFormatPebblev4},
   234  	}
   235  
   236  	// Valid versions.
   237  	for fmv := FormatMostCompatible; fmv <= internalFormatNewest; fmv++ {
   238  		got := [2]sstable.TableFormat{fmv.MinTableFormat(), fmv.MaxTableFormat()}
   239  		require.Equalf(t, m[fmv], got, "got %s; want %s", got, m[fmv])
   240  		require.True(t, got[0] <= got[1] /* min <= max */)
   241  	}
   242  
   243  	// Invalid versions.
   244  	fmv := internalFormatNewest + 1
   245  	require.Panics(t, func() { _ = fmv.MaxTableFormat() })
   246  	require.Panics(t, func() { _ = fmv.MinTableFormat() })
   247  }
   248  
   249  func TestSplitUserKeyMigration(t *testing.T) {
   250  	var d *DB
   251  	var opts *Options
   252  	var fs vfs.FS
   253  	var buf bytes.Buffer
   254  	defer func() {
   255  		if d != nil {
   256  			require.NoError(t, d.Close())
   257  		}
   258  	}()
   259  
   260  	datadriven.RunTest(t, "testdata/format_major_version_split_user_key_migration",
   261  		func(t *testing.T, td *datadriven.TestData) string {
   262  			switch td.Cmd {
   263  			case "define":
   264  				if d != nil {
   265  					if err := d.Close(); err != nil {
   266  						return err.Error()
   267  					}
   268  					buf.Reset()
   269  				}
   270  				opts = (&Options{
   271  					FormatMajorVersion: FormatBlockPropertyCollector,
   272  					EventListener: &EventListener{
   273  						CompactionEnd: func(info CompactionInfo) {
   274  							// Fix the job ID and durations for determinism.
   275  							info.JobID = 100
   276  							info.Duration = time.Second
   277  							info.TotalDuration = 2 * time.Second
   278  							fmt.Fprintln(&buf, info)
   279  						},
   280  					},
   281  					DisableAutomaticCompactions: true,
   282  				}).WithFSDefaults()
   283  				var err error
   284  				if d, err = runDBDefineCmd(td, opts); err != nil {
   285  					return err.Error()
   286  				}
   287  
   288  				fs = d.opts.FS
   289  				d.mu.Lock()
   290  				defer d.mu.Unlock()
   291  				return d.mu.versions.currentVersion().DebugString(base.DefaultFormatter)
   292  			case "reopen":
   293  				if d != nil {
   294  					if err := d.Close(); err != nil {
   295  						return err.Error()
   296  					}
   297  					buf.Reset()
   298  				}
   299  				opts.FS = fs
   300  				opts.DisableAutomaticCompactions = true
   301  				var err error
   302  				d, err = Open("", opts)
   303  				if err != nil {
   304  					return err.Error()
   305  				}
   306  				return "OK"
   307  			case "build":
   308  				if err := runBuildCmd(td, d, fs); err != nil {
   309  					return err.Error()
   310  				}
   311  				return ""
   312  			case "force-ingest":
   313  				if err := runForceIngestCmd(td, d); err != nil {
   314  					return err.Error()
   315  				}
   316  				d.mu.Lock()
   317  				defer d.mu.Unlock()
   318  				return d.mu.versions.currentVersion().DebugString(base.DefaultFormatter)
   319  			case "format-major-version":
   320  				return d.FormatMajorVersion().String()
   321  			case "ratchet-format-major-version":
   322  				v, err := strconv.Atoi(td.CmdArgs[0].String())
   323  				if err != nil {
   324  					return err.Error()
   325  				}
   326  				if err := d.RatchetFormatMajorVersion(FormatMajorVersion(v)); err != nil {
   327  					return err.Error()
   328  				}
   329  				return buf.String()
   330  			case "lsm":
   331  				return runLSMCmd(td, d)
   332  			case "marked-file-count":
   333  				m := d.Metrics()
   334  				return fmt.Sprintf("%d files marked for compaction", m.Compact.MarkedFiles)
   335  			case "disable-automatic-compactions":
   336  				d.mu.Lock()
   337  				defer d.mu.Unlock()
   338  				switch v := td.CmdArgs[0].String(); v {
   339  				case "true":
   340  					d.opts.DisableAutomaticCompactions = true
   341  				case "false":
   342  					d.opts.DisableAutomaticCompactions = false
   343  				default:
   344  					return fmt.Sprintf("unknown value %q", v)
   345  				}
   346  				return ""
   347  			default:
   348  				return fmt.Sprintf("unrecognized command %q", td.Cmd)
   349  			}
   350  		})
   351  }
   352  
   353  func TestPebblev1Migration(t *testing.T) {
   354  	var d *DB
   355  	defer func() {
   356  		if d != nil {
   357  			require.NoError(t, d.Close())
   358  		}
   359  	}()
   360  
   361  	datadriven.RunTest(t, "testdata/format_major_version_pebblev1_migration",
   362  		func(t *testing.T, td *datadriven.TestData) string {
   363  			switch cmd := td.Cmd; cmd {
   364  			case "open":
   365  				var version int
   366  				var err error
   367  				for _, cmdArg := range td.CmdArgs {
   368  					switch cmd := cmdArg.Key; cmd {
   369  					case "version":
   370  						version, err = strconv.Atoi(cmdArg.Vals[0])
   371  						if err != nil {
   372  							return err.Error()
   373  						}
   374  					default:
   375  						return fmt.Sprintf("unknown argument: %s", cmd)
   376  					}
   377  				}
   378  				opts := (&Options{
   379  					FS:                 vfs.NewMem(),
   380  					FormatMajorVersion: FormatMajorVersion(version),
   381  				}).WithFSDefaults()
   382  				d, err = Open("", opts)
   383  				if err != nil {
   384  					return err.Error()
   385  				}
   386  				return ""
   387  
   388  			case "format-major-version":
   389  				return d.FormatMajorVersion().String()
   390  
   391  			case "min-table-format":
   392  				return d.FormatMajorVersion().MinTableFormat().String()
   393  
   394  			case "max-table-format":
   395  				return d.FormatMajorVersion().MaxTableFormat().String()
   396  
   397  			case "disable-automatic-compactions":
   398  				d.mu.Lock()
   399  				defer d.mu.Unlock()
   400  				switch v := td.CmdArgs[0].String(); v {
   401  				case "true":
   402  					d.opts.DisableAutomaticCompactions = true
   403  				case "false":
   404  					d.opts.DisableAutomaticCompactions = false
   405  				default:
   406  					return fmt.Sprintf("unknown value %q", v)
   407  				}
   408  				return ""
   409  
   410  			case "batch":
   411  				b := d.NewIndexedBatch()
   412  				if err := runBatchDefineCmd(td, b); err != nil {
   413  					return err.Error()
   414  				}
   415  				if err := b.Commit(nil); err != nil {
   416  					return err.Error()
   417  				}
   418  				return ""
   419  
   420  			case "flush":
   421  				if err := d.Flush(); err != nil {
   422  					return err.Error()
   423  				}
   424  				return ""
   425  
   426  			case "ingest":
   427  				if err := runBuildCmd(td, d, d.opts.FS); err != nil {
   428  					return err.Error()
   429  				}
   430  				// Only the first arg is a filename.
   431  				td.CmdArgs = td.CmdArgs[:1]
   432  				if err := runIngestCmd(td, d, d.opts.FS); err != nil {
   433  					return err.Error()
   434  				}
   435  				return ""
   436  
   437  			case "lsm":
   438  				return runLSMCmd(td, d)
   439  
   440  			case "tally-table-formats":
   441  				d.mu.Lock()
   442  				defer d.mu.Unlock()
   443  				v := d.mu.versions.currentVersion()
   444  				tally := make([]int, sstable.TableFormatMax+1)
   445  				for _, l := range v.Levels {
   446  					iter := l.Iter()
   447  					for m := iter.First(); m != nil; m = iter.Next() {
   448  						err := d.tableCache.withReader(m.PhysicalMeta(),
   449  							func(r *sstable.Reader) error {
   450  								f, err := r.TableFormat()
   451  								if err != nil {
   452  									return err
   453  								}
   454  								tally[f]++
   455  								return nil
   456  							})
   457  						if err != nil {
   458  							return err.Error()
   459  						}
   460  					}
   461  				}
   462  				var b bytes.Buffer
   463  				for i := 1; i <= int(sstable.TableFormatMax); i++ {
   464  					_, _ = fmt.Fprintf(&b, "%s: %d\n", sstable.TableFormat(i), tally[i])
   465  				}
   466  				return b.String()
   467  
   468  			case "ratchet-format-major-version":
   469  				v, err := strconv.Atoi(td.CmdArgs[0].String())
   470  				if err != nil {
   471  					return err.Error()
   472  				}
   473  				if err = d.RatchetFormatMajorVersion(FormatMajorVersion(v)); err != nil {
   474  					return err.Error()
   475  				}
   476  				return ""
   477  
   478  			case "marked-file-count":
   479  				m := d.Metrics()
   480  				return fmt.Sprintf("%d files marked for compaction", m.Compact.MarkedFiles)
   481  
   482  			default:
   483  				return fmt.Sprintf("unknown command: %s", cmd)
   484  			}
   485  		},
   486  	)
   487  }
   488  
   489  // TestPebblev1MigrationRace exercises the race between a PrePebbleV1Marked
   490  // format major version upgrade that needs to open sstables to read their table
   491  // format, and concurrent compactions that may delete the same files from the
   492  // LSM.
   493  //
   494  // Regression test for #2019.
   495  func TestPebblev1MigrationRace(t *testing.T) {
   496  	// Use a smaller table cache size to slow down the PrePebbleV1Marked
   497  	// migration, ensuring each table read needs to re-open the file.
   498  	cache := NewCache(4 << 20)
   499  	defer cache.Unref()
   500  	tableCache := NewTableCache(cache, 1, 5)
   501  	defer tableCache.Unref()
   502  	d, err := Open("", (&Options{
   503  		Cache:              cache,
   504  		FS:                 vfs.NewMem(),
   505  		FormatMajorVersion: FormatMajorVersion(FormatPrePebblev1Marked - 1),
   506  		TableCache:         tableCache,
   507  		Levels:             []LevelOptions{{TargetFileSize: 1}},
   508  	}).WithFSDefaults())
   509  	require.NoError(t, err)
   510  	defer d.Close()
   511  
   512  	ks := testkeys.Alpha(3).EveryN(10)
   513  	var key [3]byte
   514  	for i := int64(0); i < ks.Count(); i++ {
   515  		n := testkeys.WriteKey(key[:], ks, i)
   516  		require.NoError(t, d.Set(key[:n], key[:n], nil))
   517  		require.NoError(t, d.Flush())
   518  	}
   519  
   520  	// Asynchronously write and flush range deletes that will cause compactions
   521  	// to delete the existing sstables. These deletes will race with the format
   522  	// major version upgrade's migration will attempt to delete the files.
   523  	var wg sync.WaitGroup
   524  	wg.Add(1)
   525  	go func() {
   526  		defer wg.Done()
   527  		for i := ks.Count() - 1; i > 0; i -= 50 {
   528  			endKey := testkeys.Key(ks, i)
   529  			startIndex := i - 50
   530  			if startIndex < 0 {
   531  				startIndex = 0
   532  			}
   533  			startKey := testkeys.Key(ks, startIndex)
   534  
   535  			require.NoError(t, d.DeleteRange(startKey, endKey, nil))
   536  			_, err := d.AsyncFlush()
   537  			require.NoError(t, err)
   538  		}
   539  	}()
   540  	require.NoError(t, d.RatchetFormatMajorVersion(FormatPrePebblev1Marked))
   541  	wg.Wait()
   542  }
   543  
   544  // Regression test for #2044, where multiple concurrent compactions can lead
   545  // to an indefinite wait on the compaction goroutine in compactMarkedFilesLocked.
   546  func TestPebblev1MigrationConcurrencyRace(t *testing.T) {
   547  	opts := (&Options{
   548  		Comparer:           testkeys.Comparer,
   549  		FS:                 vfs.NewMem(),
   550  		FormatMajorVersion: FormatSplitUserKeysMarked,
   551  		Levels:             []LevelOptions{{FilterPolicy: bloom.FilterPolicy(10)}},
   552  		MaxConcurrentCompactions: func() int {
   553  			return 4
   554  		},
   555  	}).WithFSDefaults()
   556  	func() {
   557  		d, err := Open("", opts)
   558  		require.NoError(t, err)
   559  		defer func() {
   560  			require.NoError(t, d.Close())
   561  		}()
   562  
   563  		ks := testkeys.Alpha(3).EveryN(10)
   564  		var key [3]byte
   565  		for i := int64(0); i < ks.Count(); i++ {
   566  			n := testkeys.WriteKey(key[:], ks, i)
   567  			require.NoError(t, d.Set(key[:n], key[:n], nil))
   568  			if i%100 == 0 {
   569  				require.NoError(t, d.Flush())
   570  			}
   571  		}
   572  		require.NoError(t, d.Flush())
   573  	}()
   574  
   575  	opts.FormatMajorVersion = formatUnusedPrePebblev1MarkedCompacted
   576  	d, err := Open("", opts)
   577  	require.NoError(t, err)
   578  	require.NoError(t, d.RatchetFormatMajorVersion(formatUnusedPrePebblev1MarkedCompacted))
   579  	require.NoError(t, d.Close())
   580  }