github.com/cockroachdb/pebble@v1.1.1-0.20240513155919-3622ade60459/flushable_test.go (about)

     1  package pebble
     2  
     3  import (
     4  	"bytes"
     5  	"fmt"
     6  	"testing"
     7  
     8  	"github.com/cockroachdb/datadriven"
     9  	"github.com/cockroachdb/pebble/internal/base"
    10  	"github.com/cockroachdb/pebble/vfs"
    11  	"github.com/stretchr/testify/require"
    12  )
    13  
    14  // Simple sanity tests for the flushable interface implementation for ingested
    15  // sstables.
    16  func TestIngestedSSTFlushableAPI(t *testing.T) {
    17  	var mem vfs.FS
    18  	var d *DB
    19  	defer func() {
    20  		require.NoError(t, d.Close())
    21  	}()
    22  	var flushable flushable
    23  
    24  	reset := func() {
    25  		if d != nil {
    26  			require.NoError(t, d.Close())
    27  		}
    28  
    29  		mem = vfs.NewMem()
    30  		require.NoError(t, mem.MkdirAll("ext", 0755))
    31  		opts := &Options{
    32  			FS:                    mem,
    33  			L0CompactionThreshold: 100,
    34  			L0StopWritesThreshold: 100,
    35  			DebugCheck:            DebugCheckLevels,
    36  			FormatMajorVersion:    internalFormatNewest,
    37  		}
    38  		// Disable automatic compactions because otherwise we'll race with
    39  		// delete-only compactions triggered by ingesting range tombstones.
    40  		opts.DisableAutomaticCompactions = true
    41  
    42  		var err error
    43  		d, err = Open("", opts)
    44  		require.NoError(t, err)
    45  		flushable = nil
    46  	}
    47  	reset()
    48  
    49  	loadFileMeta := func(paths []string) []*fileMetadata {
    50  		d.mu.Lock()
    51  		pendingOutputs := make([]base.DiskFileNum, len(paths))
    52  		for i := range paths {
    53  			pendingOutputs[i] = d.mu.versions.getNextFileNum().DiskFileNum()
    54  		}
    55  		jobID := d.mu.nextJobID
    56  		d.mu.nextJobID++
    57  		d.mu.Unlock()
    58  
    59  		// We can reuse the ingestLoad function for this test even if we're
    60  		// not actually ingesting a file.
    61  		lr, err := ingestLoad(d.opts, d.FormatMajorVersion(), paths, nil, nil, d.cacheID, pendingOutputs, d.objProvider, jobID)
    62  		if err != nil {
    63  			panic(err)
    64  		}
    65  		meta := lr.localMeta
    66  		if len(meta) == 0 {
    67  			// All of the sstables to be ingested were empty. Nothing to do.
    68  			panic("empty sstable")
    69  		}
    70  		// The table cache requires the *fileMetadata to have a positive
    71  		// reference count. Fake a reference before we try to load the file.
    72  		for _, f := range meta {
    73  			f.Ref()
    74  		}
    75  
    76  		// Verify the sstables do not overlap.
    77  		if err := ingestSortAndVerify(d.cmp, lr, KeyRange{}); err != nil {
    78  			panic("unsorted sstables")
    79  		}
    80  
    81  		// Hard link the sstables into the DB directory. Since the sstables aren't
    82  		// referenced by a version, they won't be used. If the hard linking fails
    83  		// (e.g. because the files reside on a different filesystem), ingestLink will
    84  		// fall back to copying, and if that fails we undo our work and return an
    85  		// error.
    86  		if err := ingestLink(jobID, d.opts, d.objProvider, lr, nil /* shared */); err != nil {
    87  			panic("couldn't hard link sstables")
    88  		}
    89  
    90  		// Fsync the directory we added the tables to. We need to do this at some
    91  		// point before we update the MANIFEST (via logAndApply), otherwise a crash
    92  		// can have the tables referenced in the MANIFEST, but not present in the
    93  		// directory.
    94  		if err := d.dataDir.Sync(); err != nil {
    95  			panic("Couldn't sync data directory")
    96  		}
    97  
    98  		return meta
    99  	}
   100  
   101  	datadriven.RunTest(t, "testdata/ingested_flushable_api", func(t *testing.T, td *datadriven.TestData) string {
   102  		switch td.Cmd {
   103  		case "reset":
   104  			reset()
   105  			return ""
   106  		case "build":
   107  			if err := runBuildCmd(td, d, mem); err != nil {
   108  				return err.Error()
   109  			}
   110  			return ""
   111  		case "flushable":
   112  			// Creates an ingestedFlushable over the input files.
   113  			paths := make([]string, 0, len(td.CmdArgs))
   114  			for _, arg := range td.CmdArgs {
   115  				paths = append(paths, arg.String())
   116  			}
   117  
   118  			meta := loadFileMeta(paths)
   119  			flushable = newIngestedFlushable(
   120  				meta, d.opts.Comparer, d.newIters, d.tableNewRangeKeyIter,
   121  			)
   122  			return ""
   123  		case "iter":
   124  			iter := flushable.newIter(nil)
   125  			var buf bytes.Buffer
   126  			for x, _ := iter.First(); x != nil; x, _ = iter.Next() {
   127  				buf.WriteString(x.String())
   128  				buf.WriteString("\n")
   129  			}
   130  			iter.Close()
   131  			return buf.String()
   132  		case "rangekeyIter":
   133  			iter := flushable.newRangeKeyIter(nil)
   134  			var buf bytes.Buffer
   135  			if iter != nil {
   136  				for span := iter.First(); span != nil; span = iter.Next() {
   137  					buf.WriteString(span.String())
   138  					buf.WriteString("\n")
   139  				}
   140  				iter.Close()
   141  			}
   142  			return buf.String()
   143  		case "rangedelIter":
   144  			iter := flushable.newRangeDelIter(nil)
   145  			var buf bytes.Buffer
   146  			if iter != nil {
   147  				for span := iter.First(); span != nil; span = iter.Next() {
   148  					buf.WriteString(span.String())
   149  					buf.WriteString("\n")
   150  				}
   151  				iter.Close()
   152  			}
   153  			return buf.String()
   154  		case "readyForFlush":
   155  			if flushable.readyForFlush() {
   156  				return "true"
   157  			}
   158  			return "false"
   159  		case "containsRangeKey":
   160  			if flushable.containsRangeKeys() {
   161  				return "true"
   162  			}
   163  			return "false"
   164  		default:
   165  			return fmt.Sprintf("unknown command: %s", td.Cmd)
   166  		}
   167  	})
   168  }