github.com/zuoyebang/bitalostable@v1.0.1-0.20240229032404-e3b99a834294/error_test.go (about)

     1  // Copyright 2019 The LevelDB-Go and Pebble and Bitalostored Authors. All rights reserved. Use
     2  // of this source code is governed by a BSD-style license that can be found in
     3  // the LICENSE file.
     4  
     5  package bitalostable
     6  
     7  import (
     8  	"bytes"
     9  	"fmt"
    10  	"math"
    11  	"strings"
    12  	"sync/atomic"
    13  	"testing"
    14  
    15  	"github.com/cockroachdb/errors"
    16  	"github.com/stretchr/testify/require"
    17  	"github.com/zuoyebang/bitalostable/internal/errorfs"
    18  	"github.com/zuoyebang/bitalostable/vfs"
    19  )
    20  
    21  type panicLogger struct{}
    22  
    23  func (l panicLogger) Cost(arg ...interface{}) func() {
    24  	return func() {}
    25  }
    26  func (l panicLogger) Info(args ...interface{}) {
    27  }
    28  func (l panicLogger) Warn(args ...interface{}) {
    29  }
    30  func (l panicLogger) Error(args ...interface{}) {
    31  }
    32  func (l panicLogger) Infof(format string, args ...interface{}) {
    33  }
    34  func (l panicLogger) Warnf(format string, args ...interface{}) {
    35  }
    36  func (l panicLogger) Errorf(format string, args ...interface{}) {
    37  }
    38  func (l panicLogger) Fatalf(format string, args ...interface{}) {
    39  	panic(errors.Errorf("fatal: "+format, args...))
    40  }
    41  
    42  // corruptFS injects a corruption in the `index`th byte read.
    43  type corruptFS struct {
    44  	vfs.FS
    45  	index     int32
    46  	bytesRead int32
    47  }
    48  
    49  func (fs corruptFS) maybeCorrupt(n int32, p []byte) {
    50  	newBytesRead := atomic.AddInt32(&fs.bytesRead, n)
    51  	pIdx := newBytesRead - 1 - fs.index
    52  	if pIdx >= 0 && pIdx < n {
    53  		p[pIdx]++
    54  	}
    55  }
    56  
    57  func (fs *corruptFS) Open(name string, opts ...vfs.OpenOption) (vfs.File, error) {
    58  	f, err := fs.FS.Open(name)
    59  	if err != nil {
    60  		return nil, err
    61  	}
    62  	cf := corruptFile{f, fs}
    63  	for _, opt := range opts {
    64  		opt.Apply(cf)
    65  	}
    66  	return cf, nil
    67  }
    68  
    69  type corruptFile struct {
    70  	vfs.File
    71  	fs *corruptFS
    72  }
    73  
    74  func (f corruptFile) Read(p []byte) (int, error) {
    75  	n, err := f.File.Read(p)
    76  	f.fs.maybeCorrupt(int32(n), p)
    77  	return n, err
    78  }
    79  
    80  func (f corruptFile) ReadAt(p []byte, off int64) (int, error) {
    81  	n, err := f.File.ReadAt(p, off)
    82  	f.fs.maybeCorrupt(int32(n), p)
    83  	return n, err
    84  }
    85  
    86  func expectLSM(expected string, d *DB, t *testing.T) {
    87  	t.Helper()
    88  	expected = strings.TrimSpace(expected)
    89  	d.mu.Lock()
    90  	actual := d.mu.versions.currentVersion().String()
    91  	d.mu.Unlock()
    92  	actual = strings.TrimSpace(actual)
    93  	if expected != actual {
    94  		t.Fatalf("expected\n%s\nbut found\n%s", expected, actual)
    95  	}
    96  }
    97  
    98  // TestErrors repeatedly runs a short sequence of operations, injecting FS
    99  // errors at different points, until success is achieved.
   100  func TestErrors(t *testing.T) {
   101  	run := func(fs *errorfs.FS) (err error) {
   102  		defer func() {
   103  			if r := recover(); r != nil {
   104  				if e, ok := r.(error); ok {
   105  					err = e
   106  				} else {
   107  					t.Fatal(r)
   108  				}
   109  			}
   110  		}()
   111  
   112  		d, err := Open("", &Options{
   113  			FS:     fs,
   114  			Logger: panicLogger{},
   115  		})
   116  		if err != nil {
   117  			return err
   118  		}
   119  
   120  		key := []byte("a")
   121  		value := []byte("b")
   122  		if err := d.Set(key, value, nil); err != nil {
   123  			return err
   124  		}
   125  		if err := d.Flush(); err != nil {
   126  			return err
   127  		}
   128  		if err := d.Compact(nil, []byte("\xff"), false); err != nil {
   129  			return err
   130  		}
   131  
   132  		iter := d.NewIter(nil)
   133  		for valid := iter.First(); valid; valid = iter.Next() {
   134  		}
   135  		if err := iter.Close(); err != nil {
   136  			return err
   137  		}
   138  		return d.Close()
   139  	}
   140  
   141  	errorCounts := make(map[string]int)
   142  	for i := int32(0); ; i++ {
   143  		fs := errorfs.Wrap(vfs.NewMem(), errorfs.OnIndex(i))
   144  		err := run(fs)
   145  		if err == nil {
   146  			t.Logf("success %d\n", i)
   147  			break
   148  		}
   149  		errorCounts[err.Error()]++
   150  	}
   151  
   152  	expectedErrors := []string{
   153  		"fatal: MANIFEST flush failed: injected error",
   154  		"fatal: MANIFEST sync failed: injected error",
   155  		"fatal: MANIFEST set current failed: injected error",
   156  		"fatal: MANIFEST dirsync failed: injected error",
   157  	}
   158  	for _, expected := range expectedErrors {
   159  		if errorCounts[expected] == 0 {
   160  			t.Errorf("expected error %q did not occur", expected)
   161  		}
   162  	}
   163  }
   164  
   165  // TestRequireReadError injects FS errors into read operations at successively later
   166  // points until all operations can complete. It requires an operation fails any time
   167  // an error was injected. This differs from the TestErrors case above as that one
   168  // cannot require operations fail since it involves flush/compaction, which retry
   169  // internally and succeed following an injected error.
   170  func TestRequireReadError(t *testing.T) {
   171  	run := func(formatVersion FormatMajorVersion, index int32) (err error) {
   172  		// Perform setup with error injection disabled as it involves writes/background ops.
   173  		inj := errorfs.OnIndex(-1)
   174  		fs := errorfs.Wrap(vfs.NewMem(), inj)
   175  		opts := &Options{
   176  			FS:                 fs,
   177  			Logger:             panicLogger{},
   178  			FormatMajorVersion: formatVersion,
   179  		}
   180  		opts.private.disableTableStats = true
   181  		d, err := Open("", opts)
   182  		require.NoError(t, err)
   183  
   184  		defer func() {
   185  			if d != nil {
   186  				require.NoError(t, d.Close())
   187  			}
   188  		}()
   189  
   190  		key1 := []byte("a1")
   191  		key2 := []byte("a2")
   192  		value := []byte("b")
   193  		require.NoError(t, d.Set(key1, value, nil))
   194  		require.NoError(t, d.Set(key2, value, nil))
   195  		require.NoError(t, d.Flush())
   196  		require.NoError(t, d.Compact(key1, key2, false))
   197  		require.NoError(t, d.DeleteRange(key1, key2, nil))
   198  		require.NoError(t, d.Set(key1, value, nil))
   199  		require.NoError(t, d.Flush())
   200  		if formatVersion < FormatSetWithDelete {
   201  			expectLSM(`
   202  0.0:
   203    000007:[a1#4,SET-a2#72057594037927935,RANGEDEL]
   204  6:
   205    000005:[a1#1,SET-a2#2,SET]
   206  `, d, t)
   207  		} else {
   208  			expectLSM(`
   209  0.0:
   210    000007:[a1#4,SETWITHDEL-a2#72057594037927935,RANGEDEL]
   211  6:
   212    000005:[a1#1,SET-a2#2,SET]
   213  `, d, t)
   214  		}
   215  
   216  		// Now perform foreground ops with error injection enabled.
   217  		inj.SetIndex(index)
   218  		iter := d.NewIter(nil)
   219  		if err := iter.Error(); err != nil {
   220  			return err
   221  		}
   222  		numFound := 0
   223  		expectedKeys := [][]byte{key1, key2}
   224  		for valid := iter.First(); valid; valid = iter.Next() {
   225  			if !bytes.Equal(iter.Key(), expectedKeys[numFound]) {
   226  				t.Fatalf("expected key %v; found %v", expectedKeys[numFound], iter.Key())
   227  			}
   228  			if !bytes.Equal(iter.Value(), value) {
   229  				t.Fatalf("expected value %v; found %v", value, iter.Value())
   230  			}
   231  			numFound++
   232  		}
   233  		if err := iter.Close(); err != nil {
   234  			return err
   235  		}
   236  		if err := d.Close(); err != nil {
   237  			return err
   238  		}
   239  		d = nil
   240  		// Reaching here implies all read operations succeeded. This
   241  		// should only happen when we reached a large enough index at
   242  		// which `errorfs.FS` did not return any error.
   243  		if i := inj.Index(); i < 0 {
   244  			t.Errorf("FS error injected %d ops ago went unreported", -i)
   245  		}
   246  		if numFound != 2 {
   247  			t.Fatalf("expected 2 values; found %d", numFound)
   248  		}
   249  		return nil
   250  	}
   251  
   252  	versions := []FormatMajorVersion{FormatMostCompatible, FormatSetWithDelete}
   253  	for _, version := range versions {
   254  		t.Run(fmt.Sprintf("version-%s", version), func(t *testing.T) {
   255  			for i := int32(0); ; i++ {
   256  				err := run(version, i)
   257  				if err == nil {
   258  					t.Logf("no failures reported at index %d", i)
   259  					break
   260  				}
   261  			}
   262  		})
   263  	}
   264  }
   265  
   266  // TestCorruptReadError verifies that reads to a corrupted file detect the
   267  // corruption and return an error. In this case the filesystem reads return
   268  // successful status but the data they return is corrupt.
   269  func TestCorruptReadError(t *testing.T) {
   270  	run := func(formatVersion FormatMajorVersion, index int32) (err error) {
   271  		// Perform setup with corruption injection disabled as it involves writes/background ops.
   272  		fs := &corruptFS{
   273  			FS:    vfs.NewMem(),
   274  			index: -1,
   275  		}
   276  		opts := &Options{
   277  			FS:                 fs,
   278  			Logger:             panicLogger{},
   279  			FormatMajorVersion: formatVersion,
   280  		}
   281  		opts.private.disableTableStats = true
   282  		d, err := Open("", opts)
   283  		if err != nil {
   284  			t.Fatalf("%v", err)
   285  		}
   286  		defer func() {
   287  			if d != nil {
   288  				require.NoError(t, d.Close())
   289  			}
   290  		}()
   291  
   292  		key1 := []byte("a1")
   293  		key2 := []byte("a2")
   294  		value := []byte("b")
   295  		require.NoError(t, d.Set(key1, value, nil))
   296  		require.NoError(t, d.Set(key2, value, nil))
   297  		require.NoError(t, d.Flush())
   298  		require.NoError(t, d.Compact(key1, key2, false))
   299  		require.NoError(t, d.DeleteRange(key1, key2, nil))
   300  		require.NoError(t, d.Set(key1, value, nil))
   301  		require.NoError(t, d.Flush())
   302  		if formatVersion < FormatSetWithDelete {
   303  			expectLSM(`
   304  0.0:
   305    000007:[a1#4,SET-a2#72057594037927935,RANGEDEL]
   306  6:
   307    000005:[a1#1,SET-a2#2,SET]
   308  `, d, t)
   309  
   310  		} else {
   311  			expectLSM(`
   312  0.0:
   313    000007:[a1#4,SETWITHDEL-a2#72057594037927935,RANGEDEL]
   314  6:
   315    000005:[a1#1,SET-a2#2,SET]
   316  `, d, t)
   317  		}
   318  
   319  		// Now perform foreground ops with corruption injection enabled.
   320  		atomic.StoreInt32(&fs.index, index)
   321  		iter := d.NewIter(nil)
   322  		if err := iter.Error(); err != nil {
   323  			return err
   324  		}
   325  
   326  		numFound := 0
   327  		expectedKeys := [][]byte{key1, key2}
   328  		for valid := iter.First(); valid; valid = iter.Next() {
   329  			if !bytes.Equal(iter.Key(), expectedKeys[numFound]) {
   330  				t.Fatalf("expected key %v; found %v", expectedKeys[numFound], iter.Key())
   331  			}
   332  			if !bytes.Equal(iter.Value(), value) {
   333  				t.Fatalf("expected value %v; found %v", value, iter.Value())
   334  			}
   335  			numFound++
   336  		}
   337  		if err := iter.Close(); err != nil {
   338  			return err
   339  		}
   340  		if err := d.Close(); err != nil {
   341  			return err
   342  		}
   343  		d = nil
   344  		// Reaching here implies all read operations succeeded. This
   345  		// should only happen when we reached a large enough index at
   346  		// which `corruptFS` did not inject any corruption.
   347  		if atomic.LoadInt32(&fs.bytesRead) > fs.index {
   348  			t.Errorf("corruption error injected at index %d went unreported", fs.index)
   349  		}
   350  		if numFound != 2 {
   351  			t.Fatalf("expected 2 values; found %d", numFound)
   352  		}
   353  		return nil
   354  	}
   355  	versions := []FormatMajorVersion{FormatMostCompatible, FormatSetWithDelete}
   356  	for _, version := range versions {
   357  		t.Run(fmt.Sprintf("version-%s", version), func(t *testing.T) {
   358  			for i := int32(0); ; i++ {
   359  				err := run(version, i)
   360  				if err == nil {
   361  					t.Logf("no failures reported at index %d", i)
   362  					break
   363  				}
   364  			}
   365  		})
   366  	}
   367  }
   368  
   369  func TestDBWALRotationCrash(t *testing.T) {
   370  	memfs := vfs.NewStrictMem()
   371  
   372  	var index int32
   373  	inj := errorfs.InjectorFunc(func(op errorfs.Op, _ string) error {
   374  		if op.OpKind() == errorfs.OpKindWrite && atomic.AddInt32(&index, -1) == -1 {
   375  			memfs.SetIgnoreSyncs(true)
   376  		}
   377  		return nil
   378  	})
   379  	triggered := func() bool { return atomic.LoadInt32(&index) < 0 }
   380  
   381  	run := func(fs *errorfs.FS, k int32) (err error) {
   382  		opts := &Options{
   383  			FS:           fs,
   384  			Logger:       panicLogger{},
   385  			MemTableSize: 2048,
   386  		}
   387  		opts.private.disableTableStats = true
   388  		d, err := Open("", opts)
   389  		if err != nil || triggered() {
   390  			return err
   391  		}
   392  
   393  		// Write keys with the FS set up to simulate a crash by ignoring
   394  		// syncs on the k-th write operation.
   395  		atomic.StoreInt32(&index, k)
   396  		key := []byte("test")
   397  		for i := 0; i < 10; i++ {
   398  			v := []byte(strings.Repeat("b", i))
   399  			err = d.Set(key, v, nil)
   400  			if err != nil || triggered() {
   401  				break
   402  			}
   403  		}
   404  		err = firstError(err, d.Close())
   405  		return err
   406  	}
   407  
   408  	fs := errorfs.Wrap(memfs, inj)
   409  	for k := int32(0); ; k++ {
   410  		// Run, simulating a crash by ignoring syncs after the k-th write
   411  		// operation after Open.
   412  		atomic.StoreInt32(&index, math.MaxInt32)
   413  		err := run(fs, k)
   414  		if !triggered() {
   415  			// Stop when we reach a value of k greater than the number of
   416  			// write operations performed during `run`.
   417  			t.Logf("No crash at write operation %d\n", k)
   418  			if err != nil {
   419  				t.Fatalf("Filesystem did not 'crash', but error returned: %s", err)
   420  			}
   421  			break
   422  		}
   423  		t.Logf("Crashed at write operation % 2d, error: %v\n", k, err)
   424  
   425  		// Reset the filesystem to its state right before the simulated
   426  		// "crash", restore syncs, and run again without crashing.
   427  		memfs.ResetToSyncedState()
   428  		memfs.SetIgnoreSyncs(false)
   429  		atomic.StoreInt32(&index, math.MaxInt32)
   430  		require.NoError(t, run(fs, k))
   431  	}
   432  }