github.com/palisadeinc/bor@v0.0.0-20230615125219-ab7196213d15/core/rawdb/freezer_test.go (about)

     1  // Copyright 2021 The go-ethereum Authors
     2  // This file is part of the go-ethereum library.
     3  //
     4  // The go-ethereum library is free software: you can redistribute it and/or modify
     5  // it under the terms of the GNU Lesser General Public License as published by
     6  // the Free Software Foundation, either version 3 of the License, or
     7  // (at your option) any later version.
     8  //
     9  // The go-ethereum library is distributed in the hope that it will be useful,
    10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    12  // GNU Lesser General Public License for more details.
    13  //
    14  // You should have received a copy of the GNU Lesser General Public License
    15  // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
    16  
    17  package rawdb
    18  
    19  import (
    20  	"bytes"
    21  	"errors"
    22  	"fmt"
    23  	"io/ioutil"
    24  	"math/big"
    25  	"math/rand"
    26  	"os"
    27  	"path"
    28  	"sync"
    29  	"testing"
    30  
    31  	"github.com/ethereum/go-ethereum/ethdb"
    32  	"github.com/ethereum/go-ethereum/rlp"
    33  	"github.com/stretchr/testify/require"
    34  )
    35  
    36  var freezerTestTableDef = map[string]bool{"test": true}
    37  
    38  func TestFreezerModify(t *testing.T) {
    39  	t.Parallel()
    40  
    41  	// Create test data.
    42  	var valuesRaw [][]byte
    43  	var valuesRLP []*big.Int
    44  	for x := 0; x < 100; x++ {
    45  		v := getChunk(256, x)
    46  		valuesRaw = append(valuesRaw, v)
    47  		iv := big.NewInt(int64(x))
    48  		iv = iv.Exp(iv, iv, nil)
    49  		valuesRLP = append(valuesRLP, iv)
    50  	}
    51  
    52  	tables := map[string]bool{"raw": true, "rlp": false}
    53  	f, dir := newFreezerForTesting(t, tables)
    54  	defer os.RemoveAll(dir)
    55  	defer f.Close()
    56  
    57  	// Commit test data.
    58  	_, err := f.ModifyAncients(func(op ethdb.AncientWriteOp) error {
    59  		for i := range valuesRaw {
    60  			if err := op.AppendRaw("raw", uint64(i), valuesRaw[i]); err != nil {
    61  				return err
    62  			}
    63  			if err := op.Append("rlp", uint64(i), valuesRLP[i]); err != nil {
    64  				return err
    65  			}
    66  		}
    67  		return nil
    68  	})
    69  	if err != nil {
    70  		t.Fatal("ModifyAncients failed:", err)
    71  	}
    72  
    73  	// Dump indexes.
    74  	for _, table := range f.tables {
    75  		t.Log(table.name, "index:", table.dumpIndexString(0, int64(len(valuesRaw))))
    76  	}
    77  
    78  	// Read back test data.
    79  	checkAncientCount(t, f, "raw", uint64(len(valuesRaw)))
    80  	checkAncientCount(t, f, "rlp", uint64(len(valuesRLP)))
    81  	for i := range valuesRaw {
    82  		v, _ := f.Ancient("raw", uint64(i))
    83  		if !bytes.Equal(v, valuesRaw[i]) {
    84  			t.Fatalf("wrong raw value at %d: %x", i, v)
    85  		}
    86  		ivEnc, _ := f.Ancient("rlp", uint64(i))
    87  		want, _ := rlp.EncodeToBytes(valuesRLP[i])
    88  		if !bytes.Equal(ivEnc, want) {
    89  			t.Fatalf("wrong RLP value at %d: %x", i, ivEnc)
    90  		}
    91  	}
    92  }
    93  
    94  // This checks that ModifyAncients rolls back freezer updates
    95  // when the function passed to it returns an error.
    96  func TestFreezerModifyRollback(t *testing.T) {
    97  	t.Parallel()
    98  
    99  	f, dir := newFreezerForTesting(t, freezerTestTableDef)
   100  	defer os.RemoveAll(dir)
   101  
   102  	theError := errors.New("oops")
   103  	_, err := f.ModifyAncients(func(op ethdb.AncientWriteOp) error {
   104  		// Append three items. This creates two files immediately,
   105  		// because the table size limit of the test freezer is 2048.
   106  		require.NoError(t, op.AppendRaw("test", 0, make([]byte, 2048)))
   107  		require.NoError(t, op.AppendRaw("test", 1, make([]byte, 2048)))
   108  		require.NoError(t, op.AppendRaw("test", 2, make([]byte, 2048)))
   109  		return theError
   110  	})
   111  	if err != theError {
   112  		t.Errorf("ModifyAncients returned wrong error %q", err)
   113  	}
   114  	checkAncientCount(t, f, "test", 0)
   115  	f.Close()
   116  
   117  	// Reopen and check that the rolled-back data doesn't reappear.
   118  	tables := map[string]bool{"test": true}
   119  	f2, err := newFreezer(dir, "", false, 2049, tables)
   120  	if err != nil {
   121  		t.Fatalf("can't reopen freezer after failed ModifyAncients: %v", err)
   122  	}
   123  	defer f2.Close()
   124  	checkAncientCount(t, f2, "test", 0)
   125  }
   126  
   127  // This test runs ModifyAncients and Ancient concurrently with each other.
   128  func TestFreezerConcurrentModifyRetrieve(t *testing.T) {
   129  	t.Parallel()
   130  
   131  	f, dir := newFreezerForTesting(t, freezerTestTableDef)
   132  	defer os.RemoveAll(dir)
   133  	defer f.Close()
   134  
   135  	var (
   136  		numReaders     = 5
   137  		writeBatchSize = uint64(50)
   138  		written        = make(chan uint64, numReaders*6)
   139  		wg             sync.WaitGroup
   140  	)
   141  	wg.Add(numReaders + 1)
   142  
   143  	// Launch the writer. It appends 10000 items in batches.
   144  	go func() {
   145  		defer wg.Done()
   146  		defer close(written)
   147  		for item := uint64(0); item < 10000; item += writeBatchSize {
   148  			_, err := f.ModifyAncients(func(op ethdb.AncientWriteOp) error {
   149  				for i := uint64(0); i < writeBatchSize; i++ {
   150  					item := item + i
   151  					value := getChunk(32, int(item))
   152  					if err := op.AppendRaw("test", item, value); err != nil {
   153  						return err
   154  					}
   155  				}
   156  				return nil
   157  			})
   158  			if err != nil {
   159  				panic(err)
   160  			}
   161  			for i := 0; i < numReaders; i++ {
   162  				written <- item + writeBatchSize
   163  			}
   164  		}
   165  	}()
   166  
   167  	// Launch the readers. They read random items from the freezer up to the
   168  	// current frozen item count.
   169  	for i := 0; i < numReaders; i++ {
   170  		go func() {
   171  			defer wg.Done()
   172  			for frozen := range written {
   173  				for rc := 0; rc < 80; rc++ {
   174  					num := uint64(rand.Intn(int(frozen)))
   175  					value, err := f.Ancient("test", num)
   176  					if err != nil {
   177  						panic(fmt.Errorf("error reading %d (frozen %d): %v", num, frozen, err))
   178  					}
   179  					if !bytes.Equal(value, getChunk(32, int(num))) {
   180  						panic(fmt.Errorf("wrong value at %d", num))
   181  					}
   182  				}
   183  			}
   184  		}()
   185  	}
   186  
   187  	wg.Wait()
   188  }
   189  
   190  // This test runs ModifyAncients and TruncateHead concurrently with each other.
   191  func TestFreezerConcurrentModifyTruncate(t *testing.T) {
   192  	f, dir := newFreezerForTesting(t, freezerTestTableDef)
   193  	defer os.RemoveAll(dir)
   194  	defer f.Close()
   195  
   196  	var item = make([]byte, 256)
   197  
   198  	for i := 0; i < 1000; i++ {
   199  		// First reset and write 100 items.
   200  		if err := f.TruncateHead(0); err != nil {
   201  			t.Fatal("truncate failed:", err)
   202  		}
   203  		_, err := f.ModifyAncients(func(op ethdb.AncientWriteOp) error {
   204  			for i := uint64(0); i < 100; i++ {
   205  				if err := op.AppendRaw("test", i, item); err != nil {
   206  					return err
   207  				}
   208  			}
   209  			return nil
   210  		})
   211  		if err != nil {
   212  			t.Fatal("modify failed:", err)
   213  		}
   214  		checkAncientCount(t, f, "test", 100)
   215  
   216  		// Now append 100 more items and truncate concurrently.
   217  		var (
   218  			wg          sync.WaitGroup
   219  			truncateErr error
   220  			modifyErr   error
   221  		)
   222  		wg.Add(3)
   223  		go func() {
   224  			_, modifyErr = f.ModifyAncients(func(op ethdb.AncientWriteOp) error {
   225  				for i := uint64(100); i < 200; i++ {
   226  					if err := op.AppendRaw("test", i, item); err != nil {
   227  						return err
   228  					}
   229  				}
   230  				return nil
   231  			})
   232  			wg.Done()
   233  		}()
   234  		go func() {
   235  			truncateErr = f.TruncateHead(10)
   236  			wg.Done()
   237  		}()
   238  		go func() {
   239  			f.AncientSize("test")
   240  			wg.Done()
   241  		}()
   242  		wg.Wait()
   243  
   244  		// Now check the outcome. If the truncate operation went through first, the append
   245  		// fails, otherwise it succeeds. In either case, the freezer should be positioned
   246  		// at 10 after both operations are done.
   247  		if truncateErr != nil {
   248  			t.Fatal("concurrent truncate failed:", err)
   249  		}
   250  		if !(errors.Is(modifyErr, nil) || errors.Is(modifyErr, errOutOrderInsertion)) {
   251  			t.Fatal("wrong error from concurrent modify:", modifyErr)
   252  		}
   253  		checkAncientCount(t, f, "test", 10)
   254  	}
   255  }
   256  
   257  func TestFreezerReadonlyValidate(t *testing.T) {
   258  	tables := map[string]bool{"a": true, "b": true}
   259  	dir, err := ioutil.TempDir("", "freezer")
   260  	if err != nil {
   261  		t.Fatal(err)
   262  	}
   263  	defer os.RemoveAll(dir)
   264  	// Open non-readonly freezer and fill individual tables
   265  	// with different amount of data.
   266  	f, err := newFreezer(dir, "", false, 2049, tables)
   267  	if err != nil {
   268  		t.Fatal("can't open freezer", err)
   269  	}
   270  	var item = make([]byte, 1024)
   271  	aBatch := f.tables["a"].newBatch()
   272  	require.NoError(t, aBatch.AppendRaw(0, item))
   273  	require.NoError(t, aBatch.AppendRaw(1, item))
   274  	require.NoError(t, aBatch.AppendRaw(2, item))
   275  	require.NoError(t, aBatch.commit())
   276  	bBatch := f.tables["b"].newBatch()
   277  	require.NoError(t, bBatch.AppendRaw(0, item))
   278  	require.NoError(t, bBatch.commit())
   279  	if f.tables["a"].items != 3 {
   280  		t.Fatalf("unexpected number of items in table")
   281  	}
   282  	if f.tables["b"].items != 1 {
   283  		t.Fatalf("unexpected number of items in table")
   284  	}
   285  	require.NoError(t, f.Close())
   286  
   287  	// Re-openening as readonly should fail when validating
   288  	// table lengths.
   289  	f, err = newFreezer(dir, "", true, 2049, tables)
   290  	if err == nil {
   291  		t.Fatal("readonly freezer should fail with differing table lengths")
   292  	}
   293  }
   294  
   295  func newFreezerForTesting(t *testing.T, tables map[string]bool) (*freezer, string) {
   296  	t.Helper()
   297  
   298  	dir, err := ioutil.TempDir("", "freezer")
   299  	if err != nil {
   300  		t.Fatal(err)
   301  	}
   302  	// note: using low max table size here to ensure the tests actually
   303  	// switch between multiple files.
   304  	f, err := newFreezer(dir, "", false, 2049, tables)
   305  	if err != nil {
   306  		t.Fatal("can't open freezer", err)
   307  	}
   308  	return f, dir
   309  }
   310  
   311  // checkAncientCount verifies that the freezer contains n items.
   312  func checkAncientCount(t *testing.T, f *freezer, kind string, n uint64) {
   313  	t.Helper()
   314  
   315  	if frozen, _ := f.Ancients(); frozen != n {
   316  		t.Fatalf("Ancients() returned %d, want %d", frozen, n)
   317  	}
   318  
   319  	// Check at index n-1.
   320  	if n > 0 {
   321  		index := n - 1
   322  		if ok, _ := f.HasAncient(kind, index); !ok {
   323  			t.Errorf("HasAncient(%q, %d) returned false unexpectedly", kind, index)
   324  		}
   325  		if _, err := f.Ancient(kind, index); err != nil {
   326  			t.Errorf("Ancient(%q, %d) returned unexpected error %q", kind, index, err)
   327  		}
   328  	}
   329  
   330  	// Check at index n.
   331  	index := n
   332  	if ok, _ := f.HasAncient(kind, index); ok {
   333  		t.Errorf("HasAncient(%q, %d) returned true unexpectedly", kind, index)
   334  	}
   335  	if _, err := f.Ancient(kind, index); err == nil {
   336  		t.Errorf("Ancient(%q, %d) didn't return expected error", kind, index)
   337  	} else if err != errOutOfBounds {
   338  		t.Errorf("Ancient(%q, %d) returned unexpected error %q", kind, index, err)
   339  	}
   340  }
   341  
   342  func TestRenameWindows(t *testing.T) {
   343  	var (
   344  		fname   = "file.bin"
   345  		fname2  = "file2.bin"
   346  		data    = []byte{1, 2, 3, 4}
   347  		data2   = []byte{2, 3, 4, 5}
   348  		data3   = []byte{3, 5, 6, 7}
   349  		dataLen = 4
   350  	)
   351  
   352  	// Create 2 temp dirs
   353  	dir1, err := os.MkdirTemp("", "rename-test")
   354  	if err != nil {
   355  		t.Fatal(err)
   356  	}
   357  	defer os.Remove(dir1)
   358  	dir2, err := os.MkdirTemp("", "rename-test")
   359  	if err != nil {
   360  		t.Fatal(err)
   361  	}
   362  	defer os.Remove(dir2)
   363  
   364  	// Create file in dir1 and fill with data
   365  	f, err := os.Create(path.Join(dir1, fname))
   366  	if err != nil {
   367  		t.Fatal(err)
   368  	}
   369  	f2, err := os.Create(path.Join(dir1, fname2))
   370  	if err != nil {
   371  		t.Fatal(err)
   372  	}
   373  	f3, err := os.Create(path.Join(dir2, fname2))
   374  	if err != nil {
   375  		t.Fatal(err)
   376  	}
   377  	if _, err := f.Write(data); err != nil {
   378  		t.Fatal(err)
   379  	}
   380  	if _, err := f2.Write(data2); err != nil {
   381  		t.Fatal(err)
   382  	}
   383  	if _, err := f3.Write(data3); err != nil {
   384  		t.Fatal(err)
   385  	}
   386  	if err := f.Close(); err != nil {
   387  		t.Fatal(err)
   388  	}
   389  	if err := f2.Close(); err != nil {
   390  		t.Fatal(err)
   391  	}
   392  	if err := f3.Close(); err != nil {
   393  		t.Fatal(err)
   394  	}
   395  	if err := os.Rename(f.Name(), path.Join(dir2, fname)); err != nil {
   396  		t.Fatal(err)
   397  	}
   398  	if err := os.Rename(f2.Name(), path.Join(dir2, fname2)); err != nil {
   399  		t.Fatal(err)
   400  	}
   401  
   402  	// Check file contents
   403  	f, err = os.Open(path.Join(dir2, fname))
   404  	if err != nil {
   405  		t.Fatal(err)
   406  	}
   407  	defer f.Close()
   408  	defer os.Remove(f.Name())
   409  	buf := make([]byte, dataLen)
   410  	if _, err := f.Read(buf); err != nil {
   411  		t.Fatal(err)
   412  	}
   413  	if !bytes.Equal(buf, data) {
   414  		t.Errorf("unexpected file contents. Got %v\n", buf)
   415  	}
   416  
   417  	f, err = os.Open(path.Join(dir2, fname2))
   418  	if err != nil {
   419  		t.Fatal(err)
   420  	}
   421  	defer f.Close()
   422  	defer os.Remove(f.Name())
   423  	if _, err := f.Read(buf); err != nil {
   424  		t.Fatal(err)
   425  	}
   426  	if !bytes.Equal(buf, data2) {
   427  		t.Errorf("unexpected file contents. Got %v\n", buf)
   428  	}
   429  }