github.com/ledgerwatch/erigon-lib@v1.0.0/state/gc_test.go (about)

     1  package state
     2  
     3  import (
     4  	"context"
     5  	"testing"
     6  	"time"
     7  
     8  	"github.com/ledgerwatch/erigon-lib/kv"
     9  	"github.com/ledgerwatch/log/v3"
    10  	"github.com/stretchr/testify/require"
    11  )
    12  
    13  func TestGCReadAfterRemoveFile(t *testing.T) {
    14  	logger := log.New()
    15  	logEvery := time.NewTicker(30 * time.Second)
    16  	defer logEvery.Stop()
    17  	ctx := context.Background()
    18  
    19  	test := func(t *testing.T, h *History, db kv.RwDB, txs uint64) {
    20  		t.Helper()
    21  		require := require.New(t)
    22  		collateAndMergeHistory(t, db, h, txs)
    23  
    24  		t.Run("read after: remove when have reader", func(t *testing.T) {
    25  			tx, err := db.BeginRo(ctx)
    26  			require.NoError(err)
    27  			defer tx.Rollback()
    28  
    29  			// - create immutable view
    30  			// - del cold file
    31  			// - read from canDelete file
    32  			// - close view
    33  			// - open new view
    34  			// - make sure there is no canDelete file
    35  			hc := h.MakeContext()
    36  			_ = hc
    37  			lastOnFs, _ := h.files.Max()
    38  			require.False(lastOnFs.frozen) // prepared dataset must have some non-frozen files. or it's bad dataset.
    39  			h.integrateMergedFiles(nil, []*filesItem{lastOnFs}, nil, nil)
    40  			require.NotNil(lastOnFs.decompressor)
    41  
    42  			lastInView := hc.files[len(hc.files)-1]
    43  			g := lastInView.src.decompressor.MakeGetter()
    44  			require.Equal(lastInView.startTxNum, lastOnFs.startTxNum)
    45  			require.Equal(lastInView.endTxNum, lastOnFs.endTxNum)
    46  			if g.HasNext() {
    47  				k, _ := g.Next(nil)
    48  				require.Equal(8, len(k))
    49  				v, _ := g.Next(nil)
    50  				require.Equal(8, len(v))
    51  			}
    52  
    53  			require.NotNil(lastOnFs.decompressor)
    54  			loc := hc.ic.loc // replace of locality index must not affect current HistoryContext, but expect to be closed after last reader
    55  			h.localityIndex.integrateFiles(LocalityIndexFiles{}, 0, 0)
    56  			require.NotNil(loc.file)
    57  			hc.Close()
    58  			require.Nil(lastOnFs.decompressor)
    59  			require.NotNil(loc.file)
    60  
    61  			nonDeletedOnFs, _ := h.files.Max()
    62  			require.False(nonDeletedOnFs.frozen)
    63  			require.NotNil(nonDeletedOnFs.decompressor) // non-canDelete files are not closed
    64  
    65  			hc = h.MakeContext()
    66  			newLastInView := hc.files[len(hc.files)-1]
    67  			require.False(lastOnFs.frozen)
    68  			require.False(lastInView.startTxNum == newLastInView.startTxNum && lastInView.endTxNum == newLastInView.endTxNum)
    69  
    70  			hc.Close()
    71  		})
    72  
    73  		t.Run("read after: remove when no readers", func(t *testing.T) {
    74  			tx, err := db.BeginRo(ctx)
    75  			require.NoError(err)
    76  			defer tx.Rollback()
    77  
    78  			// - del cold file
    79  			// - new reader must not see canDelete file
    80  			hc := h.MakeContext()
    81  			lastOnFs, _ := h.files.Max()
    82  			require.False(lastOnFs.frozen) // prepared dataset must have some non-frozen files. or it's bad dataset.
    83  			h.integrateMergedFiles(nil, []*filesItem{lastOnFs}, nil, nil)
    84  
    85  			require.NotNil(lastOnFs.decompressor)
    86  			hc.Close()
    87  			require.Nil(lastOnFs.decompressor)
    88  		})
    89  	}
    90  	t.Run("large_values", func(t *testing.T) {
    91  		_, db, h, txs := filledHistory(t, true, logger)
    92  		test(t, h, db, txs)
    93  	})
    94  	t.Run("small_values", func(t *testing.T) {
    95  		_, db, h, txs := filledHistory(t, false, logger)
    96  		test(t, h, db, txs)
    97  	})
    98  }
    99  
   100  func TestDomainGCReadAfterRemoveFile(t *testing.T) {
   101  	logEvery := time.NewTicker(30 * time.Second)
   102  	defer logEvery.Stop()
   103  	ctx := context.Background()
   104  
   105  	test := func(t *testing.T, h *Domain, db kv.RwDB, txs uint64) {
   106  		t.Helper()
   107  		require := require.New(t)
   108  		collateAndMerge(t, db, nil, h, txs)
   109  
   110  		t.Run("read after: remove when have reader", func(t *testing.T) {
   111  			tx, err := db.BeginRo(ctx)
   112  			require.NoError(err)
   113  			defer tx.Rollback()
   114  
   115  			// - create immutable view
   116  			// - del cold file
   117  			// - read from canDelete file
   118  			// - close view
   119  			// - open new view
   120  			// - make sure there is no canDelete file
   121  			hc := h.MakeContext()
   122  			_ = hc
   123  			lastOnFs, _ := h.files.Max()
   124  			require.False(lastOnFs.frozen) // prepared dataset must have some non-frozen files. or it's bad dataset.
   125  			h.integrateMergedFiles([]*filesItem{lastOnFs}, nil, nil, nil, nil, nil)
   126  			require.NotNil(lastOnFs.decompressor)
   127  
   128  			lastInView := hc.files[len(hc.files)-1]
   129  			g := lastInView.src.decompressor.MakeGetter()
   130  			require.Equal(lastInView.startTxNum, lastOnFs.startTxNum)
   131  			require.Equal(lastInView.endTxNum, lastOnFs.endTxNum)
   132  			if g.HasNext() {
   133  				k, _ := g.Next(nil)
   134  				require.Equal(8, len(k))
   135  				v, _ := g.Next(nil)
   136  				require.Equal(8, len(v))
   137  			}
   138  
   139  			require.NotNil(lastOnFs.decompressor)
   140  			hc.Close()
   141  			require.Nil(lastOnFs.decompressor)
   142  
   143  			nonDeletedOnFs, _ := h.files.Max()
   144  			require.False(nonDeletedOnFs.frozen)
   145  			require.NotNil(nonDeletedOnFs.decompressor) // non-canDelete files are not closed
   146  
   147  			hc = h.MakeContext()
   148  			newLastInView := hc.files[len(hc.files)-1]
   149  			require.False(lastOnFs.frozen)
   150  			require.False(lastInView.startTxNum == newLastInView.startTxNum && lastInView.endTxNum == newLastInView.endTxNum)
   151  
   152  			hc.Close()
   153  		})
   154  
   155  		t.Run("read after: remove when no readers", func(t *testing.T) {
   156  			tx, err := db.BeginRo(ctx)
   157  			require.NoError(err)
   158  			defer tx.Rollback()
   159  
   160  			// - del cold file
   161  			// - new reader must not see canDelete file
   162  			hc := h.MakeContext()
   163  			lastOnFs, _ := h.files.Max()
   164  			require.False(lastOnFs.frozen) // prepared dataset must have some non-frozen files. or it's bad dataset.
   165  			h.integrateMergedFiles([]*filesItem{lastOnFs}, nil, nil, nil, nil, nil)
   166  
   167  			require.NotNil(lastOnFs.decompressor)
   168  			hc.Close()
   169  			require.Nil(lastOnFs.decompressor)
   170  		})
   171  	}
   172  	logger := log.New()
   173  	_, db, d, txs := filledDomain(t, logger)
   174  	test(t, d, db, txs)
   175  }