code.vegaprotocol.io/vega@v0.79.0/datanode/networkhistory/store/store_test.go (about)

     1  // Copyright (C) 2023 Gobalsky Labs Limited
     2  //
     3  // This program is free software: you can redistribute it and/or modify
     4  // it under the terms of the GNU Affero General Public License as
     5  // published by the Free Software Foundation, either version 3 of the
     6  // License, or (at your option) any later version.
     7  //
     8  // This program is distributed in the hope that it will be useful,
     9  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    10  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    11  // GNU Affero General Public License for more details.
    12  //
    13  // You should have received a copy of the GNU Affero General Public License
    14  // along with this program.  If not, see <http://www.gnu.org/licenses/>.
    15  
    16  package store_test
    17  
    18  import (
    19  	"archive/zip"
    20  	"bytes"
    21  	"context"
    22  	"io"
    23  	"io/fs"
    24  	"os"
    25  	"path/filepath"
    26  	"testing"
    27  
    28  	"code.vegaprotocol.io/vega/datanode/config/encoding"
    29  	"code.vegaprotocol.io/vega/datanode/networkhistory/segment"
    30  	"code.vegaprotocol.io/vega/datanode/networkhistory/store"
    31  	"code.vegaprotocol.io/vega/logging"
    32  
    33  	uuid "github.com/satori/go.uuid"
    34  	"github.com/stretchr/testify/assert"
    35  	"github.com/stretchr/testify/require"
    36  )
    37  
    38  const expectedSizeOnDiskWithNoGc = 20016
    39  
    40  func TestRemoveWithNoEligibleSegments(t *testing.T) {
    41  	chainID := uuid.NewV4().String()
    42  
    43  	networkhistoryHome := t.TempDir()
    44  
    45  	s, snapshotsDir := createStore(t, 20000, chainID, networkhistoryHome)
    46  	addTestData(t, chainID, snapshotsDir, s)
    47  
    48  	dirSize, err := dirSize(networkhistoryHome)
    49  	require.NoError(t, err)
    50  
    51  	assertRoughlyEqual(t, expectedSizeOnDiskWithNoGc, dirSize)
    52  
    53  	postGcSegments, err := s.ListAllIndexEntriesOldestFirst()
    54  	require.NoError(t, err)
    55  
    56  	assert.Equal(t, 10, len(postGcSegments))
    57  	assert.Equal(t, int64(1), postGcSegments[0].HeightFrom)
    58  	assert.Equal(t, int64(10000), postGcSegments[9].HeightTo)
    59  }
    60  
    61  func TestPartialRemoveOfOldSegments(t *testing.T) {
    62  	chainID := uuid.NewV4().String()
    63  
    64  	networkhistoryHome := t.TempDir()
    65  
    66  	s, snapshotsDir := createStore(t, 5000, chainID, networkhistoryHome)
    67  
    68  	addTestData(t, chainID, snapshotsDir, s)
    69  
    70  	dirSize, err := dirSize(networkhistoryHome)
    71  	require.NoError(t, err)
    72  
    73  	assertRoughlyEqual(t, 18463, dirSize)
    74  
    75  	segments, err := s.ListAllIndexEntriesOldestFirst()
    76  	require.NoError(t, err)
    77  
    78  	assert.Equal(t, 6, len(segments))
    79  	assert.Equal(t, int64(4001), segments[0].HeightFrom)
    80  	assert.Equal(t, int64(10000), segments[5].HeightTo)
    81  }
    82  
    83  func TestRemoveAllOldSegments(t *testing.T) {
    84  	chainID := uuid.NewV4().String()
    85  
    86  	networkhistoryHome := t.TempDir()
    87  
    88  	s, snapshotsDir := createStore(t, 0, chainID, networkhistoryHome)
    89  
    90  	addTestData(t, chainID, snapshotsDir, s)
    91  
    92  	dirSize, err := dirSize(networkhistoryHome)
    93  	require.NoError(t, err)
    94  
    95  	assertRoughlyEqual(t, 17504, dirSize)
    96  
    97  	segments, err := s.ListAllIndexEntriesOldestFirst()
    98  	require.NoError(t, err)
    99  
   100  	assert.Equal(t, 1, len(segments))
   101  	assert.Equal(t, int64(9001), segments[0].HeightFrom)
   102  	assert.Equal(t, int64(10000), segments[0].HeightTo)
   103  }
   104  
   105  func addTestData(t *testing.T, chainID string, snapshotsDir string, s *store.Store) {
   106  	t.Helper()
   107  	for i := int64(0); i < 10; i++ {
   108  		from := (i * 1000) + 1
   109  		to := (i + 1) * 1000
   110  		segment := segment.Unpublished{
   111  			Base: segment.Base{
   112  				HeightFrom:      from,
   113  				HeightTo:        to,
   114  				ChainID:         chainID,
   115  				DatabaseVersion: 1,
   116  			},
   117  			Directory: snapshotsDir,
   118  		}
   119  
   120  		buf := new(bytes.Buffer)
   121  		zipWriter := zip.NewWriter(buf)
   122  
   123  		f, err := zipWriter.Create("dummy.txt")
   124  		require.NoError(t, err)
   125  
   126  		_, err = io.WriteString(f, "This is some dummy data.")
   127  		require.NoError(t, err)
   128  
   129  		err = zipWriter.Close()
   130  		require.NoError(t, err)
   131  
   132  		err = os.WriteFile(segment.UnpublishedSnapshotDataDirectory(), buf.Bytes(), fs.ModePerm)
   133  		require.NoError(t, err)
   134  
   135  		err = s.AddSnapshotData(context.Background(), segment)
   136  		require.NoError(t, err)
   137  	}
   138  }
   139  
   140  func assertRoughlyEqual(t *testing.T, expected, actual int64) {
   141  	t.Helper()
   142  	permissablePercentDiff := int64(5)
   143  	lowerBound := expected - ((expected * permissablePercentDiff) / 100)
   144  	upperBound := expected + ((expected * permissablePercentDiff) / 100)
   145  
   146  	assert.Less(t, lowerBound, actual)
   147  	assert.Greater(t, upperBound, actual)
   148  }
   149  
   150  func createStore(t *testing.T, historyRetentionBlockSpan int64, chainID string, networkhistoryHome string) (*store.Store, string) {
   151  	t.Helper()
   152  	log := logging.NewTestLogger()
   153  	cfg := store.NewDefaultConfig()
   154  	cfg.HistoryRetentionBlockSpan = historyRetentionBlockSpan
   155  	cfg.GarbageCollectionInterval = encoding.Duration{Duration: 0}
   156  	snapshotsDir := t.TempDir()
   157  
   158  	s, err := store.New(context.Background(), log, chainID, cfg, networkhistoryHome, 33)
   159  	require.NoError(t, err)
   160  	return s, snapshotsDir
   161  }
   162  
   163  func dirSize(path string) (int64, error) {
   164  	var size int64
   165  	err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
   166  		if err != nil || (info != nil && info.IsDir()) {
   167  			return nil //nolint:nilerr
   168  		}
   169  
   170  		size += info.Size()
   171  		return nil
   172  	})
   173  	return size, err
   174  }