github.com/koko1123/flow-go-1@v0.29.6/ledger/complete/wal/wal_test.go (about)

     1  package wal
     2  
     3  import (
     4  	"testing"
     5  
     6  	"github.com/rs/zerolog"
     7  	"github.com/stretchr/testify/require"
     8  
     9  	"github.com/koko1123/flow-go-1/module/metrics"
    10  	"github.com/koko1123/flow-go-1/storage/util"
    11  	"github.com/koko1123/flow-go-1/utils/unittest"
    12  )
    13  
    14  var (
    15  	pathByteSize = 32
    16  	segmentSize  = 32 * 1024
    17  )
    18  
    19  func RunWithWALCheckpointerWithFiles(t *testing.T, names ...interface{}) {
    20  	f := names[len(names)-1].(func(*testing.T, *DiskWAL, *Checkpointer))
    21  
    22  	fileNames := make([]string, len(names)-1)
    23  
    24  	for i := 0; i <= len(names)-2; i++ {
    25  		fileNames[i] = names[i].(string)
    26  	}
    27  
    28  	unittest.RunWithTempDir(t, func(dir string) {
    29  		util.CreateFiles(t, dir, fileNames...)
    30  
    31  		wal, err := NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dir, 10, pathByteSize, segmentSize)
    32  		require.NoError(t, err)
    33  
    34  		checkpointer, err := wal.NewCheckpointer()
    35  		require.NoError(t, err)
    36  
    37  		f(t, wal, checkpointer)
    38  
    39  		<-wal.Done()
    40  	})
    41  }
    42  
    43  func Test_emptyDir(t *testing.T) {
    44  	RunWithWALCheckpointerWithFiles(t, func(t *testing.T, wal *DiskWAL, checkpointer *Checkpointer) {
    45  		latestCheckpoint, err := checkpointer.LatestCheckpoint()
    46  		require.NoError(t, err)
    47  		require.Equal(t, -1, latestCheckpoint)
    48  
    49  		// here when starting LedgerWAL, it creates empty file for writing, hence directory isn't really empty
    50  		from, to, err := checkpointer.NotCheckpointedSegments()
    51  		require.NoError(t, err)
    52  		require.Equal(t, 0, from)
    53  		require.Equal(t, 0, to)
    54  	})
    55  }
    56  
    57  // Prometheus WAL require files to be 8 characters, otherwise it gets confused
    58  func Test_noCheckpoints(t *testing.T) {
    59  	RunWithWALCheckpointerWithFiles(t, "00000000", "00000001", "00000002", func(t *testing.T, wal *DiskWAL, checkpointer *Checkpointer) {
    60  		latestCheckpoint, err := checkpointer.LatestCheckpoint()
    61  		require.NoError(t, err)
    62  		require.Equal(t, -1, latestCheckpoint)
    63  
    64  		from, to, err := checkpointer.NotCheckpointedSegments()
    65  		require.NoError(t, err)
    66  		require.Equal(t, 0, from)
    67  		require.Equal(t, 3, to) //extra one because WAL now creates empty file on start
    68  	})
    69  }
    70  
    71  func Test_someCheckpoints(t *testing.T) {
    72  	RunWithWALCheckpointerWithFiles(t, "00000000", "00000001", "00000002", "00000003", "00000004", "00000005", "checkpoint.00000002", func(t *testing.T, wal *DiskWAL, checkpointer *Checkpointer) {
    73  		latestCheckpoint, err := checkpointer.LatestCheckpoint()
    74  		require.NoError(t, err)
    75  		require.Equal(t, 2, latestCheckpoint)
    76  
    77  		from, to, err := checkpointer.NotCheckpointedSegments()
    78  		require.NoError(t, err)
    79  		require.Equal(t, 3, from)
    80  		require.Equal(t, 6, to) //extra one because WAL now creates empty file on start
    81  	})
    82  }
    83  
    84  func Test_loneCheckpoint(t *testing.T) {
    85  	RunWithWALCheckpointerWithFiles(t, "checkpoint.00000005", func(t *testing.T, wal *DiskWAL, checkpointer *Checkpointer) {
    86  		latestCheckpoint, err := checkpointer.LatestCheckpoint()
    87  		require.NoError(t, err)
    88  		require.Equal(t, 5, latestCheckpoint)
    89  
    90  		from, to, err := checkpointer.NotCheckpointedSegments()
    91  		require.NoError(t, err)
    92  		require.Equal(t, -1, from)
    93  		require.Equal(t, -1, to)
    94  	})
    95  }
    96  
    97  func Test_lastCheckpointIsFoundByNumericValue(t *testing.T) {
    98  	RunWithWALCheckpointerWithFiles(t, "checkpoint.00000005", "checkpoint.00000004", "checkpoint.00000006", "checkpoint.00000002", "checkpoint.00000001", func(t *testing.T, wal *DiskWAL, checkpointer *Checkpointer) {
    99  		latestCheckpoint, err := checkpointer.LatestCheckpoint()
   100  		require.NoError(t, err)
   101  		require.Equal(t, 6, latestCheckpoint)
   102  	})
   103  }
   104  
   105  func Test_checkpointWithoutPrecedingSegments(t *testing.T) {
   106  	RunWithWALCheckpointerWithFiles(t, "checkpoint.00000005", "00000006", "00000007", func(t *testing.T, wal *DiskWAL, checkpointer *Checkpointer) {
   107  		latestCheckpoint, err := checkpointer.LatestCheckpoint()
   108  		require.NoError(t, err)
   109  		require.Equal(t, 5, latestCheckpoint)
   110  
   111  		from, to, err := checkpointer.NotCheckpointedSegments()
   112  		require.NoError(t, err)
   113  		require.Equal(t, 6, from)
   114  		require.Equal(t, 8, to) //extra one because WAL now creates empty file on start
   115  	})
   116  }
   117  
   118  func Test_checkpointWithSameSegment(t *testing.T) {
   119  	RunWithWALCheckpointerWithFiles(t, "checkpoint.00000005", "00000005", "00000006", "00000007", func(t *testing.T, wal *DiskWAL, checkpointer *Checkpointer) {
   120  		latestCheckpoint, err := checkpointer.LatestCheckpoint()
   121  		require.NoError(t, err)
   122  		require.Equal(t, 5, latestCheckpoint)
   123  
   124  		from, to, err := checkpointer.NotCheckpointedSegments()
   125  		require.NoError(t, err)
   126  		require.Equal(t, 6, from)
   127  		require.Equal(t, 8, to) //extra one because WAL now creates empty file on start
   128  	})
   129  }
   130  
   131  func Test_listingCheckpoints(t *testing.T) {
   132  	RunWithWALCheckpointerWithFiles(t, "checkpoint.00000005", "checkpoint.00000002", "00000003", "checkpoint.00000000", func(t *testing.T, wal *DiskWAL, checkpointer *Checkpointer) {
   133  		listCheckpoints, err := checkpointer.Checkpoints()
   134  		require.NoError(t, err)
   135  		require.Len(t, listCheckpoints, 3)
   136  		require.Equal(t, []int{0, 2, 5}, listCheckpoints)
   137  	})
   138  }
   139  
   140  func Test_NoGapBetweenSegmentsAndLastCheckpoint(t *testing.T) {
   141  	RunWithWALCheckpointerWithFiles(t, "checkpoint.00000004", "00000006", "00000007", func(t *testing.T, wal *DiskWAL, checkpointer *Checkpointer) {
   142  		latestCheckpoint, err := checkpointer.LatestCheckpoint()
   143  		require.NoError(t, err)
   144  		require.Equal(t, 4, latestCheckpoint)
   145  
   146  		_, _, err = checkpointer.NotCheckpointedSegments()
   147  		require.Error(t, err)
   148  	})
   149  }
   150  
   151  func Test_LatestPossibleCheckpoints(t *testing.T) {
   152  
   153  	require.Equal(t, []int(nil), getPossibleCheckpoints([]int{1, 2, 5}, 0, 0))
   154  
   155  	require.Equal(t, []int{1}, getPossibleCheckpoints([]int{1, 2, 5}, 0, 1))
   156  	require.Equal(t, []int{1, 2}, getPossibleCheckpoints([]int{1, 2, 5}, 0, 2))
   157  	require.Equal(t, []int{1, 2}, getPossibleCheckpoints([]int{1, 2, 5}, 0, 3))
   158  	require.Equal(t, []int{1, 2}, getPossibleCheckpoints([]int{1, 2, 5}, 0, 4))
   159  
   160  	require.Equal(t, []int{1, 2, 5}, getPossibleCheckpoints([]int{1, 2, 5}, 0, 5))
   161  	require.Equal(t, []int{1, 2, 5}, getPossibleCheckpoints([]int{1, 2, 5}, 0, 6))
   162  	require.Equal(t, []int{1, 2, 5}, getPossibleCheckpoints([]int{1, 2, 5}, 1, 6))
   163  
   164  	require.Equal(t, []int{2, 5}, getPossibleCheckpoints([]int{1, 2, 5}, 2, 6))
   165  	require.Equal(t, []int{2}, getPossibleCheckpoints([]int{1, 2, 5}, 2, 3))
   166  	require.Equal(t, []int{2}, getPossibleCheckpoints([]int{1, 2, 5}, 2, 4))
   167  	require.Equal(t, []int{2, 5}, getPossibleCheckpoints([]int{1, 2, 5}, 2, 5))
   168  
   169  	require.Equal(t, []int{5}, getPossibleCheckpoints([]int{1, 2, 5}, 3, 5))
   170  	require.Equal(t, []int{5}, getPossibleCheckpoints([]int{1, 2, 5}, 3, 6))
   171  
   172  	require.Equal(t, []int{5}, getPossibleCheckpoints([]int{1, 2, 5}, 5, 5))
   173  	require.Equal(t, []int{}, getPossibleCheckpoints([]int{1, 2, 5}, 6, 6))
   174  
   175  }