github.com/keybase/client/go@v0.0.0-20240309051027-028f7c731f8b/kbfs/libkbfs/disk_md_cache_test.go (about)

     1  // Copyright 2018 Keybase Inc. All rights reserved.
     2  // Use of this source code is governed by a BSD
     3  // license that can be found in the LICENSE file.
     4  
     5  package libkbfs
     6  
     7  import (
     8  	"crypto/rand"
     9  
    10  	"os"
    11  	"path/filepath"
    12  	"testing"
    13  	"time"
    14  
    15  	"github.com/keybase/client/go/kbfs/kbfsmd"
    16  	"github.com/keybase/client/go/kbfs/test/clocktest"
    17  	"github.com/keybase/client/go/kbfs/tlf"
    18  	"github.com/stretchr/testify/require"
    19  	"github.com/syndtr/goleveldb/leveldb/storage"
    20  	"golang.org/x/net/context"
    21  )
    22  
    23  type testDiskMDCacheConfig struct {
    24  	codecGetter
    25  	logMaker
    26  }
    27  
    28  func newDiskMDCacheLocalForTestWithStorage(
    29  	t *testing.T, s storage.Storage) *DiskMDCacheLocal {
    30  	cache, err := newDiskMDCacheLocalFromStorage(&testDiskMDCacheConfig{
    31  		newTestCodecGetter(),
    32  		newTestLogMaker(t),
    33  	}, s, modeTest{modeDefault{}})
    34  	require.NoError(t, err)
    35  	err = cache.WaitUntilStarted()
    36  	require.NoError(t, err)
    37  	return cache
    38  }
    39  
    40  func newDiskMDCacheLocalForTest(t *testing.T) (*DiskMDCacheLocal, string) {
    41  	// Use a disk-based level, instead of memory storage, because we
    42  	// want to simulate a restart and memory storages can't be reused.
    43  	tempdir, err := os.MkdirTemp(os.TempDir(), "disk_md_cache")
    44  	require.NoError(t, err)
    45  	s, err := storage.OpenFile(filepath.Join(tempdir, "heads"), false)
    46  	require.NoError(t, err)
    47  
    48  	cache := newDiskMDCacheLocalForTestWithStorage(t, s)
    49  	return cache, tempdir
    50  }
    51  
    52  func shutdownDiskMDCacheTest(cache DiskMDCache, tempdir string) {
    53  	cache.Shutdown(context.Background())
    54  	os.RemoveAll(tempdir)
    55  }
    56  
    57  func makeRandomMDBuf(t *testing.T) []byte {
    58  	b := make([]byte, 10)
    59  	_, err := rand.Read(b)
    60  	require.NoError(t, err)
    61  	return b
    62  }
    63  
    64  func TestDiskMDCacheCommitAndGet(t *testing.T) {
    65  	t.Parallel()
    66  	t.Log("Test that basic MD cache Put and Get operations work.")
    67  	cache, tempdir := newDiskMDCacheLocalForTest(t)
    68  	defer func() {
    69  		shutdownDiskMDCacheTest(cache, tempdir)
    70  	}()
    71  	clock := clocktest.NewTestClockNow()
    72  
    73  	tlf1 := tlf.FakeID(0, tlf.Private)
    74  	ctx := context.Background()
    75  
    76  	buf := makeRandomMDBuf(t)
    77  	now := clock.Now()
    78  	rev := kbfsmd.Revision(1)
    79  
    80  	t.Log("Put an MD into the cache.")
    81  	err := cache.Stage(ctx, tlf1, rev, buf, kbfsmd.ImplicitTeamsVer, now)
    82  	require.NoError(t, err)
    83  	_, _, _, err = cache.Get(ctx, tlf1)
    84  	require.Error(t, err) // not commited yet
    85  	status := cache.Status(ctx)
    86  	require.Equal(t, uint64(0), status.NumMDs)
    87  	require.Equal(t, uint64(1), status.NumStaged)
    88  	err = cache.Commit(ctx, tlf1, rev)
    89  	require.NoError(t, err)
    90  	status = cache.Status(ctx)
    91  	require.Equal(t, uint64(1), status.NumMDs)
    92  	require.Equal(t, uint64(0), status.NumStaged)
    93  
    94  	t.Log("Get an MD from the cache.")
    95  	clock.Add(1 * time.Minute)
    96  	getBuf, getVer, getTime, err := cache.Get(ctx, tlf1)
    97  	require.NoError(t, err)
    98  	require.Equal(t, buf, getBuf)
    99  	require.Equal(t, kbfsmd.ImplicitTeamsVer, getVer)
   100  	require.True(t, now.Equal(getTime))
   101  
   102  	t.Log("Check the meters.")
   103  	status = cache.Status(ctx)
   104  	require.Equal(t, int64(1), status.Hits.Count)
   105  	require.Equal(t, int64(1), status.Misses.Count)
   106  	require.Equal(t, int64(1), status.Puts.Count)
   107  
   108  	t.Log("A second TLF")
   109  	tlf2 := tlf.FakeID(0, tlf.Public)
   110  	now2 := clock.Now()
   111  	buf2 := makeRandomMDBuf(t)
   112  	err = cache.Stage(ctx, tlf2, rev, buf2, kbfsmd.ImplicitTeamsVer, now2)
   113  	require.NoError(t, err)
   114  	err = cache.Commit(ctx, tlf2, rev)
   115  	require.NoError(t, err)
   116  	getBuf2, getVer2, getTime2, err := cache.Get(ctx, tlf2)
   117  	require.NoError(t, err)
   118  	require.Equal(t, buf2, getBuf2)
   119  	require.Equal(t, kbfsmd.ImplicitTeamsVer, getVer2)
   120  	require.True(t, now2.Equal(getTime2))
   121  
   122  	t.Log("Override the first TLF")
   123  	now3 := clock.Now()
   124  	buf3 := makeRandomMDBuf(t)
   125  	rev++
   126  	err = cache.Stage(ctx, tlf1, rev, buf3, kbfsmd.ImplicitTeamsVer, now3)
   127  	require.NoError(t, err)
   128  	err = cache.Commit(ctx, tlf1, rev)
   129  	require.NoError(t, err)
   130  	getBuf3, getVer3, getTime3, err := cache.Get(ctx, tlf1)
   131  	require.NoError(t, err)
   132  	require.Equal(t, buf3, getBuf3)
   133  	require.Equal(t, kbfsmd.ImplicitTeamsVer, getVer3)
   134  	require.True(t, now2.Equal(getTime3))
   135  
   136  	t.Log("Restart the cache and check the stats")
   137  	cache.Shutdown(ctx)
   138  	s, err := storage.OpenFile(filepath.Join(tempdir, "heads"), false)
   139  	require.NoError(t, err)
   140  	cache = newDiskMDCacheLocalForTestWithStorage(t, s)
   141  	status = cache.Status(ctx)
   142  	require.Equal(t, uint64(2), status.NumMDs)
   143  	require.Equal(t, uint64(0), status.NumStaged)
   144  }
   145  
   146  func TestDiskMDCacheMultiStage(t *testing.T) {
   147  	t.Parallel()
   148  	t.Log("Stage multiple MDs, but only commit some of them.")
   149  	cache, tempdir := newDiskMDCacheLocalForTest(t)
   150  	defer func() {
   151  		shutdownDiskMDCacheTest(cache, tempdir)
   152  	}()
   153  	clock := clocktest.NewTestClockNow()
   154  
   155  	tlf1 := tlf.FakeID(0, tlf.Private)
   156  	ctx := context.Background()
   157  
   158  	buf1 := makeRandomMDBuf(t)
   159  	now := clock.Now()
   160  	rev1 := kbfsmd.Revision(1)
   161  
   162  	err := cache.Stage(ctx, tlf1, rev1, buf1, kbfsmd.ImplicitTeamsVer, now)
   163  	require.NoError(t, err)
   164  
   165  	t.Log("Out of order stages")
   166  	buf2 := makeRandomMDBuf(t)
   167  	rev2 := rev1 + 1
   168  	buf3 := makeRandomMDBuf(t)
   169  	rev3 := rev2 + 1
   170  	buf4 := makeRandomMDBuf(t)
   171  	rev4 := rev3 + 1
   172  	buf5 := makeRandomMDBuf(t)
   173  	rev5 := rev4 + 1
   174  
   175  	err = cache.Stage(ctx, tlf1, rev4, buf4, kbfsmd.ImplicitTeamsVer, now)
   176  	require.NoError(t, err)
   177  	err = cache.Stage(ctx, tlf1, rev5, buf5, kbfsmd.ImplicitTeamsVer, now)
   178  	require.NoError(t, err)
   179  	err = cache.Stage(ctx, tlf1, rev3, buf3, kbfsmd.ImplicitTeamsVer, now)
   180  	require.NoError(t, err)
   181  	err = cache.Stage(ctx, tlf1, rev2, buf2, kbfsmd.ImplicitTeamsVer, now)
   182  	require.NoError(t, err)
   183  
   184  	t.Log("Duplicate stage")
   185  	err = cache.Stage(ctx, tlf1, rev4, buf4, kbfsmd.ImplicitTeamsVer, now)
   186  	require.NoError(t, err)
   187  
   188  	status := cache.Status(ctx)
   189  	require.Equal(t, uint64(0), status.NumMDs)
   190  	require.Equal(t, uint64(6), status.NumStaged)
   191  
   192  	err = cache.Commit(ctx, tlf1, rev4)
   193  	require.NoError(t, err)
   194  	status = cache.Status(ctx)
   195  	require.Equal(t, uint64(1), status.NumMDs)
   196  	require.Equal(t, uint64(1), status.NumStaged)
   197  
   198  	err = cache.Unstage(ctx, tlf1, rev5)
   199  	require.NoError(t, err)
   200  	status = cache.Status(ctx)
   201  	require.Equal(t, uint64(1), status.NumMDs)
   202  	require.Equal(t, uint64(0), status.NumStaged)
   203  
   204  	getBuf, _, _, err := cache.Get(ctx, tlf1)
   205  	require.NoError(t, err)
   206  	require.Equal(t, buf4, getBuf)
   207  }