github.com/keybase/client/go@v0.0.0-20241007131713-f10651d043c8/kbfs/libkbfs/journal_md_ops_test.go (about)

     1  // Copyright 2016 Keybase Inc. All rights reserved.
     2  // Use of this source code is governed by a BSD
     3  // license that can be found in the LICENSE file.
     4  
     5  package libkbfs
     6  
     7  import (
     8  	"os"
     9  	"testing"
    10  
    11  	"github.com/keybase/client/go/kbfs/ioutil"
    12  	"github.com/keybase/client/go/kbfs/kbfsmd"
    13  	"github.com/keybase/client/go/kbfs/tlf"
    14  	"github.com/keybase/client/go/kbfs/tlfhandle"
    15  	"github.com/keybase/client/go/protocol/keybase1"
    16  	"github.com/stretchr/testify/assert"
    17  	"github.com/stretchr/testify/require"
    18  	"golang.org/x/net/context"
    19  )
    20  
    21  func setupJournalMDOpsTest(t *testing.T) (
    22  	tempdir string, ctx context.Context, cancel context.CancelFunc,
    23  	config *ConfigLocal, oldMDOps MDOps, jManager *JournalManager) {
    24  	tempdir, err := ioutil.TempDir(os.TempDir(), "journal_md_ops")
    25  	require.NoError(t, err)
    26  
    27  	// Clean up the tempdir if the rest of the setup fails.
    28  	setupSucceeded := false
    29  	defer func() {
    30  		if !setupSucceeded {
    31  			err := ioutil.RemoveAll(tempdir)
    32  			assert.NoError(t, err)
    33  		}
    34  	}()
    35  
    36  	ctx, cancel = context.WithTimeout(
    37  		context.Background(), individualTestTimeout)
    38  
    39  	// Clean up the context if the rest of the setup fails.
    40  	defer func() {
    41  		if !setupSucceeded {
    42  			cancel()
    43  		}
    44  	}()
    45  
    46  	config = MakeTestConfigOrBust(t, "test_user")
    47  
    48  	// Clean up the config if the rest of the setup fails.
    49  	defer func() {
    50  		if !setupSucceeded {
    51  			CheckConfigAndShutdown(ctx, t, config)
    52  		}
    53  	}()
    54  
    55  	oldMDOps = config.MDOps()
    56  	err = config.EnableDiskLimiter(tempdir)
    57  	require.NoError(t, err)
    58  	err = config.EnableJournaling(
    59  		ctx, tempdir, TLFJournalBackgroundWorkEnabled)
    60  	require.NoError(t, err)
    61  	jManager, err = GetJournalManager(config)
    62  	// Turn off listeners to avoid background MD pushes for CR.
    63  	jManager.onBranchChange = nil
    64  	jManager.onMDFlush = nil
    65  	require.NoError(t, err)
    66  
    67  	// Tests need to explicitly enable journaling, to avoid races
    68  	// where journals are enabled before they can be paused.
    69  	err = jManager.DisableAuto(ctx)
    70  	require.NoError(t, err)
    71  
    72  	setupSucceeded = true
    73  	return tempdir, ctx, cancel, config, oldMDOps, jManager
    74  }
    75  
    76  func teardownJournalMDOpsTest(
    77  	ctx context.Context, t *testing.T, tempdir string,
    78  	cancel context.CancelFunc, config Config) {
    79  	CheckConfigAndShutdown(ctx, t, config)
    80  	cancel()
    81  	err := ioutil.RemoveAll(tempdir)
    82  	assert.NoError(t, err)
    83  }
    84  
    85  func makeMDForJournalMDOpsTest(
    86  	t *testing.T, config Config, tlfID tlf.ID, h *tlfhandle.Handle,
    87  	revision kbfsmd.Revision) *RootMetadata {
    88  	rmd, err := makeInitialRootMetadata(config.MetadataVersion(), tlfID, h)
    89  	require.NoError(t, err)
    90  	rmd.SetRevision(revision)
    91  	ctx := context.Background()
    92  	rekeyDone, _, err := config.KeyManager().Rekey(ctx, rmd, false)
    93  	require.NoError(t, err)
    94  	require.True(t, rekeyDone)
    95  	return rmd
    96  }
    97  
    98  // TODO: Clean up the test below.
    99  
   100  func TestJournalMDOpsBasics(t *testing.T) {
   101  	tempdir, ctx, cancel, config, oldMDOps, jManager := setupJournalMDOpsTest(t)
   102  	defer teardownJournalMDOpsTest(ctx, t, tempdir, cancel, config)
   103  
   104  	session, err := config.KBPKI().GetCurrentSession(ctx)
   105  	require.NoError(t, err)
   106  
   107  	// (1) get metadata -- allocates an ID
   108  	bh, err := tlf.MakeHandle(
   109  		[]keybase1.UserOrTeamID{session.UID.AsUserOrTeam()}, nil, nil, nil, nil)
   110  	require.NoError(t, err)
   111  
   112  	h, err := tlfhandle.MakeHandle(
   113  		ctx, bh, bh.Type(), config.KBPKI(), config.KBPKI(), nil,
   114  		keybase1.OfflineAvailability_NONE)
   115  	require.NoError(t, err)
   116  
   117  	mdOps := jManager.mdOps()
   118  
   119  	id, err := mdOps.GetIDForHandle(ctx, h)
   120  	require.NoError(t, err)
   121  	require.NotEqual(t, tlf.NullID, id)
   122  	irmd, err := mdOps.GetForTLF(ctx, id, nil)
   123  	require.NoError(t, err)
   124  	require.Equal(t, ImmutableRootMetadata{}, irmd)
   125  	h.SetTlfID(id)
   126  
   127  	err = jManager.Enable(ctx, id, nil, TLFJournalBackgroundWorkPaused)
   128  	require.NoError(t, err)
   129  
   130  	rmd := makeMDForJournalMDOpsTest(t, config, id, h, kbfsmd.Revision(1))
   131  
   132  	irmd, err = mdOps.Put(
   133  		ctx, rmd, session.VerifyingKey, nil, keybase1.MDPriorityNormal, nil)
   134  	require.NoError(t, err)
   135  	prevRoot := irmd.mdID
   136  
   137  	// (2) push some new metadata blocks
   138  	for i := kbfsmd.Revision(2); i < 8; i++ {
   139  		rmd.SetRevision(i)
   140  		rmd.SetPrevRoot(prevRoot)
   141  		irmd, err := mdOps.Put(
   142  			ctx, rmd, session.VerifyingKey, nil, keybase1.MDPriorityNormal, nil)
   143  		require.NoError(t, err, "i=%d", i)
   144  		prevRoot = irmd.mdID
   145  	}
   146  
   147  	head, err := mdOps.GetForTLF(ctx, id, nil)
   148  	require.NoError(t, err)
   149  	require.NotEqual(t, ImmutableRootMetadata{}, head)
   150  	require.Equal(t, kbfsmd.Revision(7), head.Revision())
   151  
   152  	head, err = oldMDOps.GetForTLF(ctx, id, nil)
   153  	require.NoError(t, err)
   154  	require.Equal(t, ImmutableRootMetadata{}, head)
   155  
   156  	err = jManager.Flush(ctx, id)
   157  	require.NoError(t, err)
   158  
   159  	head, err = mdOps.GetForTLF(ctx, id, nil)
   160  	require.NoError(t, err)
   161  	require.NotEqual(t, ImmutableRootMetadata{}, head)
   162  	require.Equal(t, kbfsmd.Revision(7), head.Revision())
   163  
   164  	head, err = oldMDOps.GetForTLF(ctx, id, nil)
   165  	require.NoError(t, err)
   166  	require.NotEqual(t, ImmutableRootMetadata{}, head)
   167  	require.Equal(t, kbfsmd.Revision(7), head.Revision())
   168  
   169  	// (3) trigger a conflict
   170  	rmd.SetRevision(kbfsmd.Revision(8))
   171  	rmd.SetPrevRoot(prevRoot)
   172  	resolveMD, err := rmd.deepCopy(config.Codec())
   173  	require.NoError(t, err)
   174  	_, err = oldMDOps.Put(
   175  		ctx, rmd, session.VerifyingKey, nil, keybase1.MDPriorityNormal, nil)
   176  	require.NoError(t, err)
   177  
   178  	for i := kbfsmd.Revision(8); i <= 10; i++ {
   179  		rmd.SetRevision(i)
   180  		rmd.SetPrevRoot(prevRoot)
   181  		irmd, err := mdOps.Put(
   182  			ctx, rmd, session.VerifyingKey, nil, keybase1.MDPriorityNormal, nil)
   183  		require.NoError(t, err, "i=%d", i)
   184  		prevRoot = irmd.mdID
   185  	}
   186  
   187  	err = jManager.Flush(ctx, id)
   188  	require.NoError(t, err)
   189  
   190  	head, err = mdOps.GetForTLF(ctx, id, nil)
   191  	require.NoError(t, err)
   192  	require.NotEqual(t, ImmutableRootMetadata{}, head)
   193  	require.Equal(t, kbfsmd.Revision(8), head.Revision())
   194  
   195  	head, err = oldMDOps.GetForTLF(ctx, id, nil)
   196  	require.NoError(t, err)
   197  	require.NotEqual(t, ImmutableRootMetadata{}, head)
   198  	require.Equal(t, kbfsmd.Revision(8), head.Revision())
   199  
   200  	// Find the branch ID.
   201  	tlfJournal, ok := jManager.getTLFJournal(id, nil)
   202  	require.True(t, ok)
   203  	bid := tlfJournal.mdJournal.branchID
   204  
   205  	head, err = mdOps.GetUnmergedForTLF(ctx, id, bid)
   206  	require.NoError(t, err)
   207  	require.NotEqual(t, ImmutableRootMetadata{}, head)
   208  	require.Equal(t, kbfsmd.Revision(10), head.Revision())
   209  	require.Equal(t, bid, head.BID())
   210  
   211  	// (4) push some new unmerged metadata blocks linking to the
   212  	//     middle merged block.
   213  	for i := kbfsmd.Revision(11); i < 41; i++ {
   214  		rmd.SetRevision(i)
   215  		rmd.SetPrevRoot(prevRoot)
   216  		irmd, err := mdOps.PutUnmerged(ctx, rmd, session.VerifyingKey, nil)
   217  		require.NoError(t, err, "i=%d", i)
   218  		prevRoot = irmd.mdID
   219  		require.Equal(t, bid, rmd.BID())
   220  		bid = rmd.BID()
   221  		require.NoError(t, err)
   222  	}
   223  
   224  	// (5) check for proper unmerged head
   225  	head, err = mdOps.GetUnmergedForTLF(ctx, id, bid)
   226  	require.NoError(t, err)
   227  	require.NotEqual(t, ImmutableRootMetadata{}, head)
   228  	require.Equal(t, kbfsmd.Revision(40), head.Revision())
   229  
   230  	// (6a) try to get unmerged range
   231  	rmdses, err := mdOps.GetUnmergedRange(ctx, id, bid, 1, 100)
   232  	require.NoError(t, err)
   233  	require.Equal(t, 33, len(rmdses))
   234  	for i := kbfsmd.Revision(8); i < 41; i++ {
   235  		require.Equal(t, i, rmdses[i-8].Revision())
   236  	}
   237  
   238  	// (6b) try to get unmerged range subset.
   239  	rmdses, err = mdOps.GetUnmergedRange(ctx, id, bid, 7, 14)
   240  	require.NoError(t, err)
   241  	require.Equal(t, 7, len(rmdses))
   242  	for i := kbfsmd.Revision(8); i <= 14; i++ {
   243  		require.Equal(t, i, rmdses[i-8].Revision())
   244  	}
   245  
   246  	// (7) resolve the branch
   247  	_, err = mdOps.ResolveBranch(
   248  		ctx, id, bid, nil, resolveMD, session.VerifyingKey, nil)
   249  	require.NoError(t, err)
   250  
   251  	// (8) verify head is pruned
   252  	head, err = mdOps.GetUnmergedForTLF(ctx, id, kbfsmd.NullBranchID)
   253  	require.NoError(t, err)
   254  	require.Equal(t, ImmutableRootMetadata{}, head)
   255  
   256  	// (9) verify revision history is pruned
   257  	rmdses, err = mdOps.GetUnmergedRange(ctx, id, kbfsmd.NullBranchID, 1, 100)
   258  	require.NoError(t, err)
   259  	require.Equal(t, 0, len(rmdses))
   260  
   261  	// (10) check for proper merged head
   262  	head, err = mdOps.GetForTLF(ctx, id, nil)
   263  	require.NoError(t, err)
   264  	require.NotEqual(t, ImmutableRootMetadata{}, head)
   265  	require.Equal(t, kbfsmd.Revision(8), head.Revision())
   266  
   267  	// (11) try to get merged range
   268  	rmdses, err = mdOps.GetRange(ctx, id, 1, 100, nil)
   269  	require.NoError(t, err)
   270  	require.Equal(t, 8, len(rmdses))
   271  	for i := kbfsmd.Revision(1); i <= 8; i++ {
   272  		require.Equal(t, i, rmdses[i-1].Revision())
   273  	}
   274  }
   275  
   276  // TODO: Add a test for GetRange where the server has an overlapping
   277  // range with the journal.
   278  
   279  func TestJournalMDOpsPutUnmerged(t *testing.T) {
   280  	tempdir, ctx, cancel, config, _, jManager := setupJournalMDOpsTest(t)
   281  	defer teardownJournalMDOpsTest(ctx, t, tempdir, cancel, config)
   282  
   283  	session, err := config.KBPKI().GetCurrentSession(ctx)
   284  	require.NoError(t, err)
   285  
   286  	bh, err := tlf.MakeHandle(
   287  		[]keybase1.UserOrTeamID{session.UID.AsUserOrTeam()}, nil, nil, nil, nil)
   288  	require.NoError(t, err)
   289  
   290  	h, err := tlfhandle.MakeHandle(
   291  		ctx, bh, bh.Type(), config.KBPKI(), config.KBPKI(), nil,
   292  		keybase1.OfflineAvailability_NONE)
   293  	require.NoError(t, err)
   294  
   295  	mdOps := jManager.mdOps()
   296  
   297  	id, err := mdOps.GetIDForHandle(ctx, h)
   298  	require.NoError(t, err)
   299  	require.NotEqual(t, tlf.NullID, id)
   300  	irmd, err := mdOps.GetForTLF(ctx, id, nil)
   301  	require.NoError(t, err)
   302  	require.Equal(t, ImmutableRootMetadata{}, irmd)
   303  
   304  	err = jManager.Enable(ctx, id, nil, TLFJournalBackgroundWorkPaused)
   305  	require.NoError(t, err)
   306  
   307  	rmd := makeMDForJournalMDOpsTest(t, config, id, h, kbfsmd.Revision(2))
   308  	rmd.SetPrevRoot(kbfsmd.FakeID(1))
   309  	rmd.SetBranchID(kbfsmd.FakeBranchID(1))
   310  
   311  	_, err = mdOps.PutUnmerged(ctx, rmd, session.VerifyingKey, nil)
   312  	require.NoError(t, err)
   313  }
   314  
   315  func TestJournalMDOpsPutUnmergedError(t *testing.T) {
   316  	tempdir, ctx, cancel, config, _, jManager := setupJournalMDOpsTest(t)
   317  	defer teardownJournalMDOpsTest(ctx, t, tempdir, cancel, config)
   318  
   319  	session, err := config.KBPKI().GetCurrentSession(ctx)
   320  	require.NoError(t, err)
   321  
   322  	bh, err := tlf.MakeHandle(
   323  		[]keybase1.UserOrTeamID{session.UID.AsUserOrTeam()}, nil, nil, nil, nil)
   324  	require.NoError(t, err)
   325  
   326  	h, err := tlfhandle.MakeHandle(
   327  		ctx, bh, bh.Type(), config.KBPKI(), config.KBPKI(), nil,
   328  		keybase1.OfflineAvailability_NONE)
   329  	require.NoError(t, err)
   330  
   331  	mdOps := jManager.mdOps()
   332  
   333  	id, err := mdOps.GetIDForHandle(ctx, h)
   334  	require.NoError(t, err)
   335  	require.NotEqual(t, tlf.NullID, id)
   336  	irmd, err := mdOps.GetForTLF(ctx, id, nil)
   337  	require.NoError(t, err)
   338  	require.Equal(t, ImmutableRootMetadata{}, irmd)
   339  
   340  	err = jManager.Enable(ctx, id, nil, TLFJournalBackgroundWorkPaused)
   341  	require.NoError(t, err)
   342  
   343  	rmd := makeMDForJournalMDOpsTest(t, config, id, h, kbfsmd.Revision(1))
   344  
   345  	_, err = mdOps.PutUnmerged(ctx, rmd, session.VerifyingKey, nil)
   346  	require.Error(t, err, "Unmerged put with rmd.BID() == j.branchID == kbfsmd.NullBranchID")
   347  }
   348  
   349  func TestJournalMDOpsLocalSquashBranch(t *testing.T) {
   350  	tempdir, ctx, cancel, config, _, jManager := setupJournalMDOpsTest(t)
   351  	defer teardownJournalMDOpsTest(ctx, t, tempdir, cancel, config)
   352  
   353  	session, err := config.KBPKI().GetCurrentSession(ctx)
   354  	require.NoError(t, err)
   355  
   356  	bh, err := tlf.MakeHandle(
   357  		[]keybase1.UserOrTeamID{session.UID.AsUserOrTeam()}, nil, nil, nil, nil)
   358  	require.NoError(t, err)
   359  
   360  	h, err := tlfhandle.MakeHandle(
   361  		ctx, bh, bh.Type(), config.KBPKI(), config.KBPKI(), nil,
   362  		keybase1.OfflineAvailability_NONE)
   363  	require.NoError(t, err)
   364  
   365  	mdOps := jManager.mdOps()
   366  	id, err := mdOps.GetIDForHandle(ctx, h)
   367  	require.NoError(t, err)
   368  	irmd, err := mdOps.GetForTLF(ctx, id, nil)
   369  	require.NoError(t, err)
   370  	require.Equal(t, ImmutableRootMetadata{}, irmd)
   371  	err = jManager.Enable(ctx, id, nil, TLFJournalBackgroundWorkPaused)
   372  	require.NoError(t, err)
   373  
   374  	tlfJournal, ok := jManager.getTLFJournal(id, nil)
   375  	require.True(t, ok)
   376  
   377  	// Prepare the md journal to have a leading local squash revision.
   378  	firstRevision := kbfsmd.Revision(1)
   379  	initialRmd := makeMDForJournalMDOpsTest(t, config, id, h, firstRevision)
   380  	j := tlfJournal.mdJournal
   381  	initialMdID, _, err := j.put(ctx, config.Crypto(), config.KeyManager(),
   382  		config.BlockSplitter(), initialRmd, true)
   383  	require.NoError(t, err)
   384  
   385  	mdCount := 10
   386  	rmd := initialRmd
   387  	mdID := initialMdID
   388  	// Put several MDs after a local squash
   389  	for i := 0; i < mdCount; i++ {
   390  		rmd, err = rmd.MakeSuccessor(ctx, config.MetadataVersion(),
   391  			config.Codec(), config.KeyManager(),
   392  			config.KBPKI(), config.KBPKI(), config, mdID, true)
   393  		require.NoError(t, err)
   394  		mdID, _, err = j.put(ctx, config.Crypto(), config.KeyManager(),
   395  			config.BlockSplitter(), rmd, false)
   396  		require.NoError(t, err)
   397  	}
   398  
   399  	mdcache := NewMDCacheStandard(10)
   400  	err = j.convertToBranch(
   401  		ctx, kbfsmd.PendingLocalSquashBranchID, config.Crypto(), config.Codec(),
   402  		id, mdcache)
   403  	require.NoError(t, err)
   404  
   405  	// The merged head should still be the initial rmd, because we
   406  	// marked it as a squash and it shouldn't have gotten converted.
   407  	irmd, err = mdOps.GetForTLF(ctx, id, nil)
   408  	require.NoError(t, err)
   409  	require.Equal(t, initialMdID, irmd.mdID)
   410  	require.Equal(t, firstRevision, irmd.Revision())
   411  
   412  	// The unmerged head should be the last MD we put, converted to a
   413  	// branch.
   414  	irmd, err = mdOps.GetUnmergedForTLF(ctx, id, kbfsmd.PendingLocalSquashBranchID)
   415  	require.NoError(t, err)
   416  	require.Equal(t, rmd.Revision(), irmd.Revision())
   417  	require.Equal(t, kbfsmd.PendingLocalSquashBranchID, irmd.BID())
   418  
   419  	// The merged range should just be the initial MD.
   420  	stopRevision := firstRevision + kbfsmd.Revision(mdCount*2)
   421  	irmds, err := mdOps.GetRange(ctx, id, firstRevision, stopRevision, nil)
   422  	require.NoError(t, err)
   423  	require.Len(t, irmds, 1)
   424  	require.Equal(t, initialMdID, irmds[0].mdID)
   425  	require.Equal(t, firstRevision, irmds[0].Revision())
   426  
   427  	irmds, err = mdOps.GetUnmergedRange(ctx, id, kbfsmd.PendingLocalSquashBranchID,
   428  		firstRevision, stopRevision)
   429  	require.NoError(t, err)
   430  	require.Len(t, irmds, mdCount)
   431  	require.Equal(t, firstRevision+kbfsmd.Revision(1), irmds[0].Revision())
   432  }