github.com/hasnat/dolt/go@v0.0.0-20210628190320-9eb5d843fbb7/store/nbs/dynamo_manifest_test.go (about)

     1  // Copyright 2019 Dolthub, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  //
    15  // This file incorporates work covered by the following copyright and
    16  // permission notice:
    17  //
    18  // Copyright 2016 Attic Labs, Inc. All rights reserved.
    19  // Licensed under the Apache License, version 2.0:
    20  // http://www.apache.org/licenses/LICENSE-2.0
    21  
    22  package nbs
    23  
    24  import (
    25  	"context"
    26  	"testing"
    27  
    28  	"github.com/stretchr/testify/assert"
    29  	"github.com/stretchr/testify/require"
    30  
    31  	"github.com/dolthub/dolt/go/store/constants"
    32  	"github.com/dolthub/dolt/go/store/hash"
    33  )
    34  
    35  const (
    36  	table = "testTable"
    37  	db    = "testDB"
    38  )
    39  
    40  func makeDynamoManifestFake(t *testing.T) (mm manifest, ddb *fakeDDB) {
    41  	ddb = makeFakeDDB(t)
    42  	mm = newDynamoManifest(table, db, ddb)
    43  	return
    44  }
    45  
    46  func TestDynamoManifestParseIfExists(t *testing.T) {
    47  	assert := assert.New(t)
    48  	mm, ddb := makeDynamoManifestFake(t)
    49  	stats := &Stats{}
    50  
    51  	exists, _, err := mm.ParseIfExists(context.Background(), stats, nil)
    52  	require.NoError(t, err)
    53  	assert.False(exists)
    54  
    55  	// Simulate another process writing a manifest and appendix (with an old Noms version).
    56  	newLock := computeAddr([]byte("locker"))
    57  	newRoot := hash.Of([]byte("new root"))
    58  	tableName := hash.Of([]byte("table1"))
    59  	app := tableName.String() + ":" + "0"
    60  	specsWithAppendix := app + ":" + tableName.String() + ":" + "0"
    61  	ddb.putRecord(db, newLock[:], newRoot[:], "0", specsWithAppendix, app)
    62  
    63  	// ParseIfExists should now reflect the manifest written above.
    64  	exists, contents, err := mm.ParseIfExists(context.Background(), stats, nil)
    65  	require.NoError(t, err)
    66  	assert.True(exists)
    67  	assert.Equal("0", contents.vers)
    68  	assert.Equal(newLock, contents.lock)
    69  	assert.Equal(newRoot, contents.root)
    70  	if assert.Len(contents.appendix, 1) {
    71  		assert.Equal(tableName.String(), contents.specs[0].name.String())
    72  		assert.Equal(uint32(0), contents.specs[0].chunkCount)
    73  		assert.Equal(tableName.String(), contents.appendix[0].name.String())
    74  		assert.Equal(uint32(0), contents.appendix[0].chunkCount)
    75  	}
    76  	if assert.Len(contents.specs, 2) {
    77  		assert.Equal(tableName.String(), contents.specs[1].name.String())
    78  		assert.Equal(uint32(0), contents.specs[1].chunkCount)
    79  	}
    80  }
    81  
    82  func makeContents(lock, root string, specs, appendix []tableSpec) manifestContents {
    83  	return manifestContents{
    84  		vers:     constants.NomsVersion,
    85  		lock:     computeAddr([]byte(lock)),
    86  		root:     hash.Of([]byte(root)),
    87  		specs:    specs,
    88  		appendix: appendix,
    89  	}
    90  }
    91  
    92  func TestDynamoManifestUpdateWontClobberOldVersion(t *testing.T) {
    93  	assert := assert.New(t)
    94  	mm, ddb := makeDynamoManifestFake(t)
    95  	stats := &Stats{}
    96  
    97  	// Simulate another process having already put old Noms data in dir/.
    98  	lock := computeAddr([]byte("locker"))
    99  	badRoot := hash.Of([]byte("bad root"))
   100  	ddb.putRecord(db, lock[:], badRoot[:], "0", "", "")
   101  
   102  	_, err := mm.Update(context.Background(), lock, manifestContents{vers: constants.NomsVersion}, stats, nil)
   103  	assert.Error(err)
   104  }
   105  
   106  func TestDynamoManifestUpdate(t *testing.T) {
   107  	assert := assert.New(t)
   108  	mm, ddb := makeDynamoManifestFake(t)
   109  	stats := &Stats{}
   110  
   111  	// First, test winning the race against another process.
   112  	contents := makeContents("locker", "nuroot", []tableSpec{{computeAddr([]byte("a")), 3}}, nil)
   113  	upstream, err := mm.Update(context.Background(), addr{}, contents, stats, func() error {
   114  		// This should fail to get the lock, and therefore _not_ clobber the manifest. So the Update should succeed.
   115  		lock := computeAddr([]byte("nolock"))
   116  		newRoot2 := hash.Of([]byte("noroot"))
   117  		ddb.putRecord(db, lock[:], newRoot2[:], constants.NomsVersion, "", "")
   118  		return nil
   119  	})
   120  	require.NoError(t, err)
   121  	assert.Equal(contents.lock, upstream.lock)
   122  	assert.Equal(contents.root, upstream.root)
   123  	assert.Equal(contents.specs, upstream.specs)
   124  
   125  	// Now, test the case where the optimistic lock fails, and someone else updated the root since last we checked.
   126  	rejected := makeContents("locker 2", "new root 2", nil, nil)
   127  	upstream, err = mm.Update(context.Background(), addr{}, rejected, stats, nil)
   128  	require.NoError(t, err)
   129  	assert.Equal(contents.lock, upstream.lock)
   130  	assert.Equal(contents.root, upstream.root)
   131  	assert.Equal(contents.specs, upstream.specs)
   132  	upstream, err = mm.Update(context.Background(), upstream.lock, rejected, stats, nil)
   133  	require.NoError(t, err)
   134  	assert.Equal(rejected.lock, upstream.lock)
   135  	assert.Equal(rejected.root, upstream.root)
   136  	assert.Empty(upstream.specs)
   137  
   138  	// Now, test the case where the optimistic lock fails because someone else updated only the tables since last we checked
   139  	jerkLock := computeAddr([]byte("jerk"))
   140  	tableName := computeAddr([]byte("table1"))
   141  	ddb.putRecord(db, jerkLock[:], upstream.root[:], constants.NomsVersion, tableName.String()+":1", "")
   142  
   143  	newContents3 := makeContents("locker 3", "new root 3", nil, nil)
   144  	upstream, err = mm.Update(context.Background(), upstream.lock, newContents3, stats, nil)
   145  	require.NoError(t, err)
   146  	assert.Equal(jerkLock, upstream.lock)
   147  	assert.Equal(rejected.root, upstream.root)
   148  	assert.Equal([]tableSpec{{tableName, 1}}, upstream.specs)
   149  }
   150  
   151  func TestDynamoManifestUpdateAppendix(t *testing.T) {
   152  	assert := assert.New(t)
   153  	mm, ddb := makeDynamoManifestFake(t)
   154  	stats := &Stats{}
   155  
   156  	// First, test winning the race against another process.
   157  	specs := []tableSpec{
   158  		{computeAddr([]byte("app-a")), 3},
   159  		{computeAddr([]byte("a")), 3},
   160  	}
   161  
   162  	app := []tableSpec{{computeAddr([]byte("app-a")), 3}}
   163  	contents := makeContents("locker", "nuroot", specs, app)
   164  
   165  	upstream, err := mm.Update(context.Background(), addr{}, contents, stats, func() error {
   166  		// This should fail to get the lock, and therefore _not_ clobber the manifest. So the Update should succeed.
   167  		lock := computeAddr([]byte("nolock"))
   168  		newRoot2 := hash.Of([]byte("noroot"))
   169  		ddb.putRecord(db, lock[:], newRoot2[:], constants.NomsVersion, "", "")
   170  		return nil
   171  	})
   172  	require.NoError(t, err)
   173  	assert.Equal(contents.lock, upstream.lock)
   174  	assert.Equal(contents.root, upstream.root)
   175  	assert.Equal(contents.specs, upstream.specs)
   176  	assert.Equal(contents.appendix, upstream.appendix)
   177  
   178  	// Now, test the case where the optimistic lock fails, and someone else updated the root since last we checked.
   179  	rejected := makeContents("locker 2", "new root 2", nil, nil)
   180  	upstream, err = mm.Update(context.Background(), addr{}, rejected, stats, nil)
   181  	require.NoError(t, err)
   182  	assert.Equal(contents.lock, upstream.lock)
   183  	assert.Equal(contents.root, upstream.root)
   184  	assert.Equal(contents.specs, upstream.specs)
   185  	assert.Equal(contents.appendix, upstream.appendix)
   186  
   187  	upstream, err = mm.Update(context.Background(), upstream.lock, rejected, stats, nil)
   188  	require.NoError(t, err)
   189  	assert.Equal(rejected.lock, upstream.lock)
   190  	assert.Equal(rejected.root, upstream.root)
   191  	assert.Empty(upstream.specs)
   192  	assert.Empty(upstream.appendix)
   193  
   194  	// Now, test the case where the optimistic lock fails because someone else updated only the tables since last we checked
   195  	jerkLock := computeAddr([]byte("jerk"))
   196  	tableName := computeAddr([]byte("table1"))
   197  	appTableName := computeAddr([]byte("table1-appendix"))
   198  	appStr := appTableName.String() + ":1"
   199  	specsStr := appStr + ":" + tableName.String() + ":1"
   200  	ddb.putRecord(db, jerkLock[:], upstream.root[:], constants.NomsVersion, specsStr, appStr)
   201  
   202  	newContents3 := makeContents("locker 3", "new root 3", nil, nil)
   203  	upstream, err = mm.Update(context.Background(), upstream.lock, newContents3, stats, nil)
   204  	require.NoError(t, err)
   205  	assert.Equal(jerkLock, upstream.lock)
   206  	assert.Equal(rejected.root, upstream.root)
   207  	assert.Equal([]tableSpec{{appTableName, 1}, {tableName, 1}}, upstream.specs)
   208  	assert.Equal([]tableSpec{{appTableName, 1}}, upstream.appendix)
   209  }
   210  
   211  func TestDynamoManifestCaching(t *testing.T) {
   212  	assert := assert.New(t)
   213  	mm, ddb := makeDynamoManifestFake(t)
   214  	stats := &Stats{}
   215  
   216  	// ParseIfExists should hit persistent storage no matter what
   217  	reads := ddb.NumGets()
   218  	exists, _, err := mm.ParseIfExists(context.Background(), stats, nil)
   219  	require.NoError(t, err)
   220  	assert.False(exists)
   221  	assert.Equal(reads+1, ddb.NumGets())
   222  
   223  	lock, root := computeAddr([]byte("lock")), hash.Of([]byte("root"))
   224  	ddb.putRecord(db, lock[:], root[:], constants.NomsVersion, "", "")
   225  
   226  	reads = ddb.NumGets()
   227  	exists, _, err = mm.ParseIfExists(context.Background(), stats, nil)
   228  	require.NoError(t, err)
   229  	assert.True(exists)
   230  	assert.Equal(reads+1, ddb.NumGets())
   231  
   232  	// When failing the optimistic lock, we should hit persistent storage.
   233  	reads = ddb.NumGets()
   234  	contents := makeContents("lock2", "nuroot", []tableSpec{{computeAddr([]byte("a")), 3}}, nil)
   235  	upstream, err := mm.Update(context.Background(), addr{}, contents, stats, nil)
   236  	require.NoError(t, err)
   237  	assert.NotEqual(contents.lock, upstream.lock)
   238  	assert.Equal(reads+1, ddb.NumGets())
   239  
   240  	// Successful update should NOT hit persistent storage.
   241  	reads = ddb.NumGets()
   242  	upstream, err = mm.Update(context.Background(), upstream.lock, contents, stats, nil)
   243  	require.NoError(t, err)
   244  	assert.Equal(contents.lock, upstream.lock)
   245  	assert.Equal(reads, ddb.NumGets())
   246  }
   247  
   248  func TestDynamoManifestUpdateEmpty(t *testing.T) {
   249  	assert := assert.New(t)
   250  	mm, _ := makeDynamoManifestFake(t)
   251  	stats := &Stats{}
   252  
   253  	contents := manifestContents{vers: constants.NomsVersion, lock: computeAddr([]byte{0x01})}
   254  	upstream, err := mm.Update(context.Background(), addr{}, contents, stats, nil)
   255  	require.NoError(t, err)
   256  	assert.Equal(contents.lock, upstream.lock)
   257  	assert.True(upstream.root.IsEmpty())
   258  	assert.Empty(upstream.specs)
   259  	assert.Empty(upstream.appendix)
   260  }