github.com/hasnat/dolt/go@v0.0.0-20210628190320-9eb5d843fbb7/store/nbs/dynamo_manifest_test.go (about) 1 // Copyright 2019 Dolthub, Inc. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 // 15 // This file incorporates work covered by the following copyright and 16 // permission notice: 17 // 18 // Copyright 2016 Attic Labs, Inc. All rights reserved. 19 // Licensed under the Apache License, version 2.0: 20 // http://www.apache.org/licenses/LICENSE-2.0 21 22 package nbs 23 24 import ( 25 "context" 26 "testing" 27 28 "github.com/stretchr/testify/assert" 29 "github.com/stretchr/testify/require" 30 31 "github.com/dolthub/dolt/go/store/constants" 32 "github.com/dolthub/dolt/go/store/hash" 33 ) 34 35 const ( 36 table = "testTable" 37 db = "testDB" 38 ) 39 40 func makeDynamoManifestFake(t *testing.T) (mm manifest, ddb *fakeDDB) { 41 ddb = makeFakeDDB(t) 42 mm = newDynamoManifest(table, db, ddb) 43 return 44 } 45 46 func TestDynamoManifestParseIfExists(t *testing.T) { 47 assert := assert.New(t) 48 mm, ddb := makeDynamoManifestFake(t) 49 stats := &Stats{} 50 51 exists, _, err := mm.ParseIfExists(context.Background(), stats, nil) 52 require.NoError(t, err) 53 assert.False(exists) 54 55 // Simulate another process writing a manifest and appendix (with an old Noms version). 56 newLock := computeAddr([]byte("locker")) 57 newRoot := hash.Of([]byte("new root")) 58 tableName := hash.Of([]byte("table1")) 59 app := tableName.String() + ":" + "0" 60 specsWithAppendix := app + ":" + tableName.String() + ":" + "0" 61 ddb.putRecord(db, newLock[:], newRoot[:], "0", specsWithAppendix, app) 62 63 // ParseIfExists should now reflect the manifest written above. 64 exists, contents, err := mm.ParseIfExists(context.Background(), stats, nil) 65 require.NoError(t, err) 66 assert.True(exists) 67 assert.Equal("0", contents.vers) 68 assert.Equal(newLock, contents.lock) 69 assert.Equal(newRoot, contents.root) 70 if assert.Len(contents.appendix, 1) { 71 assert.Equal(tableName.String(), contents.specs[0].name.String()) 72 assert.Equal(uint32(0), contents.specs[0].chunkCount) 73 assert.Equal(tableName.String(), contents.appendix[0].name.String()) 74 assert.Equal(uint32(0), contents.appendix[0].chunkCount) 75 } 76 if assert.Len(contents.specs, 2) { 77 assert.Equal(tableName.String(), contents.specs[1].name.String()) 78 assert.Equal(uint32(0), contents.specs[1].chunkCount) 79 } 80 } 81 82 func makeContents(lock, root string, specs, appendix []tableSpec) manifestContents { 83 return manifestContents{ 84 vers: constants.NomsVersion, 85 lock: computeAddr([]byte(lock)), 86 root: hash.Of([]byte(root)), 87 specs: specs, 88 appendix: appendix, 89 } 90 } 91 92 func TestDynamoManifestUpdateWontClobberOldVersion(t *testing.T) { 93 assert := assert.New(t) 94 mm, ddb := makeDynamoManifestFake(t) 95 stats := &Stats{} 96 97 // Simulate another process having already put old Noms data in dir/. 98 lock := computeAddr([]byte("locker")) 99 badRoot := hash.Of([]byte("bad root")) 100 ddb.putRecord(db, lock[:], badRoot[:], "0", "", "") 101 102 _, err := mm.Update(context.Background(), lock, manifestContents{vers: constants.NomsVersion}, stats, nil) 103 assert.Error(err) 104 } 105 106 func TestDynamoManifestUpdate(t *testing.T) { 107 assert := assert.New(t) 108 mm, ddb := makeDynamoManifestFake(t) 109 stats := &Stats{} 110 111 // First, test winning the race against another process. 112 contents := makeContents("locker", "nuroot", []tableSpec{{computeAddr([]byte("a")), 3}}, nil) 113 upstream, err := mm.Update(context.Background(), addr{}, contents, stats, func() error { 114 // This should fail to get the lock, and therefore _not_ clobber the manifest. So the Update should succeed. 115 lock := computeAddr([]byte("nolock")) 116 newRoot2 := hash.Of([]byte("noroot")) 117 ddb.putRecord(db, lock[:], newRoot2[:], constants.NomsVersion, "", "") 118 return nil 119 }) 120 require.NoError(t, err) 121 assert.Equal(contents.lock, upstream.lock) 122 assert.Equal(contents.root, upstream.root) 123 assert.Equal(contents.specs, upstream.specs) 124 125 // Now, test the case where the optimistic lock fails, and someone else updated the root since last we checked. 126 rejected := makeContents("locker 2", "new root 2", nil, nil) 127 upstream, err = mm.Update(context.Background(), addr{}, rejected, stats, nil) 128 require.NoError(t, err) 129 assert.Equal(contents.lock, upstream.lock) 130 assert.Equal(contents.root, upstream.root) 131 assert.Equal(contents.specs, upstream.specs) 132 upstream, err = mm.Update(context.Background(), upstream.lock, rejected, stats, nil) 133 require.NoError(t, err) 134 assert.Equal(rejected.lock, upstream.lock) 135 assert.Equal(rejected.root, upstream.root) 136 assert.Empty(upstream.specs) 137 138 // Now, test the case where the optimistic lock fails because someone else updated only the tables since last we checked 139 jerkLock := computeAddr([]byte("jerk")) 140 tableName := computeAddr([]byte("table1")) 141 ddb.putRecord(db, jerkLock[:], upstream.root[:], constants.NomsVersion, tableName.String()+":1", "") 142 143 newContents3 := makeContents("locker 3", "new root 3", nil, nil) 144 upstream, err = mm.Update(context.Background(), upstream.lock, newContents3, stats, nil) 145 require.NoError(t, err) 146 assert.Equal(jerkLock, upstream.lock) 147 assert.Equal(rejected.root, upstream.root) 148 assert.Equal([]tableSpec{{tableName, 1}}, upstream.specs) 149 } 150 151 func TestDynamoManifestUpdateAppendix(t *testing.T) { 152 assert := assert.New(t) 153 mm, ddb := makeDynamoManifestFake(t) 154 stats := &Stats{} 155 156 // First, test winning the race against another process. 157 specs := []tableSpec{ 158 {computeAddr([]byte("app-a")), 3}, 159 {computeAddr([]byte("a")), 3}, 160 } 161 162 app := []tableSpec{{computeAddr([]byte("app-a")), 3}} 163 contents := makeContents("locker", "nuroot", specs, app) 164 165 upstream, err := mm.Update(context.Background(), addr{}, contents, stats, func() error { 166 // This should fail to get the lock, and therefore _not_ clobber the manifest. So the Update should succeed. 167 lock := computeAddr([]byte("nolock")) 168 newRoot2 := hash.Of([]byte("noroot")) 169 ddb.putRecord(db, lock[:], newRoot2[:], constants.NomsVersion, "", "") 170 return nil 171 }) 172 require.NoError(t, err) 173 assert.Equal(contents.lock, upstream.lock) 174 assert.Equal(contents.root, upstream.root) 175 assert.Equal(contents.specs, upstream.specs) 176 assert.Equal(contents.appendix, upstream.appendix) 177 178 // Now, test the case where the optimistic lock fails, and someone else updated the root since last we checked. 179 rejected := makeContents("locker 2", "new root 2", nil, nil) 180 upstream, err = mm.Update(context.Background(), addr{}, rejected, stats, nil) 181 require.NoError(t, err) 182 assert.Equal(contents.lock, upstream.lock) 183 assert.Equal(contents.root, upstream.root) 184 assert.Equal(contents.specs, upstream.specs) 185 assert.Equal(contents.appendix, upstream.appendix) 186 187 upstream, err = mm.Update(context.Background(), upstream.lock, rejected, stats, nil) 188 require.NoError(t, err) 189 assert.Equal(rejected.lock, upstream.lock) 190 assert.Equal(rejected.root, upstream.root) 191 assert.Empty(upstream.specs) 192 assert.Empty(upstream.appendix) 193 194 // Now, test the case where the optimistic lock fails because someone else updated only the tables since last we checked 195 jerkLock := computeAddr([]byte("jerk")) 196 tableName := computeAddr([]byte("table1")) 197 appTableName := computeAddr([]byte("table1-appendix")) 198 appStr := appTableName.String() + ":1" 199 specsStr := appStr + ":" + tableName.String() + ":1" 200 ddb.putRecord(db, jerkLock[:], upstream.root[:], constants.NomsVersion, specsStr, appStr) 201 202 newContents3 := makeContents("locker 3", "new root 3", nil, nil) 203 upstream, err = mm.Update(context.Background(), upstream.lock, newContents3, stats, nil) 204 require.NoError(t, err) 205 assert.Equal(jerkLock, upstream.lock) 206 assert.Equal(rejected.root, upstream.root) 207 assert.Equal([]tableSpec{{appTableName, 1}, {tableName, 1}}, upstream.specs) 208 assert.Equal([]tableSpec{{appTableName, 1}}, upstream.appendix) 209 } 210 211 func TestDynamoManifestCaching(t *testing.T) { 212 assert := assert.New(t) 213 mm, ddb := makeDynamoManifestFake(t) 214 stats := &Stats{} 215 216 // ParseIfExists should hit persistent storage no matter what 217 reads := ddb.NumGets() 218 exists, _, err := mm.ParseIfExists(context.Background(), stats, nil) 219 require.NoError(t, err) 220 assert.False(exists) 221 assert.Equal(reads+1, ddb.NumGets()) 222 223 lock, root := computeAddr([]byte("lock")), hash.Of([]byte("root")) 224 ddb.putRecord(db, lock[:], root[:], constants.NomsVersion, "", "") 225 226 reads = ddb.NumGets() 227 exists, _, err = mm.ParseIfExists(context.Background(), stats, nil) 228 require.NoError(t, err) 229 assert.True(exists) 230 assert.Equal(reads+1, ddb.NumGets()) 231 232 // When failing the optimistic lock, we should hit persistent storage. 233 reads = ddb.NumGets() 234 contents := makeContents("lock2", "nuroot", []tableSpec{{computeAddr([]byte("a")), 3}}, nil) 235 upstream, err := mm.Update(context.Background(), addr{}, contents, stats, nil) 236 require.NoError(t, err) 237 assert.NotEqual(contents.lock, upstream.lock) 238 assert.Equal(reads+1, ddb.NumGets()) 239 240 // Successful update should NOT hit persistent storage. 241 reads = ddb.NumGets() 242 upstream, err = mm.Update(context.Background(), upstream.lock, contents, stats, nil) 243 require.NoError(t, err) 244 assert.Equal(contents.lock, upstream.lock) 245 assert.Equal(reads, ddb.NumGets()) 246 } 247 248 func TestDynamoManifestUpdateEmpty(t *testing.T) { 249 assert := assert.New(t) 250 mm, _ := makeDynamoManifestFake(t) 251 stats := &Stats{} 252 253 contents := manifestContents{vers: constants.NomsVersion, lock: computeAddr([]byte{0x01})} 254 upstream, err := mm.Update(context.Background(), addr{}, contents, stats, nil) 255 require.NoError(t, err) 256 assert.Equal(contents.lock, upstream.lock) 257 assert.True(upstream.root.IsEmpty()) 258 assert.Empty(upstream.specs) 259 assert.Empty(upstream.appendix) 260 }