github.com/hasnat/dolt/go@v0.0.0-20210628190320-9eb5d843fbb7/store/nbs/file_manifest_test.go (about) 1 // Copyright 2019 Dolthub, Inc. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 // 15 // This file incorporates work covered by the following copyright and 16 // permission notice: 17 // 18 // Copyright 2016 Attic Labs, Inc. All rights reserved. 19 // Licensed under the Apache License, version 2.0: 20 // http://www.apache.org/licenses/LICENSE-2.0 21 22 package nbs 23 24 import ( 25 "context" 26 "io/ioutil" 27 "os" 28 "os/exec" 29 "path/filepath" 30 "runtime" 31 "strings" 32 "testing" 33 34 "github.com/stretchr/testify/assert" 35 "github.com/stretchr/testify/require" 36 37 "github.com/dolthub/dolt/go/store/constants" 38 "github.com/dolthub/dolt/go/store/hash" 39 ) 40 41 func makeFileManifestTempDir(t *testing.T) fileManifestV5 { 42 dir, err := ioutil.TempDir("", "") 43 require.NoError(t, err) 44 return fileManifestV5{dir: dir} //, cache: newManifestCache(defaultManifestCacheSize)} 45 } 46 47 func TestFileManifestLoadIfExists(t *testing.T) { 48 assert := assert.New(t) 49 fm := makeFileManifestTempDir(t) 50 defer os.RemoveAll(fm.dir) 51 stats := &Stats{} 52 53 exists, upstream, err := fm.ParseIfExists(context.Background(), stats, nil) 54 require.NoError(t, err) 55 assert.False(exists) 56 57 // Simulate another process writing a manifest (with an old Noms version). 58 jerk := computeAddr([]byte("jerk")) 59 newRoot := hash.Of([]byte("new root")) 60 tableName := hash.Of([]byte("table1")) 61 gcGen := addr{} 62 m := strings.Join([]string{StorageVersion, "0", jerk.String(), newRoot.String(), gcGen.String(), tableName.String(), "0"}, ":") 63 err = clobberManifest(fm.dir, m) 64 require.NoError(t, err) 65 66 // ParseIfExists should now reflect the manifest written above. 67 exists, upstream, err = fm.ParseIfExists(context.Background(), stats, nil) 68 require.NoError(t, err) 69 assert.True(exists) 70 assert.Equal("0", upstream.vers) 71 assert.Equal(jerk, upstream.lock) 72 assert.Equal(newRoot, upstream.root) 73 if assert.Len(upstream.specs, 1) { 74 assert.Equal(tableName.String(), upstream.specs[0].name.String()) 75 assert.Equal(uint32(0), upstream.specs[0].chunkCount) 76 } 77 } 78 79 func TestFileManifestLoadIfExistsHoldsLock(t *testing.T) { 80 assert := assert.New(t) 81 fm := makeFileManifestTempDir(t) 82 defer os.RemoveAll(fm.dir) 83 stats := &Stats{} 84 85 // Simulate another process writing a manifest. 86 lock := computeAddr([]byte("locker")) 87 newRoot := hash.Of([]byte("new root")) 88 tableName := hash.Of([]byte("table1")) 89 gcGen := addr{} 90 m := strings.Join([]string{StorageVersion, constants.NomsVersion, lock.String(), newRoot.String(), gcGen.String(), tableName.String(), "0"}, ":") 91 err := clobberManifest(fm.dir, m) 92 require.NoError(t, err) 93 94 // ParseIfExists should now reflect the manifest written above. 95 exists, upstream, err := fm.ParseIfExists(context.Background(), stats, func() error { 96 // This should fail to get the lock, and therefore _not_ clobber the manifest. 97 lock := computeAddr([]byte("newlock")) 98 badRoot := hash.Of([]byte("bad root")) 99 m = strings.Join([]string{StorageVersion, "0", lock.String(), badRoot.String(), gcGen.String(), tableName.String(), "0"}, ":") 100 b, err := tryClobberManifest(fm.dir, m) 101 require.NoError(t, err, string(b)) 102 return err 103 }) 104 105 require.NoError(t, err) 106 assert.True(exists) 107 assert.Equal(constants.NomsVersion, upstream.vers) 108 assert.Equal(newRoot, upstream.root) 109 if assert.Len(upstream.specs, 1) { 110 assert.Equal(tableName.String(), upstream.specs[0].name.String()) 111 assert.Equal(uint32(0), upstream.specs[0].chunkCount) 112 } 113 } 114 115 func TestFileManifestUpdateWontClobberOldVersion(t *testing.T) { 116 assert := assert.New(t) 117 fm := makeFileManifestTempDir(t) 118 defer os.RemoveAll(fm.dir) 119 stats := &Stats{} 120 121 // Simulate another process having already put old Noms data in dir/. 122 m := strings.Join([]string{StorageVersion, "0", addr{}.String(), hash.Hash{}.String(), addr{}.String()}, ":") 123 err := clobberManifest(fm.dir, m) 124 require.NoError(t, err) 125 126 _, err = fm.Update(context.Background(), addr{}, manifestContents{}, stats, nil) 127 assert.Error(err) 128 } 129 130 func TestFileManifestUpdateEmpty(t *testing.T) { 131 assert := assert.New(t) 132 fm := makeFileManifestTempDir(t) 133 defer os.RemoveAll(fm.dir) 134 stats := &Stats{} 135 136 l := computeAddr([]byte{0x01}) 137 upstream, err := fm.Update(context.Background(), addr{}, manifestContents{vers: constants.NomsVersion, lock: l}, stats, nil) 138 require.NoError(t, err) 139 assert.Equal(l, upstream.lock) 140 assert.True(upstream.root.IsEmpty()) 141 assert.Empty(upstream.specs) 142 143 fm2 := fileManifestV5{fm.dir} // Open existent, but empty manifest 144 exists, upstream, err := fm2.ParseIfExists(context.Background(), stats, nil) 145 require.NoError(t, err) 146 assert.True(exists) 147 assert.Equal(l, upstream.lock) 148 assert.True(upstream.root.IsEmpty()) 149 assert.Empty(upstream.specs) 150 151 l2 := computeAddr([]byte{0x02}) 152 upstream, err = fm2.Update(context.Background(), l, manifestContents{vers: constants.NomsVersion, lock: l2}, stats, nil) 153 require.NoError(t, err) 154 assert.Equal(l2, upstream.lock) 155 assert.True(upstream.root.IsEmpty()) 156 assert.Empty(upstream.specs) 157 } 158 159 func TestFileManifestUpdate(t *testing.T) { 160 assert := assert.New(t) 161 fm := makeFileManifestTempDir(t) 162 defer os.RemoveAll(fm.dir) 163 stats := &Stats{} 164 165 // First, test winning the race against another process. 166 contents := manifestContents{ 167 vers: constants.NomsVersion, 168 lock: computeAddr([]byte("locker")), 169 root: hash.Of([]byte("new root")), 170 specs: []tableSpec{{computeAddr([]byte("a")), 3}}, 171 } 172 upstream, err := fm.Update(context.Background(), addr{}, contents, stats, func() error { 173 // This should fail to get the lock, and therefore _not_ clobber the manifest. So the Update should succeed. 174 lock := computeAddr([]byte("nolock")) 175 newRoot2 := hash.Of([]byte("noroot")) 176 gcGen := addr{} 177 m := strings.Join([]string{StorageVersion, constants.NomsVersion, lock.String(), newRoot2.String(), gcGen.String()}, ":") 178 b, err := tryClobberManifest(fm.dir, m) 179 require.NoError(t, err, string(b)) 180 return nil 181 }) 182 require.NoError(t, err) 183 assert.Equal(contents.lock, upstream.lock) 184 assert.Equal(contents.root, upstream.root) 185 assert.Equal(contents.specs, upstream.specs) 186 187 // Now, test the case where the optimistic lock fails, and someone else updated the root since last we checked. 188 contents2 := manifestContents{lock: computeAddr([]byte("locker 2")), root: hash.Of([]byte("new root 2")), vers: constants.NomsVersion} 189 upstream, err = fm.Update(context.Background(), addr{}, contents2, stats, nil) 190 require.NoError(t, err) 191 assert.Equal(contents.lock, upstream.lock) 192 assert.Equal(contents.root, upstream.root) 193 assert.Equal(contents.specs, upstream.specs) 194 upstream, err = fm.Update(context.Background(), upstream.lock, contents2, stats, nil) 195 require.NoError(t, err) 196 assert.Equal(contents2.lock, upstream.lock) 197 assert.Equal(contents2.root, upstream.root) 198 assert.Empty(upstream.specs) 199 200 // Now, test the case where the optimistic lock fails because someone else updated only the tables since last we checked 201 jerkLock := computeAddr([]byte("jerk")) 202 tableName := computeAddr([]byte("table1")) 203 gcGen := addr{} 204 m := strings.Join([]string{StorageVersion, constants.NomsVersion, jerkLock.String(), contents2.root.String(), gcGen.String(), tableName.String(), "1"}, ":") 205 err = clobberManifest(fm.dir, m) 206 require.NoError(t, err) 207 208 contents3 := manifestContents{lock: computeAddr([]byte("locker 3")), root: hash.Of([]byte("new root 3")), vers: constants.NomsVersion} 209 upstream, err = fm.Update(context.Background(), upstream.lock, contents3, stats, nil) 210 require.NoError(t, err) 211 assert.Equal(jerkLock, upstream.lock) 212 assert.Equal(contents2.root, upstream.root) 213 assert.Equal([]tableSpec{{tableName, 1}}, upstream.specs) 214 } 215 216 // tryClobberManifest simulates another process trying to access dir/manifestFileName concurrently. To avoid deadlock, it does a non-blocking lock of dir/lockFileName. If it can get the lock, it clobbers the manifest. 217 func tryClobberManifest(dir, contents string) ([]byte, error) { 218 return runClobber(dir, contents) 219 } 220 221 // clobberManifest simulates another process writing dir/manifestFileName concurrently. It ignores the lock file, so it's up to the caller to ensure correctness. 222 func clobberManifest(dir, contents string) error { 223 if err := ioutil.WriteFile(filepath.Join(dir, lockFileName), nil, 0666); err != nil { 224 return err 225 } 226 return ioutil.WriteFile(filepath.Join(dir, manifestFileName), []byte(contents), 0666) 227 } 228 229 func runClobber(dir, contents string) ([]byte, error) { 230 _, filename, _, _ := runtime.Caller(1) 231 clobber := filepath.Join(filepath.Dir(filename), "test/manifest_clobber.go") 232 mkPath := func(f string) string { 233 return filepath.Join(dir, f) 234 } 235 236 c := exec.Command("go", "run", clobber, mkPath(lockFileName), mkPath(manifestFileName), contents) 237 return c.CombinedOutput() 238 }