github.com/hasnat/dolt/go@v0.0.0-20210628190320-9eb5d843fbb7/store/nbs/file_manifest_test.go (about)

     1  // Copyright 2019 Dolthub, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  //
    15  // This file incorporates work covered by the following copyright and
    16  // permission notice:
    17  //
    18  // Copyright 2016 Attic Labs, Inc. All rights reserved.
    19  // Licensed under the Apache License, version 2.0:
    20  // http://www.apache.org/licenses/LICENSE-2.0
    21  
    22  package nbs
    23  
    24  import (
    25  	"context"
    26  	"io/ioutil"
    27  	"os"
    28  	"os/exec"
    29  	"path/filepath"
    30  	"runtime"
    31  	"strings"
    32  	"testing"
    33  
    34  	"github.com/stretchr/testify/assert"
    35  	"github.com/stretchr/testify/require"
    36  
    37  	"github.com/dolthub/dolt/go/store/constants"
    38  	"github.com/dolthub/dolt/go/store/hash"
    39  )
    40  
    41  func makeFileManifestTempDir(t *testing.T) fileManifestV5 {
    42  	dir, err := ioutil.TempDir("", "")
    43  	require.NoError(t, err)
    44  	return fileManifestV5{dir: dir} //, cache: newManifestCache(defaultManifestCacheSize)}
    45  }
    46  
    47  func TestFileManifestLoadIfExists(t *testing.T) {
    48  	assert := assert.New(t)
    49  	fm := makeFileManifestTempDir(t)
    50  	defer os.RemoveAll(fm.dir)
    51  	stats := &Stats{}
    52  
    53  	exists, upstream, err := fm.ParseIfExists(context.Background(), stats, nil)
    54  	require.NoError(t, err)
    55  	assert.False(exists)
    56  
    57  	// Simulate another process writing a manifest (with an old Noms version).
    58  	jerk := computeAddr([]byte("jerk"))
    59  	newRoot := hash.Of([]byte("new root"))
    60  	tableName := hash.Of([]byte("table1"))
    61  	gcGen := addr{}
    62  	m := strings.Join([]string{StorageVersion, "0", jerk.String(), newRoot.String(), gcGen.String(), tableName.String(), "0"}, ":")
    63  	err = clobberManifest(fm.dir, m)
    64  	require.NoError(t, err)
    65  
    66  	// ParseIfExists should now reflect the manifest written above.
    67  	exists, upstream, err = fm.ParseIfExists(context.Background(), stats, nil)
    68  	require.NoError(t, err)
    69  	assert.True(exists)
    70  	assert.Equal("0", upstream.vers)
    71  	assert.Equal(jerk, upstream.lock)
    72  	assert.Equal(newRoot, upstream.root)
    73  	if assert.Len(upstream.specs, 1) {
    74  		assert.Equal(tableName.String(), upstream.specs[0].name.String())
    75  		assert.Equal(uint32(0), upstream.specs[0].chunkCount)
    76  	}
    77  }
    78  
    79  func TestFileManifestLoadIfExistsHoldsLock(t *testing.T) {
    80  	assert := assert.New(t)
    81  	fm := makeFileManifestTempDir(t)
    82  	defer os.RemoveAll(fm.dir)
    83  	stats := &Stats{}
    84  
    85  	// Simulate another process writing a manifest.
    86  	lock := computeAddr([]byte("locker"))
    87  	newRoot := hash.Of([]byte("new root"))
    88  	tableName := hash.Of([]byte("table1"))
    89  	gcGen := addr{}
    90  	m := strings.Join([]string{StorageVersion, constants.NomsVersion, lock.String(), newRoot.String(), gcGen.String(), tableName.String(), "0"}, ":")
    91  	err := clobberManifest(fm.dir, m)
    92  	require.NoError(t, err)
    93  
    94  	// ParseIfExists should now reflect the manifest written above.
    95  	exists, upstream, err := fm.ParseIfExists(context.Background(), stats, func() error {
    96  		// This should fail to get the lock, and therefore _not_ clobber the manifest.
    97  		lock := computeAddr([]byte("newlock"))
    98  		badRoot := hash.Of([]byte("bad root"))
    99  		m = strings.Join([]string{StorageVersion, "0", lock.String(), badRoot.String(), gcGen.String(), tableName.String(), "0"}, ":")
   100  		b, err := tryClobberManifest(fm.dir, m)
   101  		require.NoError(t, err, string(b))
   102  		return err
   103  	})
   104  
   105  	require.NoError(t, err)
   106  	assert.True(exists)
   107  	assert.Equal(constants.NomsVersion, upstream.vers)
   108  	assert.Equal(newRoot, upstream.root)
   109  	if assert.Len(upstream.specs, 1) {
   110  		assert.Equal(tableName.String(), upstream.specs[0].name.String())
   111  		assert.Equal(uint32(0), upstream.specs[0].chunkCount)
   112  	}
   113  }
   114  
   115  func TestFileManifestUpdateWontClobberOldVersion(t *testing.T) {
   116  	assert := assert.New(t)
   117  	fm := makeFileManifestTempDir(t)
   118  	defer os.RemoveAll(fm.dir)
   119  	stats := &Stats{}
   120  
   121  	// Simulate another process having already put old Noms data in dir/.
   122  	m := strings.Join([]string{StorageVersion, "0", addr{}.String(), hash.Hash{}.String(), addr{}.String()}, ":")
   123  	err := clobberManifest(fm.dir, m)
   124  	require.NoError(t, err)
   125  
   126  	_, err = fm.Update(context.Background(), addr{}, manifestContents{}, stats, nil)
   127  	assert.Error(err)
   128  }
   129  
   130  func TestFileManifestUpdateEmpty(t *testing.T) {
   131  	assert := assert.New(t)
   132  	fm := makeFileManifestTempDir(t)
   133  	defer os.RemoveAll(fm.dir)
   134  	stats := &Stats{}
   135  
   136  	l := computeAddr([]byte{0x01})
   137  	upstream, err := fm.Update(context.Background(), addr{}, manifestContents{vers: constants.NomsVersion, lock: l}, stats, nil)
   138  	require.NoError(t, err)
   139  	assert.Equal(l, upstream.lock)
   140  	assert.True(upstream.root.IsEmpty())
   141  	assert.Empty(upstream.specs)
   142  
   143  	fm2 := fileManifestV5{fm.dir} // Open existent, but empty manifest
   144  	exists, upstream, err := fm2.ParseIfExists(context.Background(), stats, nil)
   145  	require.NoError(t, err)
   146  	assert.True(exists)
   147  	assert.Equal(l, upstream.lock)
   148  	assert.True(upstream.root.IsEmpty())
   149  	assert.Empty(upstream.specs)
   150  
   151  	l2 := computeAddr([]byte{0x02})
   152  	upstream, err = fm2.Update(context.Background(), l, manifestContents{vers: constants.NomsVersion, lock: l2}, stats, nil)
   153  	require.NoError(t, err)
   154  	assert.Equal(l2, upstream.lock)
   155  	assert.True(upstream.root.IsEmpty())
   156  	assert.Empty(upstream.specs)
   157  }
   158  
   159  func TestFileManifestUpdate(t *testing.T) {
   160  	assert := assert.New(t)
   161  	fm := makeFileManifestTempDir(t)
   162  	defer os.RemoveAll(fm.dir)
   163  	stats := &Stats{}
   164  
   165  	// First, test winning the race against another process.
   166  	contents := manifestContents{
   167  		vers:  constants.NomsVersion,
   168  		lock:  computeAddr([]byte("locker")),
   169  		root:  hash.Of([]byte("new root")),
   170  		specs: []tableSpec{{computeAddr([]byte("a")), 3}},
   171  	}
   172  	upstream, err := fm.Update(context.Background(), addr{}, contents, stats, func() error {
   173  		// This should fail to get the lock, and therefore _not_ clobber the manifest. So the Update should succeed.
   174  		lock := computeAddr([]byte("nolock"))
   175  		newRoot2 := hash.Of([]byte("noroot"))
   176  		gcGen := addr{}
   177  		m := strings.Join([]string{StorageVersion, constants.NomsVersion, lock.String(), newRoot2.String(), gcGen.String()}, ":")
   178  		b, err := tryClobberManifest(fm.dir, m)
   179  		require.NoError(t, err, string(b))
   180  		return nil
   181  	})
   182  	require.NoError(t, err)
   183  	assert.Equal(contents.lock, upstream.lock)
   184  	assert.Equal(contents.root, upstream.root)
   185  	assert.Equal(contents.specs, upstream.specs)
   186  
   187  	// Now, test the case where the optimistic lock fails, and someone else updated the root since last we checked.
   188  	contents2 := manifestContents{lock: computeAddr([]byte("locker 2")), root: hash.Of([]byte("new root 2")), vers: constants.NomsVersion}
   189  	upstream, err = fm.Update(context.Background(), addr{}, contents2, stats, nil)
   190  	require.NoError(t, err)
   191  	assert.Equal(contents.lock, upstream.lock)
   192  	assert.Equal(contents.root, upstream.root)
   193  	assert.Equal(contents.specs, upstream.specs)
   194  	upstream, err = fm.Update(context.Background(), upstream.lock, contents2, stats, nil)
   195  	require.NoError(t, err)
   196  	assert.Equal(contents2.lock, upstream.lock)
   197  	assert.Equal(contents2.root, upstream.root)
   198  	assert.Empty(upstream.specs)
   199  
   200  	// Now, test the case where the optimistic lock fails because someone else updated only the tables since last we checked
   201  	jerkLock := computeAddr([]byte("jerk"))
   202  	tableName := computeAddr([]byte("table1"))
   203  	gcGen := addr{}
   204  	m := strings.Join([]string{StorageVersion, constants.NomsVersion, jerkLock.String(), contents2.root.String(), gcGen.String(), tableName.String(), "1"}, ":")
   205  	err = clobberManifest(fm.dir, m)
   206  	require.NoError(t, err)
   207  
   208  	contents3 := manifestContents{lock: computeAddr([]byte("locker 3")), root: hash.Of([]byte("new root 3")), vers: constants.NomsVersion}
   209  	upstream, err = fm.Update(context.Background(), upstream.lock, contents3, stats, nil)
   210  	require.NoError(t, err)
   211  	assert.Equal(jerkLock, upstream.lock)
   212  	assert.Equal(contents2.root, upstream.root)
   213  	assert.Equal([]tableSpec{{tableName, 1}}, upstream.specs)
   214  }
   215  
   216  // tryClobberManifest simulates another process trying to access dir/manifestFileName concurrently. To avoid deadlock, it does a non-blocking lock of dir/lockFileName. If it can get the lock, it clobbers the manifest.
   217  func tryClobberManifest(dir, contents string) ([]byte, error) {
   218  	return runClobber(dir, contents)
   219  }
   220  
   221  // clobberManifest simulates another process writing dir/manifestFileName concurrently. It ignores the lock file, so it's up to the caller to ensure correctness.
   222  func clobberManifest(dir, contents string) error {
   223  	if err := ioutil.WriteFile(filepath.Join(dir, lockFileName), nil, 0666); err != nil {
   224  		return err
   225  	}
   226  	return ioutil.WriteFile(filepath.Join(dir, manifestFileName), []byte(contents), 0666)
   227  }
   228  
   229  func runClobber(dir, contents string) ([]byte, error) {
   230  	_, filename, _, _ := runtime.Caller(1)
   231  	clobber := filepath.Join(filepath.Dir(filename), "test/manifest_clobber.go")
   232  	mkPath := func(f string) string {
   233  		return filepath.Join(dir, f)
   234  	}
   235  
   236  	c := exec.Command("go", "run", clobber, mkPath(lockFileName), mkPath(manifestFileName), contents)
   237  	return c.CombinedOutput()
   238  }