github.com/dolthub/dolt/go@v0.40.5-0.20240520175717-68db7794bea6/store/spec/spec_test.go (about)

     1  // Copyright 2019 Dolthub, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  //
    15  // This file incorporates work covered by the following copyright and
    16  // permission notice:
    17  //
    18  // Copyright 2016 Attic Labs, Inc. All rights reserved.
    19  // Licensed under the Apache License, version 2.0:
    20  // http://www.apache.org/licenses/LICENSE-2.0
    21  
    22  package spec
    23  
    24  import (
    25  	"context"
    26  	"os"
    27  	"path/filepath"
    28  	"testing"
    29  
    30  	"github.com/stretchr/testify/assert"
    31  	"github.com/stretchr/testify/require"
    32  
    33  	"github.com/dolthub/dolt/go/libraries/utils/file"
    34  	"github.com/dolthub/dolt/go/store/chunks"
    35  	"github.com/dolthub/dolt/go/store/d"
    36  	"github.com/dolthub/dolt/go/store/datas"
    37  	"github.com/dolthub/dolt/go/store/hash"
    38  	"github.com/dolthub/dolt/go/store/nbs"
    39  	"github.com/dolthub/dolt/go/store/prolly/tree"
    40  	"github.com/dolthub/dolt/go/store/types"
    41  )
    42  
    43  func mustValue(val types.Value, err error) types.Value {
    44  	d.PanicIfError(err)
    45  	return val
    46  }
    47  
    48  func mustType(t *types.Type, err error) *types.Type {
    49  	d.PanicIfError(err)
    50  	return t
    51  }
    52  
    53  func mustString(str string, err error) string {
    54  	d.PanicIfError(err)
    55  	return str
    56  }
    57  
    58  func mustList(l types.List, err error) types.List {
    59  	d.PanicIfError(err)
    60  	return l
    61  }
    62  
    63  func mustHash(h hash.Hash, err error) hash.Hash {
    64  	d.PanicIfError(err)
    65  	return h
    66  }
    67  
    68  func mustGetValue(v types.Value, found bool, err error) types.Value {
    69  	d.PanicIfError(err)
    70  	d.PanicIfFalse(found)
    71  	return v
    72  }
    73  
    74  func TestMemDatabaseSpec(t *testing.T) {
    75  	assert := assert.New(t)
    76  
    77  	spec, err := ForDatabase("mem")
    78  	assert.NoError(err)
    79  	defer spec.Close()
    80  
    81  	assert.Equal("mem", spec.Protocol)
    82  	assert.Equal("", spec.DatabaseName)
    83  	assert.True(spec.Path.IsEmpty())
    84  
    85  	s := types.String("hello")
    86  	vrw := spec.GetVRW(context.Background())
    87  	vrw.WriteValue(context.Background(), s)
    88  	assert.Equal(s, mustValue(vrw.ReadValue(context.Background(), mustHash(s.Hash(vrw.Format())))))
    89  }
    90  
    91  func TestMemDatasetSpec(t *testing.T) {
    92  	assert := assert.New(t)
    93  
    94  	spec, err := ForDataset("mem::test")
    95  	assert.NoError(err)
    96  	defer spec.Close()
    97  
    98  	assert.Equal("mem", spec.Protocol)
    99  	assert.Equal("", spec.DatabaseName)
   100  	assert.Equal("test", spec.Path.Dataset)
   101  
   102  	ds := spec.GetDataset(context.Background())
   103  	_, ok, err := spec.GetDataset(context.Background()).MaybeHeadValue()
   104  	assert.NoError(err)
   105  	assert.False(ok)
   106  
   107  	s := types.String("hello")
   108  	db := spec.GetDatabase(context.Background())
   109  	ds, err = datas.CommitValue(context.Background(), db, ds, s)
   110  	assert.NoError(err)
   111  	currHeadVal, ok, err := ds.MaybeHeadValue()
   112  	assert.NoError(err)
   113  	assert.True(ok)
   114  	assert.Equal(s, currHeadVal)
   115  }
   116  
   117  func TestMemHashPathSpec(t *testing.T) {
   118  	assert := assert.New(t)
   119  
   120  	s := types.String("hello")
   121  
   122  	spec, err := ForPath("mem::#" + mustHash(s.Hash(types.Format_Default)).String())
   123  	assert.NoError(err)
   124  	defer spec.Close()
   125  
   126  	assert.Equal("mem", spec.Protocol)
   127  	assert.Equal("", spec.DatabaseName)
   128  	assert.False(spec.Path.IsEmpty())
   129  
   130  	// This is a reasonable check but it causes the next GetValue to return nil:
   131  	// assert.Nil(spec.GetValue())
   132  
   133  	spec.GetVRW(context.Background()).WriteValue(context.Background(), s)
   134  	value, err := spec.GetValue(context.Background())
   135  	assert.NoError(err)
   136  	assert.Equal(s, value)
   137  }
   138  
   139  func TestMemDatasetPathSpec(t *testing.T) {
   140  	assert := assert.New(t)
   141  
   142  	spec, err := ForPath("mem::test")
   143  	assert.NoError(err)
   144  	defer spec.Close()
   145  
   146  	assert.Equal("mem", spec.Protocol)
   147  	assert.Equal("", spec.DatabaseName)
   148  	assert.False(spec.Path.IsEmpty())
   149  
   150  	assert.Nil(spec.GetValue(context.Background()))
   151  
   152  	db := spec.GetDatabase(context.Background())
   153  	ds, err := db.GetDataset(context.Background(), "test")
   154  	assert.NoError(err)
   155  	_, err = datas.CommitValue(context.Background(), db, ds, mustList(types.NewList(context.Background(), spec.GetVRW(context.Background()), types.Float(42))))
   156  	assert.NoError(err)
   157  
   158  	value, err := spec.GetValue(context.Background())
   159  	require.NoError(t, err)
   160  	assert.NotNil(value)
   161  }
   162  
   163  func TestNBSDatabaseSpec(t *testing.T) {
   164  	assert := assert.New(t)
   165  
   166  	run := func(prefix string) {
   167  		tmpDir, err := os.MkdirTemp("", "spec_test")
   168  		assert.NoError(err)
   169  		defer file.RemoveAll(tmpDir)
   170  
   171  		s := types.String("string")
   172  
   173  		// Existing database in the database are read from the spec.
   174  		store1 := filepath.Join(tmpDir, "store1")
   175  		os.Mkdir(store1, 0777)
   176  		func() {
   177  			cs, err := nbs.NewLocalStore(context.Background(), types.Format_Default.VersionString(), store1, 8*(1<<20), nbs.NewUnlimitedMemQuotaProvider())
   178  			assert.NoError(err)
   179  			vrw := types.NewValueStore(cs)
   180  			db := datas.NewTypesDatabase(vrw, tree.NewNodeStore(cs))
   181  			defer db.Close()
   182  			r, err := vrw.WriteValue(context.Background(), s)
   183  			assert.NoError(err)
   184  			ds, err := db.GetDataset(context.Background(), "datasetID")
   185  			assert.NoError(err)
   186  			_, err = datas.CommitValue(context.Background(), db, ds, r)
   187  			assert.NoError(err)
   188  		}()
   189  
   190  		spec1, err := ForDatabase(prefix + store1)
   191  		assert.NoError(err)
   192  		defer spec1.Close()
   193  
   194  		assert.Equal("nbs", spec1.Protocol)
   195  		assert.Equal(store1, spec1.DatabaseName)
   196  
   197  		vrw := spec1.GetVRW(context.Background())
   198  
   199  		assert.Equal(s, mustValue(vrw.ReadValue(context.Background(), mustHash(s.Hash(vrw.Format())))))
   200  
   201  		// New databases can be created and read/written from.
   202  		store2 := filepath.Join(tmpDir, "store2")
   203  		os.Mkdir(store2, 0777)
   204  		spec2, err := ForDatabase(prefix + store2)
   205  		assert.NoError(err)
   206  		defer spec2.Close()
   207  
   208  		assert.Equal("nbs", spec2.Protocol)
   209  		assert.Equal(store2, spec2.DatabaseName)
   210  
   211  		db := spec2.GetDatabase(context.Background())
   212  		vrw = spec2.GetVRW(context.Background())
   213  		vrw.WriteValue(context.Background(), s)
   214  		r, err := vrw.WriteValue(context.Background(), s)
   215  		assert.NoError(err)
   216  		ds, err := db.GetDataset(context.Background(), "datasetID")
   217  		assert.NoError(err)
   218  		_, err = datas.CommitValue(context.Background(), db, ds, r)
   219  		assert.NoError(err)
   220  		assert.Equal(s, mustValue(vrw.ReadValue(context.Background(), mustHash(s.Hash(vrw.Format())))))
   221  	}
   222  
   223  	run("")
   224  	run("nbs:")
   225  }
   226  
   227  // Skip LDB dataset and path tests: the database behaviour is tested in
   228  // TestLDBDatabaseSpec, TestMemDatasetSpec/TestMem*PathSpec cover general
   229  // dataset/path behaviour, and ForDataset/ForPath test LDB parsing.
   230  
   231  func TestCloseSpecWithoutOpen(t *testing.T) {
   232  	s, err := ForDatabase("mem")
   233  	assert.NoError(t, err)
   234  	s.Close()
   235  }
   236  
   237  func TestHref(t *testing.T) {
   238  	assert := assert.New(t)
   239  
   240  	sp, _ := ForDatabase("aws://table/foo/bar/baz")
   241  	assert.Equal("aws://table/foo/bar/baz", sp.Href())
   242  	sp.Close()
   243  	sp, _ = ForDataset("aws://[table:bucket]/foo/bar/baz::myds")
   244  	assert.Equal("aws://[table:bucket]/foo/bar/baz", sp.Href())
   245  	sp.Close()
   246  	sp, _ = ForPath("aws://[table:bucket]/foo/bar/baz::myds.my.path")
   247  	assert.Equal("aws://[table:bucket]/foo/bar/baz", sp.Href())
   248  	sp.Close()
   249  
   250  	sp, err := ForPath("mem::myds.my.path")
   251  	assert.NoError(err)
   252  	assert.Equal("", sp.Href())
   253  	sp.Close()
   254  }
   255  
   256  func TestForDatabase(t *testing.T) {
   257  	assert := assert.New(t)
   258  
   259  	badSpecs := []string{
   260  		"mem:stuff",
   261  		"mem::",
   262  		"mem:",
   263  		"ldb:",
   264  		"random:",
   265  		"random:random",
   266  		"/file/ba:d",
   267  		"aws://[t:b]",
   268  		"aws://t",
   269  		"aws://t:",
   270  	}
   271  
   272  	for _, spec := range badSpecs {
   273  		_, err := ForDatabase(spec)
   274  		assert.Error(err, spec)
   275  	}
   276  
   277  	tmpDir, err := os.MkdirTemp("", "spec_test")
   278  	assert.NoError(err)
   279  	defer file.RemoveAll(tmpDir)
   280  
   281  	testCases := []struct {
   282  		spec, protocol, databaseName, canonicalSpecIfAny string
   283  	}{
   284  		{"mem", "mem", "", ""},
   285  		{tmpDir, "nbs", tmpDir, "nbs:" + tmpDir},
   286  		{"nbs:" + tmpDir, "nbs", tmpDir, ""},
   287  		{"aws://[table:bucket]/db", "aws", "//[table:bucket]/db", ""},
   288  		{"aws://table/db", "aws", "//table/db", ""},
   289  	}
   290  
   291  	for _, tc := range testCases {
   292  		spec, err := ForDatabase(tc.spec)
   293  		assert.NoError(err, tc.spec)
   294  		defer spec.Close()
   295  
   296  		assert.Equal(tc.protocol, spec.Protocol)
   297  		assert.Equal(tc.databaseName, spec.DatabaseName)
   298  		assert.True(spec.Path.IsEmpty())
   299  
   300  		if tc.canonicalSpecIfAny == "" {
   301  			assert.Equal(tc.spec, spec.String())
   302  		} else {
   303  			assert.Equal(tc.canonicalSpecIfAny, spec.String())
   304  		}
   305  	}
   306  }
   307  
   308  func TestForDataset(t *testing.T) {
   309  	badSpecs := []string{
   310  		"mem",
   311  		"mem:",
   312  		"mem:::ds",
   313  		"monkey",
   314  		"monkey:balls",
   315  		"mem:/a/bogus/path:dsname",
   316  		"nbs:",
   317  		"nbs:hello",
   318  		"aws://[t:b]/db",
   319  	}
   320  
   321  	for _, spec := range badSpecs {
   322  		t.Run(spec, func(t *testing.T) {
   323  			_, err := ForDataset(spec)
   324  			assert.Error(t, err, spec)
   325  		})
   326  	}
   327  
   328  	validDatasetNames := []string{"a", "Z", "0", "/", "-", "_"}
   329  	for _, s := range validDatasetNames {
   330  		spec, err := ForDataset("mem::" + s)
   331  		assert.NoError(t, err)
   332  		spec.Close()
   333  	}
   334  
   335  	tmpDir, err := os.MkdirTemp("", "spec_test")
   336  	assert.NoError(t, err)
   337  	defer file.RemoveAll(tmpDir)
   338  
   339  	testCases := []struct {
   340  		spec, protocol, databaseName, datasetName, canonicalSpecIfAny string
   341  	}{
   342  		{"nbs:" + tmpDir + "::ds/one", "nbs", tmpDir, "ds/one", ""},
   343  		{tmpDir + "::ds/one", "nbs", tmpDir, "ds/one", "nbs:" + tmpDir + "::ds/one"},
   344  		{"aws://[table:bucket]/db::ds", "aws", "//[table:bucket]/db", "ds", ""},
   345  		{"aws://table/db::ds", "aws", "//table/db", "ds", ""},
   346  	}
   347  
   348  	for _, tc := range testCases {
   349  		assert := assert.New(t)
   350  		spec, err := ForDataset(tc.spec)
   351  		assert.NoError(err, tc.spec)
   352  		defer spec.Close()
   353  
   354  		assert.Equal(tc.protocol, spec.Protocol)
   355  		assert.Equal(tc.databaseName, spec.DatabaseName)
   356  		assert.Equal(tc.datasetName, spec.Path.Dataset)
   357  
   358  		if tc.canonicalSpecIfAny == "" {
   359  			assert.Equal(tc.spec, spec.String())
   360  		} else {
   361  			assert.Equal(tc.canonicalSpecIfAny, spec.String())
   362  		}
   363  	}
   364  }
   365  
   366  func TestForPath(t *testing.T) {
   367  	assert := assert.New(t)
   368  
   369  	badSpecs := []string{
   370  		"mem::#",
   371  		"mem::#s",
   372  		"mem::#foobarbaz",
   373  		"mem::#wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww",
   374  	}
   375  
   376  	for _, bs := range badSpecs {
   377  		_, err := ForPath(bs)
   378  		assert.Error(err)
   379  	}
   380  
   381  	tmpDir, err := os.MkdirTemp("", "spec_test")
   382  	assert.NoError(err)
   383  	defer file.RemoveAll(tmpDir)
   384  
   385  	testCases := []struct {
   386  		spec, protocol, databaseName, pathString, canonicalSpecIfAny string
   387  	}{
   388  		{tmpDir + "::#0123456789abcdefghijklmnopqrstuv", "nbs", tmpDir, "#0123456789abcdefghijklmnopqrstuv", "nbs:" + tmpDir + "::#0123456789abcdefghijklmnopqrstuv"},
   389  		{"nbs:" + tmpDir + "::#0123456789abcdefghijklmnopqrstuv", "nbs", tmpDir, "#0123456789abcdefghijklmnopqrstuv", ""},
   390  		{"mem::#0123456789abcdefghijklmnopqrstuv", "mem", "", "#0123456789abcdefghijklmnopqrstuv", ""},
   391  		{"aws://[table:bucket]/db::foo.foo", "aws", "//[table:bucket]/db", "foo.foo", ""},
   392  		{"aws://table/db::foo.foo", "aws", "//table/db", "foo.foo", ""},
   393  	}
   394  
   395  	for _, tc := range testCases {
   396  		spec, err := ForPath(tc.spec)
   397  		assert.NoError(err)
   398  		defer spec.Close()
   399  
   400  		assert.Equal(tc.protocol, spec.Protocol)
   401  		assert.Equal(tc.databaseName, spec.DatabaseName)
   402  		assert.Equal(tc.pathString, spec.Path.String())
   403  
   404  		if tc.canonicalSpecIfAny == "" {
   405  			assert.Equal(tc.spec, spec.String())
   406  		} else {
   407  			assert.Equal(tc.canonicalSpecIfAny, spec.String())
   408  		}
   409  	}
   410  }
   411  
   412  func TestMultipleSpecsSameNBS(t *testing.T) {
   413  	assert := assert.New(t)
   414  
   415  	tmpDir, err := os.MkdirTemp("", "spec_test")
   416  	assert.NoError(err)
   417  	defer file.RemoveAll(tmpDir)
   418  
   419  	spec1, err1 := ForDatabase(tmpDir)
   420  	spec2, err2 := ForDatabase(tmpDir)
   421  
   422  	assert.NoError(err1)
   423  	assert.NoError(err2)
   424  	defer spec1.Close()
   425  	defer spec2.Close()
   426  
   427  	s := types.String("hello")
   428  	db := spec1.GetDatabase(context.Background())
   429  	vrw := spec1.GetVRW(context.Background())
   430  	r, err := vrw.WriteValue(context.Background(), s)
   431  	assert.NoError(err)
   432  	ds, err := db.GetDataset(context.Background(), "datasetID")
   433  	assert.NoError(err)
   434  	_, err = datas.CommitValue(context.Background(), db, ds, r)
   435  	assert.NoError(err)
   436  	assert.Equal(s, mustValue(spec2.GetVRW(context.Background()).ReadValue(context.Background(), mustHash(s.Hash(vrw.Format())))))
   437  }
   438  
   439  func TestAcccessingInvalidSpec(t *testing.T) {
   440  	assert := assert.New(t)
   441  
   442  	test := func(spec string) {
   443  		sp, err := ForDatabase(spec)
   444  		assert.Error(err)
   445  		assert.Equal("", sp.Href())
   446  		assert.Panics(func() { sp.GetDatabase(context.Background()) })
   447  		assert.Panics(func() { sp.GetDatabase(context.Background()) })
   448  		assert.Panics(func() { sp.NewChunkStore(context.Background()) })
   449  		assert.Panics(func() { sp.NewChunkStore(context.Background()) })
   450  		assert.Panics(func() { sp.Close() })
   451  		assert.Panics(func() { sp.Close() })
   452  		// Spec was created with ForDatabase, so dataset/path related functions
   453  		// should just fail not panic.
   454  		assert.Equal(datas.Dataset{}, sp.GetDataset(context.Background()))
   455  		assert.Nil(sp.GetValue(context.Background()))
   456  	}
   457  
   458  	test("")
   459  	test("invalid:spec")
   460  	test("💩:spec")
   461  }
   462  
   463  type testProtocol struct {
   464  	name string
   465  }
   466  
   467  func (t *testProtocol) NewChunkStore(sp Spec) (chunks.ChunkStore, error) {
   468  	t.name = sp.DatabaseName
   469  	return chunks.NewMemoryStoreFactory().CreateStore(context.Background(), ""), nil
   470  }
   471  func (t *testProtocol) NewDatabase(sp Spec) (datas.Database, error) {
   472  	t.name = sp.DatabaseName
   473  	cs, err := t.NewChunkStore(sp)
   474  	d.PanicIfError(err)
   475  	return datas.NewDatabase(cs), nil
   476  }
   477  
   478  func noopGetAddrs(c chunks.Chunk) chunks.GetAddrsCb {
   479  	return func(ctx context.Context, addrs hash.HashSet, _ chunks.PendingRefExists) error {
   480  		return nil
   481  	}
   482  }
   483  
   484  func TestExternalProtocol(t *testing.T) {
   485  	assert := assert.New(t)
   486  	tp := testProtocol{}
   487  	ExternalProtocols["test"] = &tp
   488  
   489  	sp, err := ForDataset("test:foo::bar")
   490  	assert.NoError(err)
   491  	defer sp.Close()
   492  	assert.Equal("test", sp.Protocol)
   493  	assert.Equal("foo", sp.DatabaseName)
   494  
   495  	cs := sp.NewChunkStore(context.Background())
   496  	assert.Equal("foo", tp.name)
   497  	c := chunks.NewChunk([]byte("hi!"))
   498  	err = cs.Put(context.Background(), c, noopGetAddrs)
   499  	assert.NoError(err)
   500  	ok, err := cs.Has(context.Background(), c.Hash())
   501  	assert.NoError(err)
   502  	assert.True(ok)
   503  
   504  	tp.name = ""
   505  	ds := sp.GetDataset(context.Background())
   506  	assert.Equal("foo", tp.name)
   507  
   508  	ds, err = datas.CommitValue(context.Background(), ds.Database(), ds, types.String("hi!"))
   509  	d.PanicIfError(err)
   510  
   511  	headVal, ok, err := ds.MaybeHeadValue()
   512  	assert.NoError(err)
   513  	assert.True(ok)
   514  	assert.True(types.String("hi!").Equals(headVal))
   515  }