github.com/dolthub/dolt/go@v0.40.5-0.20240520175717-68db7794bea6/store/spec/spec_test.go (about) 1 // Copyright 2019 Dolthub, Inc. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 // 15 // This file incorporates work covered by the following copyright and 16 // permission notice: 17 // 18 // Copyright 2016 Attic Labs, Inc. All rights reserved. 19 // Licensed under the Apache License, version 2.0: 20 // http://www.apache.org/licenses/LICENSE-2.0 21 22 package spec 23 24 import ( 25 "context" 26 "os" 27 "path/filepath" 28 "testing" 29 30 "github.com/stretchr/testify/assert" 31 "github.com/stretchr/testify/require" 32 33 "github.com/dolthub/dolt/go/libraries/utils/file" 34 "github.com/dolthub/dolt/go/store/chunks" 35 "github.com/dolthub/dolt/go/store/d" 36 "github.com/dolthub/dolt/go/store/datas" 37 "github.com/dolthub/dolt/go/store/hash" 38 "github.com/dolthub/dolt/go/store/nbs" 39 "github.com/dolthub/dolt/go/store/prolly/tree" 40 "github.com/dolthub/dolt/go/store/types" 41 ) 42 43 func mustValue(val types.Value, err error) types.Value { 44 d.PanicIfError(err) 45 return val 46 } 47 48 func mustType(t *types.Type, err error) *types.Type { 49 d.PanicIfError(err) 50 return t 51 } 52 53 func mustString(str string, err error) string { 54 d.PanicIfError(err) 55 return str 56 } 57 58 func mustList(l types.List, err error) types.List { 59 d.PanicIfError(err) 60 return l 61 } 62 63 func mustHash(h hash.Hash, err error) hash.Hash { 64 d.PanicIfError(err) 65 return h 66 } 67 68 func mustGetValue(v types.Value, found bool, err error) types.Value { 69 d.PanicIfError(err) 70 d.PanicIfFalse(found) 71 return v 72 } 73 74 func TestMemDatabaseSpec(t *testing.T) { 75 assert := assert.New(t) 76 77 spec, err := ForDatabase("mem") 78 assert.NoError(err) 79 defer spec.Close() 80 81 assert.Equal("mem", spec.Protocol) 82 assert.Equal("", spec.DatabaseName) 83 assert.True(spec.Path.IsEmpty()) 84 85 s := types.String("hello") 86 vrw := spec.GetVRW(context.Background()) 87 vrw.WriteValue(context.Background(), s) 88 assert.Equal(s, mustValue(vrw.ReadValue(context.Background(), mustHash(s.Hash(vrw.Format()))))) 89 } 90 91 func TestMemDatasetSpec(t *testing.T) { 92 assert := assert.New(t) 93 94 spec, err := ForDataset("mem::test") 95 assert.NoError(err) 96 defer spec.Close() 97 98 assert.Equal("mem", spec.Protocol) 99 assert.Equal("", spec.DatabaseName) 100 assert.Equal("test", spec.Path.Dataset) 101 102 ds := spec.GetDataset(context.Background()) 103 _, ok, err := spec.GetDataset(context.Background()).MaybeHeadValue() 104 assert.NoError(err) 105 assert.False(ok) 106 107 s := types.String("hello") 108 db := spec.GetDatabase(context.Background()) 109 ds, err = datas.CommitValue(context.Background(), db, ds, s) 110 assert.NoError(err) 111 currHeadVal, ok, err := ds.MaybeHeadValue() 112 assert.NoError(err) 113 assert.True(ok) 114 assert.Equal(s, currHeadVal) 115 } 116 117 func TestMemHashPathSpec(t *testing.T) { 118 assert := assert.New(t) 119 120 s := types.String("hello") 121 122 spec, err := ForPath("mem::#" + mustHash(s.Hash(types.Format_Default)).String()) 123 assert.NoError(err) 124 defer spec.Close() 125 126 assert.Equal("mem", spec.Protocol) 127 assert.Equal("", spec.DatabaseName) 128 assert.False(spec.Path.IsEmpty()) 129 130 // This is a reasonable check but it causes the next GetValue to return nil: 131 // assert.Nil(spec.GetValue()) 132 133 spec.GetVRW(context.Background()).WriteValue(context.Background(), s) 134 value, err := spec.GetValue(context.Background()) 135 assert.NoError(err) 136 assert.Equal(s, value) 137 } 138 139 func TestMemDatasetPathSpec(t *testing.T) { 140 assert := assert.New(t) 141 142 spec, err := ForPath("mem::test") 143 assert.NoError(err) 144 defer spec.Close() 145 146 assert.Equal("mem", spec.Protocol) 147 assert.Equal("", spec.DatabaseName) 148 assert.False(spec.Path.IsEmpty()) 149 150 assert.Nil(spec.GetValue(context.Background())) 151 152 db := spec.GetDatabase(context.Background()) 153 ds, err := db.GetDataset(context.Background(), "test") 154 assert.NoError(err) 155 _, err = datas.CommitValue(context.Background(), db, ds, mustList(types.NewList(context.Background(), spec.GetVRW(context.Background()), types.Float(42)))) 156 assert.NoError(err) 157 158 value, err := spec.GetValue(context.Background()) 159 require.NoError(t, err) 160 assert.NotNil(value) 161 } 162 163 func TestNBSDatabaseSpec(t *testing.T) { 164 assert := assert.New(t) 165 166 run := func(prefix string) { 167 tmpDir, err := os.MkdirTemp("", "spec_test") 168 assert.NoError(err) 169 defer file.RemoveAll(tmpDir) 170 171 s := types.String("string") 172 173 // Existing database in the database are read from the spec. 174 store1 := filepath.Join(tmpDir, "store1") 175 os.Mkdir(store1, 0777) 176 func() { 177 cs, err := nbs.NewLocalStore(context.Background(), types.Format_Default.VersionString(), store1, 8*(1<<20), nbs.NewUnlimitedMemQuotaProvider()) 178 assert.NoError(err) 179 vrw := types.NewValueStore(cs) 180 db := datas.NewTypesDatabase(vrw, tree.NewNodeStore(cs)) 181 defer db.Close() 182 r, err := vrw.WriteValue(context.Background(), s) 183 assert.NoError(err) 184 ds, err := db.GetDataset(context.Background(), "datasetID") 185 assert.NoError(err) 186 _, err = datas.CommitValue(context.Background(), db, ds, r) 187 assert.NoError(err) 188 }() 189 190 spec1, err := ForDatabase(prefix + store1) 191 assert.NoError(err) 192 defer spec1.Close() 193 194 assert.Equal("nbs", spec1.Protocol) 195 assert.Equal(store1, spec1.DatabaseName) 196 197 vrw := spec1.GetVRW(context.Background()) 198 199 assert.Equal(s, mustValue(vrw.ReadValue(context.Background(), mustHash(s.Hash(vrw.Format()))))) 200 201 // New databases can be created and read/written from. 202 store2 := filepath.Join(tmpDir, "store2") 203 os.Mkdir(store2, 0777) 204 spec2, err := ForDatabase(prefix + store2) 205 assert.NoError(err) 206 defer spec2.Close() 207 208 assert.Equal("nbs", spec2.Protocol) 209 assert.Equal(store2, spec2.DatabaseName) 210 211 db := spec2.GetDatabase(context.Background()) 212 vrw = spec2.GetVRW(context.Background()) 213 vrw.WriteValue(context.Background(), s) 214 r, err := vrw.WriteValue(context.Background(), s) 215 assert.NoError(err) 216 ds, err := db.GetDataset(context.Background(), "datasetID") 217 assert.NoError(err) 218 _, err = datas.CommitValue(context.Background(), db, ds, r) 219 assert.NoError(err) 220 assert.Equal(s, mustValue(vrw.ReadValue(context.Background(), mustHash(s.Hash(vrw.Format()))))) 221 } 222 223 run("") 224 run("nbs:") 225 } 226 227 // Skip LDB dataset and path tests: the database behaviour is tested in 228 // TestLDBDatabaseSpec, TestMemDatasetSpec/TestMem*PathSpec cover general 229 // dataset/path behaviour, and ForDataset/ForPath test LDB parsing. 230 231 func TestCloseSpecWithoutOpen(t *testing.T) { 232 s, err := ForDatabase("mem") 233 assert.NoError(t, err) 234 s.Close() 235 } 236 237 func TestHref(t *testing.T) { 238 assert := assert.New(t) 239 240 sp, _ := ForDatabase("aws://table/foo/bar/baz") 241 assert.Equal("aws://table/foo/bar/baz", sp.Href()) 242 sp.Close() 243 sp, _ = ForDataset("aws://[table:bucket]/foo/bar/baz::myds") 244 assert.Equal("aws://[table:bucket]/foo/bar/baz", sp.Href()) 245 sp.Close() 246 sp, _ = ForPath("aws://[table:bucket]/foo/bar/baz::myds.my.path") 247 assert.Equal("aws://[table:bucket]/foo/bar/baz", sp.Href()) 248 sp.Close() 249 250 sp, err := ForPath("mem::myds.my.path") 251 assert.NoError(err) 252 assert.Equal("", sp.Href()) 253 sp.Close() 254 } 255 256 func TestForDatabase(t *testing.T) { 257 assert := assert.New(t) 258 259 badSpecs := []string{ 260 "mem:stuff", 261 "mem::", 262 "mem:", 263 "ldb:", 264 "random:", 265 "random:random", 266 "/file/ba:d", 267 "aws://[t:b]", 268 "aws://t", 269 "aws://t:", 270 } 271 272 for _, spec := range badSpecs { 273 _, err := ForDatabase(spec) 274 assert.Error(err, spec) 275 } 276 277 tmpDir, err := os.MkdirTemp("", "spec_test") 278 assert.NoError(err) 279 defer file.RemoveAll(tmpDir) 280 281 testCases := []struct { 282 spec, protocol, databaseName, canonicalSpecIfAny string 283 }{ 284 {"mem", "mem", "", ""}, 285 {tmpDir, "nbs", tmpDir, "nbs:" + tmpDir}, 286 {"nbs:" + tmpDir, "nbs", tmpDir, ""}, 287 {"aws://[table:bucket]/db", "aws", "//[table:bucket]/db", ""}, 288 {"aws://table/db", "aws", "//table/db", ""}, 289 } 290 291 for _, tc := range testCases { 292 spec, err := ForDatabase(tc.spec) 293 assert.NoError(err, tc.spec) 294 defer spec.Close() 295 296 assert.Equal(tc.protocol, spec.Protocol) 297 assert.Equal(tc.databaseName, spec.DatabaseName) 298 assert.True(spec.Path.IsEmpty()) 299 300 if tc.canonicalSpecIfAny == "" { 301 assert.Equal(tc.spec, spec.String()) 302 } else { 303 assert.Equal(tc.canonicalSpecIfAny, spec.String()) 304 } 305 } 306 } 307 308 func TestForDataset(t *testing.T) { 309 badSpecs := []string{ 310 "mem", 311 "mem:", 312 "mem:::ds", 313 "monkey", 314 "monkey:balls", 315 "mem:/a/bogus/path:dsname", 316 "nbs:", 317 "nbs:hello", 318 "aws://[t:b]/db", 319 } 320 321 for _, spec := range badSpecs { 322 t.Run(spec, func(t *testing.T) { 323 _, err := ForDataset(spec) 324 assert.Error(t, err, spec) 325 }) 326 } 327 328 validDatasetNames := []string{"a", "Z", "0", "/", "-", "_"} 329 for _, s := range validDatasetNames { 330 spec, err := ForDataset("mem::" + s) 331 assert.NoError(t, err) 332 spec.Close() 333 } 334 335 tmpDir, err := os.MkdirTemp("", "spec_test") 336 assert.NoError(t, err) 337 defer file.RemoveAll(tmpDir) 338 339 testCases := []struct { 340 spec, protocol, databaseName, datasetName, canonicalSpecIfAny string 341 }{ 342 {"nbs:" + tmpDir + "::ds/one", "nbs", tmpDir, "ds/one", ""}, 343 {tmpDir + "::ds/one", "nbs", tmpDir, "ds/one", "nbs:" + tmpDir + "::ds/one"}, 344 {"aws://[table:bucket]/db::ds", "aws", "//[table:bucket]/db", "ds", ""}, 345 {"aws://table/db::ds", "aws", "//table/db", "ds", ""}, 346 } 347 348 for _, tc := range testCases { 349 assert := assert.New(t) 350 spec, err := ForDataset(tc.spec) 351 assert.NoError(err, tc.spec) 352 defer spec.Close() 353 354 assert.Equal(tc.protocol, spec.Protocol) 355 assert.Equal(tc.databaseName, spec.DatabaseName) 356 assert.Equal(tc.datasetName, spec.Path.Dataset) 357 358 if tc.canonicalSpecIfAny == "" { 359 assert.Equal(tc.spec, spec.String()) 360 } else { 361 assert.Equal(tc.canonicalSpecIfAny, spec.String()) 362 } 363 } 364 } 365 366 func TestForPath(t *testing.T) { 367 assert := assert.New(t) 368 369 badSpecs := []string{ 370 "mem::#", 371 "mem::#s", 372 "mem::#foobarbaz", 373 "mem::#wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww", 374 } 375 376 for _, bs := range badSpecs { 377 _, err := ForPath(bs) 378 assert.Error(err) 379 } 380 381 tmpDir, err := os.MkdirTemp("", "spec_test") 382 assert.NoError(err) 383 defer file.RemoveAll(tmpDir) 384 385 testCases := []struct { 386 spec, protocol, databaseName, pathString, canonicalSpecIfAny string 387 }{ 388 {tmpDir + "::#0123456789abcdefghijklmnopqrstuv", "nbs", tmpDir, "#0123456789abcdefghijklmnopqrstuv", "nbs:" + tmpDir + "::#0123456789abcdefghijklmnopqrstuv"}, 389 {"nbs:" + tmpDir + "::#0123456789abcdefghijklmnopqrstuv", "nbs", tmpDir, "#0123456789abcdefghijklmnopqrstuv", ""}, 390 {"mem::#0123456789abcdefghijklmnopqrstuv", "mem", "", "#0123456789abcdefghijklmnopqrstuv", ""}, 391 {"aws://[table:bucket]/db::foo.foo", "aws", "//[table:bucket]/db", "foo.foo", ""}, 392 {"aws://table/db::foo.foo", "aws", "//table/db", "foo.foo", ""}, 393 } 394 395 for _, tc := range testCases { 396 spec, err := ForPath(tc.spec) 397 assert.NoError(err) 398 defer spec.Close() 399 400 assert.Equal(tc.protocol, spec.Protocol) 401 assert.Equal(tc.databaseName, spec.DatabaseName) 402 assert.Equal(tc.pathString, spec.Path.String()) 403 404 if tc.canonicalSpecIfAny == "" { 405 assert.Equal(tc.spec, spec.String()) 406 } else { 407 assert.Equal(tc.canonicalSpecIfAny, spec.String()) 408 } 409 } 410 } 411 412 func TestMultipleSpecsSameNBS(t *testing.T) { 413 assert := assert.New(t) 414 415 tmpDir, err := os.MkdirTemp("", "spec_test") 416 assert.NoError(err) 417 defer file.RemoveAll(tmpDir) 418 419 spec1, err1 := ForDatabase(tmpDir) 420 spec2, err2 := ForDatabase(tmpDir) 421 422 assert.NoError(err1) 423 assert.NoError(err2) 424 defer spec1.Close() 425 defer spec2.Close() 426 427 s := types.String("hello") 428 db := spec1.GetDatabase(context.Background()) 429 vrw := spec1.GetVRW(context.Background()) 430 r, err := vrw.WriteValue(context.Background(), s) 431 assert.NoError(err) 432 ds, err := db.GetDataset(context.Background(), "datasetID") 433 assert.NoError(err) 434 _, err = datas.CommitValue(context.Background(), db, ds, r) 435 assert.NoError(err) 436 assert.Equal(s, mustValue(spec2.GetVRW(context.Background()).ReadValue(context.Background(), mustHash(s.Hash(vrw.Format()))))) 437 } 438 439 func TestAcccessingInvalidSpec(t *testing.T) { 440 assert := assert.New(t) 441 442 test := func(spec string) { 443 sp, err := ForDatabase(spec) 444 assert.Error(err) 445 assert.Equal("", sp.Href()) 446 assert.Panics(func() { sp.GetDatabase(context.Background()) }) 447 assert.Panics(func() { sp.GetDatabase(context.Background()) }) 448 assert.Panics(func() { sp.NewChunkStore(context.Background()) }) 449 assert.Panics(func() { sp.NewChunkStore(context.Background()) }) 450 assert.Panics(func() { sp.Close() }) 451 assert.Panics(func() { sp.Close() }) 452 // Spec was created with ForDatabase, so dataset/path related functions 453 // should just fail not panic. 454 assert.Equal(datas.Dataset{}, sp.GetDataset(context.Background())) 455 assert.Nil(sp.GetValue(context.Background())) 456 } 457 458 test("") 459 test("invalid:spec") 460 test("💩:spec") 461 } 462 463 type testProtocol struct { 464 name string 465 } 466 467 func (t *testProtocol) NewChunkStore(sp Spec) (chunks.ChunkStore, error) { 468 t.name = sp.DatabaseName 469 return chunks.NewMemoryStoreFactory().CreateStore(context.Background(), ""), nil 470 } 471 func (t *testProtocol) NewDatabase(sp Spec) (datas.Database, error) { 472 t.name = sp.DatabaseName 473 cs, err := t.NewChunkStore(sp) 474 d.PanicIfError(err) 475 return datas.NewDatabase(cs), nil 476 } 477 478 func noopGetAddrs(c chunks.Chunk) chunks.GetAddrsCb { 479 return func(ctx context.Context, addrs hash.HashSet, _ chunks.PendingRefExists) error { 480 return nil 481 } 482 } 483 484 func TestExternalProtocol(t *testing.T) { 485 assert := assert.New(t) 486 tp := testProtocol{} 487 ExternalProtocols["test"] = &tp 488 489 sp, err := ForDataset("test:foo::bar") 490 assert.NoError(err) 491 defer sp.Close() 492 assert.Equal("test", sp.Protocol) 493 assert.Equal("foo", sp.DatabaseName) 494 495 cs := sp.NewChunkStore(context.Background()) 496 assert.Equal("foo", tp.name) 497 c := chunks.NewChunk([]byte("hi!")) 498 err = cs.Put(context.Background(), c, noopGetAddrs) 499 assert.NoError(err) 500 ok, err := cs.Has(context.Background(), c.Hash()) 501 assert.NoError(err) 502 assert.True(ok) 503 504 tp.name = "" 505 ds := sp.GetDataset(context.Background()) 506 assert.Equal("foo", tp.name) 507 508 ds, err = datas.CommitValue(context.Background(), ds.Database(), ds, types.String("hi!")) 509 d.PanicIfError(err) 510 511 headVal, ok, err := ds.MaybeHeadValue() 512 assert.NoError(err) 513 assert.True(ok) 514 assert.True(types.String("hi!").Equals(headVal)) 515 }