github.com/hasnat/dolt/go@v0.0.0-20210628190320-9eb5d843fbb7/store/spec/spec_test.go (about) 1 // Copyright 2019 Dolthub, Inc. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 // 15 // This file incorporates work covered by the following copyright and 16 // permission notice: 17 // 18 // Copyright 2016 Attic Labs, Inc. All rights reserved. 19 // Licensed under the Apache License, version 2.0: 20 // http://www.apache.org/licenses/LICENSE-2.0 21 22 package spec 23 24 import ( 25 "context" 26 "fmt" 27 "io/ioutil" 28 "os" 29 "path/filepath" 30 "testing" 31 32 "github.com/stretchr/testify/assert" 33 34 "github.com/dolthub/dolt/go/store/chunks" 35 "github.com/dolthub/dolt/go/store/d" 36 "github.com/dolthub/dolt/go/store/datas" 37 "github.com/dolthub/dolt/go/store/hash" 38 "github.com/dolthub/dolt/go/store/nbs" 39 "github.com/dolthub/dolt/go/store/types" 40 ) 41 42 func mustValue(val types.Value, err error) types.Value { 43 d.PanicIfError(err) 44 return val 45 } 46 47 func mustType(t *types.Type, err error) *types.Type { 48 d.PanicIfError(err) 49 return t 50 } 51 52 func mustString(str string, err error) string { 53 d.PanicIfError(err) 54 return str 55 } 56 57 func mustList(l types.List, err error) types.List { 58 d.PanicIfError(err) 59 return l 60 } 61 62 func mustHash(h hash.Hash, err error) hash.Hash { 63 d.PanicIfError(err) 64 return h 65 } 66 67 func mustGetValue(v types.Value, found bool, err error) types.Value { 68 d.PanicIfError(err) 69 d.PanicIfFalse(found) 70 return v 71 } 72 73 func TestMemDatabaseSpec(t *testing.T) { 74 assert := assert.New(t) 75 76 spec, err := ForDatabase("mem") 77 assert.NoError(err) 78 defer spec.Close() 79 80 assert.Equal("mem", spec.Protocol) 81 assert.Equal("", spec.DatabaseName) 82 assert.True(spec.Path.IsEmpty()) 83 84 s := types.String("hello") 85 db := spec.GetDatabase(context.Background()) 86 db.WriteValue(context.Background(), s) 87 assert.Equal(s, mustValue(db.ReadValue(context.Background(), mustHash(s.Hash(types.Format_7_18))))) 88 } 89 90 func TestMemDatasetSpec(t *testing.T) { 91 assert := assert.New(t) 92 93 spec, err := ForDataset("mem::test") 94 assert.NoError(err) 95 defer spec.Close() 96 97 assert.Equal("mem", spec.Protocol) 98 assert.Equal("", spec.DatabaseName) 99 assert.Equal("test", spec.Path.Dataset) 100 assert.True(spec.Path.Path.IsEmpty()) 101 102 ds := spec.GetDataset(context.Background()) 103 _, ok, err := spec.GetDataset(context.Background()).MaybeHeadValue() 104 assert.NoError(err) 105 assert.False(ok) 106 107 s := types.String("hello") 108 ds, err = spec.GetDatabase(context.Background()).CommitValue(context.Background(), ds, s) 109 assert.NoError(err) 110 currHeadVal, ok, err := ds.MaybeHeadValue() 111 assert.NoError(err) 112 assert.True(ok) 113 assert.Equal(s, currHeadVal) 114 } 115 116 func TestMemHashPathSpec(t *testing.T) { 117 assert := assert.New(t) 118 119 s := types.String("hello") 120 121 spec, err := ForPath("mem::#" + mustHash(s.Hash(types.Format_7_18)).String()) 122 assert.NoError(err) 123 defer spec.Close() 124 125 assert.Equal("mem", spec.Protocol) 126 assert.Equal("", spec.DatabaseName) 127 assert.False(spec.Path.IsEmpty()) 128 129 // This is a reasonable check but it causes the next GetValue to return nil: 130 // assert.Nil(spec.GetValue()) 131 132 spec.GetDatabase(context.Background()).WriteValue(context.Background(), s) 133 assert.Equal(s, spec.GetValue(context.Background())) 134 } 135 136 func TestMemDatasetPathSpec(t *testing.T) { 137 assert := assert.New(t) 138 139 spec, err := ForPath("mem::test.value[0]") 140 assert.NoError(err) 141 defer spec.Close() 142 143 assert.Equal("mem", spec.Protocol) 144 assert.Equal("", spec.DatabaseName) 145 assert.False(spec.Path.IsEmpty()) 146 147 assert.Nil(spec.GetValue(context.Background())) 148 149 db := spec.GetDatabase(context.Background()) 150 ds, err := db.GetDataset(context.Background(), "test") 151 assert.NoError(err) 152 _, err = db.CommitValue(context.Background(), ds, mustList(types.NewList(context.Background(), db, types.Float(42)))) 153 assert.NoError(err) 154 155 assert.Equal(types.Float(42), spec.GetValue(context.Background())) 156 } 157 158 func TestNBSDatabaseSpec(t *testing.T) { 159 assert := assert.New(t) 160 161 run := func(prefix string) { 162 tmpDir, err := ioutil.TempDir("", "spec_test") 163 assert.NoError(err) 164 defer os.RemoveAll(tmpDir) 165 166 s := types.String("string") 167 168 // Existing database in the database are read from the spec. 169 store1 := filepath.Join(tmpDir, "store1") 170 os.Mkdir(store1, 0777) 171 func() { 172 cs, err := nbs.NewLocalStore(context.Background(), types.Format_Default.VersionString(), store1, 8*(1<<20)) 173 assert.NoError(err) 174 db := datas.NewDatabase(cs) 175 defer db.Close() 176 r, err := db.WriteValue(context.Background(), s) 177 assert.NoError(err) 178 ds, err := db.GetDataset(context.Background(), "datasetID") 179 assert.NoError(err) 180 _, err = db.CommitValue(context.Background(), ds, r) 181 assert.NoError(err) 182 }() 183 184 spec1, err := ForDatabase(prefix + store1) 185 assert.NoError(err) 186 defer spec1.Close() 187 188 assert.Equal("nbs", spec1.Protocol) 189 assert.Equal(store1, spec1.DatabaseName) 190 191 assert.Equal(s, mustValue(spec1.GetDatabase(context.Background()).ReadValue(context.Background(), mustHash(s.Hash(types.Format_7_18))))) 192 193 // New databases can be created and read/written from. 194 store2 := filepath.Join(tmpDir, "store2") 195 os.Mkdir(store2, 0777) 196 spec2, err := ForDatabase(prefix + store2) 197 assert.NoError(err) 198 defer spec2.Close() 199 200 assert.Equal("nbs", spec2.Protocol) 201 assert.Equal(store2, spec2.DatabaseName) 202 203 db := spec2.GetDatabase(context.Background()) 204 db.WriteValue(context.Background(), s) 205 r, err := db.WriteValue(context.Background(), s) 206 assert.NoError(err) 207 ds, err := db.GetDataset(context.Background(), "datasetID") 208 assert.NoError(err) 209 _, err = db.CommitValue(context.Background(), ds, r) 210 assert.NoError(err) 211 assert.Equal(s, mustValue(db.ReadValue(context.Background(), mustHash(s.Hash(types.Format_7_18))))) 212 } 213 214 run("") 215 run("nbs:") 216 } 217 218 // Skip LDB dataset and path tests: the database behaviour is tested in 219 // TestLDBDatabaseSpec, TestMemDatasetSpec/TestMem*PathSpec cover general 220 // dataset/path behaviour, and ForDataset/ForPath test LDB parsing. 221 222 func TestCloseSpecWithoutOpen(t *testing.T) { 223 s, err := ForDatabase("mem") 224 assert.NoError(t, err) 225 s.Close() 226 } 227 228 func TestHref(t *testing.T) { 229 assert := assert.New(t) 230 231 sp, _ := ForDatabase("aws://table/foo/bar/baz") 232 assert.Equal("aws://table/foo/bar/baz", sp.Href()) 233 sp, _ = ForDataset("aws://[table:bucket]/foo/bar/baz::myds") 234 assert.Equal("aws://[table:bucket]/foo/bar/baz", sp.Href()) 235 sp, _ = ForPath("aws://[table:bucket]/foo/bar/baz::myds.my.path") 236 assert.Equal("aws://[table:bucket]/foo/bar/baz", sp.Href()) 237 238 sp, err := ForPath("mem::myds.my.path") 239 assert.NoError(err) 240 assert.Equal("", sp.Href()) 241 } 242 243 func TestForDatabase(t *testing.T) { 244 assert := assert.New(t) 245 246 badSpecs := []string{ 247 "mem:stuff", 248 "mem::", 249 "mem:", 250 "ldb:", 251 "random:", 252 "random:random", 253 "/file/ba:d", 254 "aws://[t:b]", 255 "aws://t", 256 "aws://t:", 257 } 258 259 for _, spec := range badSpecs { 260 _, err := ForDatabase(spec) 261 assert.Error(err, spec) 262 } 263 264 tmpDir, err := ioutil.TempDir("", "spec_test") 265 assert.NoError(err) 266 defer os.RemoveAll(tmpDir) 267 268 testCases := []struct { 269 spec, protocol, databaseName, canonicalSpecIfAny string 270 }{ 271 {"mem", "mem", "", ""}, 272 {tmpDir, "nbs", tmpDir, "nbs:" + tmpDir}, 273 {"nbs:" + tmpDir, "nbs", tmpDir, ""}, 274 {"aws://[table:bucket]/db", "aws", "//[table:bucket]/db", ""}, 275 {"aws://table/db", "aws", "//table/db", ""}, 276 } 277 278 for _, tc := range testCases { 279 spec, err := ForDatabase(tc.spec) 280 assert.NoError(err, tc.spec) 281 defer spec.Close() 282 283 assert.Equal(tc.protocol, spec.Protocol) 284 assert.Equal(tc.databaseName, spec.DatabaseName) 285 assert.True(spec.Path.IsEmpty()) 286 287 if tc.canonicalSpecIfAny == "" { 288 assert.Equal(tc.spec, spec.String()) 289 } else { 290 assert.Equal(tc.canonicalSpecIfAny, spec.String()) 291 } 292 } 293 } 294 295 func TestForDataset(t *testing.T) { 296 assert := assert.New(t) 297 298 badSpecs := []string{ 299 "mem", 300 "mem:", 301 "mem:::ds", 302 "monkey", 303 "monkey:balls", 304 "mem:/a/bogus/path:dsname", 305 "nbs:", 306 "nbs:hello", 307 "aws://[t:b]/db", 308 "mem::foo.value", 309 } 310 311 for _, spec := range badSpecs { 312 _, err := ForDataset(spec) 313 assert.Error(err, spec) 314 } 315 316 invalidDatasetNames := []string{" ", "", "$", "#", ":", "\n", "💩"} 317 for _, s := range invalidDatasetNames { 318 _, err := ForDataset("mem::" + s) 319 assert.Error(err) 320 } 321 322 validDatasetNames := []string{"a", "Z", "0", "/", "-", "_"} 323 for _, s := range validDatasetNames { 324 _, err := ForDataset("mem::" + s) 325 assert.NoError(err) 326 } 327 328 tmpDir, err := ioutil.TempDir("", "spec_test") 329 assert.NoError(err) 330 defer os.RemoveAll(tmpDir) 331 332 testCases := []struct { 333 spec, protocol, databaseName, datasetName, canonicalSpecIfAny string 334 }{ 335 {"nbs:" + tmpDir + "::ds/one", "nbs", tmpDir, "ds/one", ""}, 336 {tmpDir + "::ds/one", "nbs", tmpDir, "ds/one", "nbs:" + tmpDir + "::ds/one"}, 337 {"aws://[table:bucket]/db::ds", "aws", "//[table:bucket]/db", "ds", ""}, 338 {"aws://table/db::ds", "aws", "//table/db", "ds", ""}, 339 } 340 341 for _, tc := range testCases { 342 spec, err := ForDataset(tc.spec) 343 assert.NoError(err, tc.spec) 344 defer spec.Close() 345 346 assert.Equal(tc.protocol, spec.Protocol) 347 assert.Equal(tc.databaseName, spec.DatabaseName) 348 assert.Equal(tc.datasetName, spec.Path.Dataset) 349 350 if tc.canonicalSpecIfAny == "" { 351 assert.Equal(tc.spec, spec.String()) 352 } else { 353 assert.Equal(tc.canonicalSpecIfAny, spec.String()) 354 } 355 } 356 } 357 358 func TestForPath(t *testing.T) { 359 assert := assert.New(t) 360 361 badSpecs := []string{ 362 "mem::#", 363 "mem::#s", 364 "mem::#foobarbaz", 365 "mem::#wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww", 366 } 367 368 for _, bs := range badSpecs { 369 _, err := ForPath(bs) 370 assert.Error(err) 371 } 372 373 tmpDir, err := ioutil.TempDir("", "spec_test") 374 assert.NoError(err) 375 defer os.RemoveAll(tmpDir) 376 377 testCases := []struct { 378 spec, protocol, databaseName, pathString, canonicalSpecIfAny string 379 }{ 380 {tmpDir + "::#0123456789abcdefghijklmnopqrstuv", "nbs", tmpDir, "#0123456789abcdefghijklmnopqrstuv", "nbs:" + tmpDir + "::#0123456789abcdefghijklmnopqrstuv"}, 381 {"nbs:" + tmpDir + "::#0123456789abcdefghijklmnopqrstuv", "nbs", tmpDir, "#0123456789abcdefghijklmnopqrstuv", ""}, 382 {"mem::#0123456789abcdefghijklmnopqrstuv", "mem", "", "#0123456789abcdefghijklmnopqrstuv", ""}, 383 {"aws://[table:bucket]/db::foo.foo", "aws", "//[table:bucket]/db", "foo.foo", ""}, 384 {"aws://table/db::foo.foo", "aws", "//table/db", "foo.foo", ""}, 385 } 386 387 for _, tc := range testCases { 388 spec, err := ForPath(tc.spec) 389 assert.NoError(err) 390 defer spec.Close() 391 392 assert.Equal(tc.protocol, spec.Protocol) 393 assert.Equal(tc.databaseName, spec.DatabaseName) 394 assert.Equal(tc.pathString, spec.Path.String()) 395 396 if tc.canonicalSpecIfAny == "" { 397 assert.Equal(tc.spec, spec.String()) 398 } else { 399 assert.Equal(tc.canonicalSpecIfAny, spec.String()) 400 } 401 } 402 } 403 404 func TestPinPathSpec(t *testing.T) { 405 assert := assert.New(t) 406 407 unpinned, err := ForPath("mem::foo.value") 408 assert.NoError(err) 409 defer unpinned.Close() 410 411 db := unpinned.GetDatabase(context.Background()) 412 ds, err := db.GetDataset(context.Background(), "foo") 413 assert.NoError(err) 414 _, err = db.CommitValue(context.Background(), ds, types.Float(42)) 415 assert.NoError(err) 416 417 pinned, ok := unpinned.Pin(context.Background()) 418 assert.True(ok) 419 defer pinned.Close() 420 421 ds, err = db.GetDataset(context.Background(), "foo") 422 assert.NoError(err) 423 head, ok := ds.MaybeHead() 424 assert.True(ok) 425 426 assert.Equal(mustHash(head.Hash(types.Format_7_18)), pinned.Path.Hash) 427 assert.Equal(fmt.Sprintf("mem::#%s.value", mustHash(head.Hash(types.Format_7_18)).String()), pinned.String()) 428 assert.Equal(types.Float(42), pinned.GetValue(context.Background())) 429 assert.Equal(types.Float(42), unpinned.GetValue(context.Background())) 430 431 ds, err = db.GetDataset(context.Background(), "foo") 432 assert.NoError(err) 433 _, err = db.CommitValue(context.Background(), ds, types.Float(43)) 434 assert.NoError(err) 435 assert.Equal(types.Float(42), pinned.GetValue(context.Background())) 436 assert.Equal(types.Float(43), unpinned.GetValue(context.Background())) 437 } 438 439 func TestPinDatasetSpec(t *testing.T) { 440 assert := assert.New(t) 441 442 unpinned, err := ForDataset("mem::foo") 443 assert.NoError(err) 444 defer unpinned.Close() 445 446 db := unpinned.GetDatabase(context.Background()) 447 ds, err := db.GetDataset(context.Background(), "foo") 448 assert.NoError(err) 449 _, err = db.CommitValue(context.Background(), ds, types.Float(42)) 450 assert.NoError(err) 451 452 pinned, ok := unpinned.Pin(context.Background()) 453 assert.True(ok) 454 defer pinned.Close() 455 456 ds, err = db.GetDataset(context.Background(), "foo") 457 assert.NoError(err) 458 head, ok := ds.MaybeHead() 459 assert.True(ok) 460 461 commitValue := func(val types.Value) types.Value { 462 v, ok, err := val.(types.Struct).MaybeGet(datas.ValueField) 463 d.PanicIfError(err) 464 d.PanicIfFalse(ok) 465 return v 466 } 467 468 assert.Equal(mustHash(head.Hash(types.Format_7_18)), pinned.Path.Hash) 469 assert.Equal(fmt.Sprintf("mem::#%s", mustHash(head.Hash(types.Format_7_18)).String()), pinned.String()) 470 assert.Equal(types.Float(42), commitValue(pinned.GetValue(context.Background()))) 471 headVal, ok, err := unpinned.GetDataset(context.Background()).MaybeHeadValue() 472 assert.NoError(err) 473 assert.True(ok) 474 assert.Equal(types.Float(42), headVal) 475 476 ds, err = db.GetDataset(context.Background(), "foo") 477 assert.NoError(err) 478 _, err = db.CommitValue(context.Background(), ds, types.Float(43)) 479 assert.NoError(err) 480 assert.Equal(types.Float(42), commitValue(pinned.GetValue(context.Background()))) 481 headVal, ok, err = unpinned.GetDataset(context.Background()).MaybeHeadValue() 482 assert.NoError(err) 483 assert.True(ok) 484 assert.Equal(types.Float(43), headVal) 485 } 486 487 func TestAlreadyPinnedPathSpec(t *testing.T) { 488 assert := assert.New(t) 489 490 unpinned, err := ForPath("mem::#imgp9mp1h3b9nv0gna6mri53dlj9f4ql.value") 491 assert.NoError(err) 492 pinned, ok := unpinned.Pin(context.Background()) 493 assert.True(ok) 494 assert.Equal(unpinned, pinned) 495 } 496 497 func TestMultipleSpecsSameNBS(t *testing.T) { 498 assert := assert.New(t) 499 500 tmpDir, err := ioutil.TempDir("", "spec_test") 501 assert.NoError(err) 502 defer os.RemoveAll(tmpDir) 503 504 spec1, err1 := ForDatabase(tmpDir) 505 spec2, err2 := ForDatabase(tmpDir) 506 507 assert.NoError(err1) 508 assert.NoError(err2) 509 510 s := types.String("hello") 511 db := spec1.GetDatabase(context.Background()) 512 r, err := db.WriteValue(context.Background(), s) 513 assert.NoError(err) 514 ds, err := db.GetDataset(context.Background(), "datasetID") 515 assert.NoError(err) 516 _, err = db.CommitValue(context.Background(), ds, r) 517 assert.NoError(err) 518 assert.Equal(s, mustValue(spec2.GetDatabase(context.Background()).ReadValue(context.Background(), mustHash(s.Hash(types.Format_7_18))))) 519 } 520 521 func TestAcccessingInvalidSpec(t *testing.T) { 522 assert := assert.New(t) 523 524 test := func(spec string) { 525 sp, err := ForDatabase(spec) 526 assert.Error(err) 527 assert.Equal("", sp.Href()) 528 assert.Panics(func() { sp.GetDatabase(context.Background()) }) 529 assert.Panics(func() { sp.GetDatabase(context.Background()) }) 530 assert.Panics(func() { sp.NewChunkStore(context.Background()) }) 531 assert.Panics(func() { sp.NewChunkStore(context.Background()) }) 532 assert.Panics(func() { sp.Close() }) 533 assert.Panics(func() { sp.Close() }) 534 // Spec was created with ForDatabase, so dataset/path related functions 535 // should just fail not panic. 536 _, ok := sp.Pin(context.Background()) 537 assert.False(ok) 538 assert.Equal(datas.Dataset{}, sp.GetDataset(context.Background())) 539 assert.Nil(sp.GetValue(context.Background())) 540 } 541 542 test("") 543 test("invalid:spec") 544 test("💩:spec") 545 } 546 547 type testProtocol struct { 548 name string 549 } 550 551 func (t *testProtocol) NewChunkStore(sp Spec) (chunks.ChunkStore, error) { 552 t.name = sp.DatabaseName 553 return chunks.NewMemoryStoreFactory().CreateStore(context.Background(), ""), nil 554 } 555 func (t *testProtocol) NewDatabase(sp Spec) (datas.Database, error) { 556 t.name = sp.DatabaseName 557 cs, err := t.NewChunkStore(sp) 558 d.PanicIfError(err) 559 return datas.NewDatabase(cs), nil 560 } 561 562 func TestExternalProtocol(t *testing.T) { 563 assert := assert.New(t) 564 tp := testProtocol{} 565 ExternalProtocols["test"] = &tp 566 567 sp, err := ForDataset("test:foo::bar") 568 assert.NoError(err) 569 assert.Equal("test", sp.Protocol) 570 assert.Equal("foo", sp.DatabaseName) 571 572 cs := sp.NewChunkStore(context.Background()) 573 assert.Equal("foo", tp.name) 574 c := chunks.NewChunk([]byte("hi!")) 575 err = cs.Put(context.Background(), c) 576 assert.NoError(err) 577 ok, err := cs.Has(context.Background(), c.Hash()) 578 assert.NoError(err) 579 assert.True(ok) 580 581 tp.name = "" 582 ds := sp.GetDataset(context.Background()) 583 assert.Equal("foo", tp.name) 584 585 ds, err = ds.Database().CommitValue(context.Background(), ds, types.String("hi!")) 586 d.PanicIfError(err) 587 588 headVal, ok, err := ds.MaybeHeadValue() 589 assert.NoError(err) 590 assert.True(ok) 591 assert.True(types.String("hi!").Equals(headVal)) 592 }