github.com/lusis/distribution@v2.0.1+incompatible/registry/storage/driver/testsuites/testsuites.go (about) 1 package testsuites 2 3 import ( 4 "bytes" 5 "crypto/sha1" 6 "io" 7 "io/ioutil" 8 "math/rand" 9 "net/http" 10 "os" 11 "path" 12 "sort" 13 "sync" 14 "testing" 15 "time" 16 17 storagedriver "github.com/docker/distribution/registry/storage/driver" 18 "gopkg.in/check.v1" 19 ) 20 21 // Test hooks up gocheck into the "go test" runner. 22 func Test(t *testing.T) { check.TestingT(t) } 23 24 // RegisterInProcessSuite registers an in-process storage driver test suite with 25 // the go test runner. 26 func RegisterInProcessSuite(driverConstructor DriverConstructor, skipCheck SkipCheck) { 27 check.Suite(&DriverSuite{ 28 Constructor: driverConstructor, 29 SkipCheck: skipCheck, 30 }) 31 } 32 33 // RegisterIPCSuite registers a storage driver test suite which runs the named 34 // driver as a child process with the given parameters. 35 func RegisterIPCSuite(driverName string, ipcParams map[string]string, skipCheck SkipCheck) { 36 panic("ipc testing is disabled for now") 37 38 // NOTE(stevvooe): IPC testing is disabled for now. Uncomment the code 39 // block before and remove the panic when we phase it back in. 40 41 // suite := &DriverSuite{ 42 // Constructor: func() (storagedriver.StorageDriver, error) { 43 // d, err := ipc.NewDriverClient(driverName, ipcParams) 44 // if err != nil { 45 // return nil, err 46 // } 47 // err = d.Start() 48 // if err != nil { 49 // return nil, err 50 // } 51 // return d, nil 52 // }, 53 // SkipCheck: skipCheck, 54 // } 55 // suite.Teardown = func() error { 56 // if suite.StorageDriver == nil { 57 // return nil 58 // } 59 60 // driverClient := suite.StorageDriver.(*ipc.StorageDriverClient) 61 // return driverClient.Stop() 62 // } 63 // check.Suite(suite) 64 } 65 66 // SkipCheck is a function used to determine if a test suite should be skipped. 67 // If a SkipCheck returns a non-empty skip reason, the suite is skipped with 68 // the given reason. 69 type SkipCheck func() (reason string) 70 71 // NeverSkip is a default SkipCheck which never skips the suite. 72 var NeverSkip SkipCheck = func() string { return "" } 73 74 // DriverConstructor is a function which returns a new 75 // storagedriver.StorageDriver. 76 type DriverConstructor func() (storagedriver.StorageDriver, error) 77 78 // DriverTeardown is a function which cleans up a suite's 79 // storagedriver.StorageDriver. 80 type DriverTeardown func() error 81 82 // DriverSuite is a gocheck test suite designed to test a 83 // storagedriver.StorageDriver. 84 // The intended way to create a DriverSuite is with RegisterInProcessSuite or 85 // RegisterIPCSuite. 86 type DriverSuite struct { 87 Constructor DriverConstructor 88 Teardown DriverTeardown 89 SkipCheck 90 storagedriver.StorageDriver 91 } 92 93 // SetUpSuite sets up the gocheck test suite. 94 func (suite *DriverSuite) SetUpSuite(c *check.C) { 95 if reason := suite.SkipCheck(); reason != "" { 96 c.Skip(reason) 97 } 98 d, err := suite.Constructor() 99 c.Assert(err, check.IsNil) 100 suite.StorageDriver = d 101 } 102 103 // TearDownSuite tears down the gocheck test suite. 104 func (suite *DriverSuite) TearDownSuite(c *check.C) { 105 if suite.Teardown != nil { 106 err := suite.Teardown() 107 c.Assert(err, check.IsNil) 108 } 109 } 110 111 // TearDownTest tears down the gocheck test. 112 // This causes the suite to abort if any files are left around in the storage 113 // driver. 114 func (suite *DriverSuite) TearDownTest(c *check.C) { 115 files, _ := suite.StorageDriver.List("/") 116 if len(files) > 0 { 117 c.Fatalf("Storage driver did not clean up properly. Offending files: %#v", files) 118 } 119 } 120 121 // TestValidPaths checks that various valid file paths are accepted by the 122 // storage driver. 123 func (suite *DriverSuite) TestValidPaths(c *check.C) { 124 contents := randomContents(64) 125 validFiles := []string{ 126 "/a", 127 "/2", 128 "/aa", 129 "/a.a", 130 "/0-9/abcdefg", 131 "/abcdefg/z.75", 132 "/abc/1.2.3.4.5-6_zyx/123.z/4", 133 "/docker/docker-registry", 134 "/123.abc", 135 "/abc./abc", 136 "/.abc", 137 "/a--b", 138 "/a-.b", 139 "/_.abc", 140 "/Docker/docker-registry", 141 "/Abc/Cba"} 142 143 for _, filename := range validFiles { 144 err := suite.StorageDriver.PutContent(filename, contents) 145 defer suite.StorageDriver.Delete(firstPart(filename)) 146 c.Assert(err, check.IsNil) 147 148 received, err := suite.StorageDriver.GetContent(filename) 149 c.Assert(err, check.IsNil) 150 c.Assert(received, check.DeepEquals, contents) 151 } 152 } 153 154 // TestInvalidPaths checks that various invalid file paths are rejected by the 155 // storage driver. 156 func (suite *DriverSuite) TestInvalidPaths(c *check.C) { 157 contents := randomContents(64) 158 invalidFiles := []string{ 159 "", 160 "/", 161 "abc", 162 "123.abc", 163 "//bcd", 164 "/abc_123/"} 165 166 for _, filename := range invalidFiles { 167 err := suite.StorageDriver.PutContent(filename, contents) 168 defer suite.StorageDriver.Delete(firstPart(filename)) 169 c.Assert(err, check.NotNil) 170 c.Assert(err, check.FitsTypeOf, storagedriver.InvalidPathError{}) 171 172 _, err = suite.StorageDriver.GetContent(filename) 173 c.Assert(err, check.NotNil) 174 c.Assert(err, check.FitsTypeOf, storagedriver.InvalidPathError{}) 175 } 176 } 177 178 // TestWriteRead1 tests a simple write-read workflow. 179 func (suite *DriverSuite) TestWriteRead1(c *check.C) { 180 filename := randomPath(32) 181 contents := []byte("a") 182 suite.writeReadCompare(c, filename, contents) 183 } 184 185 // TestWriteRead2 tests a simple write-read workflow with unicode data. 186 func (suite *DriverSuite) TestWriteRead2(c *check.C) { 187 filename := randomPath(32) 188 contents := []byte("\xc3\x9f") 189 suite.writeReadCompare(c, filename, contents) 190 } 191 192 // TestWriteRead3 tests a simple write-read workflow with a small string. 193 func (suite *DriverSuite) TestWriteRead3(c *check.C) { 194 filename := randomPath(32) 195 contents := randomContents(32) 196 suite.writeReadCompare(c, filename, contents) 197 } 198 199 // TestWriteRead4 tests a simple write-read workflow with 1MB of data. 200 func (suite *DriverSuite) TestWriteRead4(c *check.C) { 201 filename := randomPath(32) 202 contents := randomContents(1024 * 1024) 203 suite.writeReadCompare(c, filename, contents) 204 } 205 206 // TestWriteReadNonUTF8 tests that non-utf8 data may be written to the storage 207 // driver safely. 208 func (suite *DriverSuite) TestWriteReadNonUTF8(c *check.C) { 209 filename := randomPath(32) 210 contents := []byte{0x80, 0x80, 0x80, 0x80} 211 suite.writeReadCompare(c, filename, contents) 212 } 213 214 // TestTruncate tests that putting smaller contents than an original file does 215 // remove the excess contents. 216 func (suite *DriverSuite) TestTruncate(c *check.C) { 217 filename := randomPath(32) 218 contents := randomContents(1024 * 1024) 219 suite.writeReadCompare(c, filename, contents) 220 221 contents = randomContents(1024) 222 suite.writeReadCompare(c, filename, contents) 223 } 224 225 // TestReadNonexistent tests reading content from an empty path. 226 func (suite *DriverSuite) TestReadNonexistent(c *check.C) { 227 filename := randomPath(32) 228 _, err := suite.StorageDriver.GetContent(filename) 229 c.Assert(err, check.NotNil) 230 c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) 231 } 232 233 // TestWriteReadStreams1 tests a simple write-read streaming workflow. 234 func (suite *DriverSuite) TestWriteReadStreams1(c *check.C) { 235 filename := randomPath(32) 236 contents := []byte("a") 237 suite.writeReadCompareStreams(c, filename, contents) 238 } 239 240 // TestWriteReadStreams2 tests a simple write-read streaming workflow with 241 // unicode data. 242 func (suite *DriverSuite) TestWriteReadStreams2(c *check.C) { 243 filename := randomPath(32) 244 contents := []byte("\xc3\x9f") 245 suite.writeReadCompareStreams(c, filename, contents) 246 } 247 248 // TestWriteReadStreams3 tests a simple write-read streaming workflow with a 249 // small amount of data. 250 func (suite *DriverSuite) TestWriteReadStreams3(c *check.C) { 251 filename := randomPath(32) 252 contents := randomContents(32) 253 suite.writeReadCompareStreams(c, filename, contents) 254 } 255 256 // TestWriteReadStreams4 tests a simple write-read streaming workflow with 1MB 257 // of data. 258 func (suite *DriverSuite) TestWriteReadStreams4(c *check.C) { 259 filename := randomPath(32) 260 contents := randomContents(1024 * 1024) 261 suite.writeReadCompareStreams(c, filename, contents) 262 } 263 264 // TestWriteReadStreamsNonUTF8 tests that non-utf8 data may be written to the 265 // storage driver safely. 266 func (suite *DriverSuite) TestWriteReadStreamsNonUTF8(c *check.C) { 267 filename := randomPath(32) 268 contents := []byte{0x80, 0x80, 0x80, 0x80} 269 suite.writeReadCompareStreams(c, filename, contents) 270 } 271 272 // TestWriteReadLargeStreams tests that a 5GB file may be written to the storage 273 // driver safely. 274 func (suite *DriverSuite) TestWriteReadLargeStreams(c *check.C) { 275 if testing.Short() { 276 c.Skip("Skipping test in short mode") 277 } 278 279 filename := randomPath(32) 280 defer suite.StorageDriver.Delete(firstPart(filename)) 281 282 checksum := sha1.New() 283 var fileSize int64 = 5 * 1024 * 1024 * 1024 284 285 contents := newRandReader(fileSize) 286 written, err := suite.StorageDriver.WriteStream(filename, 0, io.TeeReader(contents, checksum)) 287 c.Assert(err, check.IsNil) 288 c.Assert(written, check.Equals, fileSize) 289 290 reader, err := suite.StorageDriver.ReadStream(filename, 0) 291 c.Assert(err, check.IsNil) 292 293 writtenChecksum := sha1.New() 294 io.Copy(writtenChecksum, reader) 295 296 c.Assert(writtenChecksum.Sum(nil), check.DeepEquals, checksum.Sum(nil)) 297 } 298 299 // TestReadStreamWithOffset tests that the appropriate data is streamed when 300 // reading with a given offset. 301 func (suite *DriverSuite) TestReadStreamWithOffset(c *check.C) { 302 filename := randomPath(32) 303 defer suite.StorageDriver.Delete(firstPart(filename)) 304 305 chunkSize := int64(32) 306 307 contentsChunk1 := randomContents(chunkSize) 308 contentsChunk2 := randomContents(chunkSize) 309 contentsChunk3 := randomContents(chunkSize) 310 311 err := suite.StorageDriver.PutContent(filename, append(append(contentsChunk1, contentsChunk2...), contentsChunk3...)) 312 c.Assert(err, check.IsNil) 313 314 reader, err := suite.StorageDriver.ReadStream(filename, 0) 315 c.Assert(err, check.IsNil) 316 defer reader.Close() 317 318 readContents, err := ioutil.ReadAll(reader) 319 c.Assert(err, check.IsNil) 320 321 c.Assert(readContents, check.DeepEquals, append(append(contentsChunk1, contentsChunk2...), contentsChunk3...)) 322 323 reader, err = suite.StorageDriver.ReadStream(filename, chunkSize) 324 c.Assert(err, check.IsNil) 325 defer reader.Close() 326 327 readContents, err = ioutil.ReadAll(reader) 328 c.Assert(err, check.IsNil) 329 330 c.Assert(readContents, check.DeepEquals, append(contentsChunk2, contentsChunk3...)) 331 332 reader, err = suite.StorageDriver.ReadStream(filename, chunkSize*2) 333 c.Assert(err, check.IsNil) 334 defer reader.Close() 335 336 readContents, err = ioutil.ReadAll(reader) 337 c.Assert(err, check.IsNil) 338 c.Assert(readContents, check.DeepEquals, contentsChunk3) 339 340 // Ensure we get invalid offest for negative offsets. 341 reader, err = suite.StorageDriver.ReadStream(filename, -1) 342 c.Assert(err, check.FitsTypeOf, storagedriver.InvalidOffsetError{}) 343 c.Assert(err.(storagedriver.InvalidOffsetError).Offset, check.Equals, int64(-1)) 344 c.Assert(err.(storagedriver.InvalidOffsetError).Path, check.Equals, filename) 345 c.Assert(reader, check.IsNil) 346 347 // Read past the end of the content and make sure we get a reader that 348 // returns 0 bytes and io.EOF 349 reader, err = suite.StorageDriver.ReadStream(filename, chunkSize*3) 350 c.Assert(err, check.IsNil) 351 defer reader.Close() 352 353 buf := make([]byte, chunkSize) 354 n, err := reader.Read(buf) 355 c.Assert(err, check.Equals, io.EOF) 356 c.Assert(n, check.Equals, 0) 357 358 // Check the N-1 boundary condition, ensuring we get 1 byte then io.EOF. 359 reader, err = suite.StorageDriver.ReadStream(filename, chunkSize*3-1) 360 c.Assert(err, check.IsNil) 361 defer reader.Close() 362 363 n, err = reader.Read(buf) 364 c.Assert(n, check.Equals, 1) 365 366 // We don't care whether the io.EOF comes on the this read or the first 367 // zero read, but the only error acceptable here is io.EOF. 368 if err != nil { 369 c.Assert(err, check.Equals, io.EOF) 370 } 371 372 // Any more reads should result in zero bytes and io.EOF 373 n, err = reader.Read(buf) 374 c.Assert(n, check.Equals, 0) 375 c.Assert(err, check.Equals, io.EOF) 376 } 377 378 // TestContinueStreamAppendLarge tests that a stream write can be appended to without 379 // corrupting the data with a large chunk size. 380 func (suite *DriverSuite) TestContinueStreamAppendLarge(c *check.C) { 381 suite.testContinueStreamAppend(c, int64(10*1024*1024)) 382 } 383 384 // TestContinueStreamAppendSmall is the same as TestContinueStreamAppendLarge, but only 385 // with a tiny chunk size in order to test corner cases for some cloud storage drivers. 386 func (suite *DriverSuite) TestContinueStreamAppendSmall(c *check.C) { 387 suite.testContinueStreamAppend(c, int64(32)) 388 } 389 390 func (suite *DriverSuite) testContinueStreamAppend(c *check.C, chunkSize int64) { 391 filename := randomPath(32) 392 defer suite.StorageDriver.Delete(firstPart(filename)) 393 394 contentsChunk1 := randomContents(chunkSize) 395 contentsChunk2 := randomContents(chunkSize) 396 contentsChunk3 := randomContents(chunkSize) 397 contentsChunk4 := randomContents(chunkSize) 398 zeroChunk := make([]byte, int64(chunkSize)) 399 400 fullContents := append(append(contentsChunk1, contentsChunk2...), contentsChunk3...) 401 402 nn, err := suite.StorageDriver.WriteStream(filename, 0, bytes.NewReader(contentsChunk1)) 403 c.Assert(err, check.IsNil) 404 c.Assert(nn, check.Equals, int64(len(contentsChunk1))) 405 406 fi, err := suite.StorageDriver.Stat(filename) 407 c.Assert(err, check.IsNil) 408 c.Assert(fi, check.NotNil) 409 c.Assert(fi.Size(), check.Equals, int64(len(contentsChunk1))) 410 411 nn, err = suite.StorageDriver.WriteStream(filename, fi.Size(), bytes.NewReader(contentsChunk2)) 412 c.Assert(err, check.IsNil) 413 c.Assert(nn, check.Equals, int64(len(contentsChunk2))) 414 415 fi, err = suite.StorageDriver.Stat(filename) 416 c.Assert(err, check.IsNil) 417 c.Assert(fi, check.NotNil) 418 c.Assert(fi.Size(), check.Equals, 2*chunkSize) 419 420 // Test re-writing the last chunk 421 nn, err = suite.StorageDriver.WriteStream(filename, fi.Size()-chunkSize, bytes.NewReader(contentsChunk2)) 422 c.Assert(err, check.IsNil) 423 c.Assert(nn, check.Equals, int64(len(contentsChunk2))) 424 425 fi, err = suite.StorageDriver.Stat(filename) 426 c.Assert(err, check.IsNil) 427 c.Assert(fi, check.NotNil) 428 c.Assert(fi.Size(), check.Equals, 2*chunkSize) 429 430 nn, err = suite.StorageDriver.WriteStream(filename, fi.Size(), bytes.NewReader(fullContents[fi.Size():])) 431 c.Assert(err, check.IsNil) 432 c.Assert(nn, check.Equals, int64(len(fullContents[fi.Size():]))) 433 434 received, err := suite.StorageDriver.GetContent(filename) 435 c.Assert(err, check.IsNil) 436 c.Assert(received, check.DeepEquals, fullContents) 437 438 // Writing past size of file extends file (no offset error). We would like 439 // to write chunk 4 one chunk length past chunk 3. It should be successful 440 // and the resulting file will be 5 chunks long, with a chunk of all 441 // zeros. 442 443 fullContents = append(fullContents, zeroChunk...) 444 fullContents = append(fullContents, contentsChunk4...) 445 446 nn, err = suite.StorageDriver.WriteStream(filename, int64(len(fullContents))-chunkSize, bytes.NewReader(contentsChunk4)) 447 c.Assert(err, check.IsNil) 448 c.Assert(nn, check.Equals, chunkSize) 449 450 fi, err = suite.StorageDriver.Stat(filename) 451 c.Assert(err, check.IsNil) 452 c.Assert(fi, check.NotNil) 453 c.Assert(fi.Size(), check.Equals, int64(len(fullContents))) 454 455 received, err = suite.StorageDriver.GetContent(filename) 456 c.Assert(err, check.IsNil) 457 c.Assert(len(received), check.Equals, len(fullContents)) 458 c.Assert(received[chunkSize*3:chunkSize*4], check.DeepEquals, zeroChunk) 459 c.Assert(received[chunkSize*4:chunkSize*5], check.DeepEquals, contentsChunk4) 460 c.Assert(received, check.DeepEquals, fullContents) 461 462 // Ensure that negative offsets return correct error. 463 nn, err = suite.StorageDriver.WriteStream(filename, -1, bytes.NewReader(zeroChunk)) 464 c.Assert(err, check.NotNil) 465 c.Assert(err, check.FitsTypeOf, storagedriver.InvalidOffsetError{}) 466 c.Assert(err.(storagedriver.InvalidOffsetError).Path, check.Equals, filename) 467 c.Assert(err.(storagedriver.InvalidOffsetError).Offset, check.Equals, int64(-1)) 468 } 469 470 // TestReadNonexistentStream tests that reading a stream for a nonexistent path 471 // fails. 472 func (suite *DriverSuite) TestReadNonexistentStream(c *check.C) { 473 filename := randomPath(32) 474 475 _, err := suite.StorageDriver.ReadStream(filename, 0) 476 c.Assert(err, check.NotNil) 477 c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) 478 479 _, err = suite.StorageDriver.ReadStream(filename, 64) 480 c.Assert(err, check.NotNil) 481 c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) 482 } 483 484 // TestList checks the returned list of keys after populating a directory tree. 485 func (suite *DriverSuite) TestList(c *check.C) { 486 rootDirectory := "/" + randomFilename(int64(8+rand.Intn(8))) 487 defer suite.StorageDriver.Delete(rootDirectory) 488 489 parentDirectory := rootDirectory + "/" + randomFilename(int64(8+rand.Intn(8))) 490 childFiles := make([]string, 50) 491 for i := 0; i < len(childFiles); i++ { 492 childFile := parentDirectory + "/" + randomFilename(int64(8+rand.Intn(8))) 493 childFiles[i] = childFile 494 err := suite.StorageDriver.PutContent(childFile, randomContents(32)) 495 c.Assert(err, check.IsNil) 496 } 497 sort.Strings(childFiles) 498 499 keys, err := suite.StorageDriver.List("/") 500 c.Assert(err, check.IsNil) 501 c.Assert(keys, check.DeepEquals, []string{rootDirectory}) 502 503 keys, err = suite.StorageDriver.List(rootDirectory) 504 c.Assert(err, check.IsNil) 505 c.Assert(keys, check.DeepEquals, []string{parentDirectory}) 506 507 keys, err = suite.StorageDriver.List(parentDirectory) 508 c.Assert(err, check.IsNil) 509 510 sort.Strings(keys) 511 c.Assert(keys, check.DeepEquals, childFiles) 512 513 // A few checks to add here (check out #819 for more discussion on this): 514 // 1. Ensure that all paths are absolute. 515 // 2. Ensure that listings only include direct children. 516 // 3. Ensure that we only respond to directory listings that end with a slash (maybe?). 517 } 518 519 // TestMove checks that a moved object no longer exists at the source path and 520 // does exist at the destination. 521 func (suite *DriverSuite) TestMove(c *check.C) { 522 contents := randomContents(32) 523 sourcePath := randomPath(32) 524 destPath := randomPath(32) 525 526 defer suite.StorageDriver.Delete(firstPart(sourcePath)) 527 defer suite.StorageDriver.Delete(firstPart(destPath)) 528 529 err := suite.StorageDriver.PutContent(sourcePath, contents) 530 c.Assert(err, check.IsNil) 531 532 err = suite.StorageDriver.Move(sourcePath, destPath) 533 c.Assert(err, check.IsNil) 534 535 received, err := suite.StorageDriver.GetContent(destPath) 536 c.Assert(err, check.IsNil) 537 c.Assert(received, check.DeepEquals, contents) 538 539 _, err = suite.StorageDriver.GetContent(sourcePath) 540 c.Assert(err, check.NotNil) 541 c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) 542 } 543 544 // TestMoveOverwrite checks that a moved object no longer exists at the source 545 // path and overwrites the contents at the destination. 546 func (suite *DriverSuite) TestMoveOverwrite(c *check.C) { 547 sourcePath := randomPath(32) 548 destPath := randomPath(32) 549 sourceContents := randomContents(32) 550 destContents := randomContents(64) 551 552 defer suite.StorageDriver.Delete(firstPart(sourcePath)) 553 defer suite.StorageDriver.Delete(firstPart(destPath)) 554 555 err := suite.StorageDriver.PutContent(sourcePath, sourceContents) 556 c.Assert(err, check.IsNil) 557 558 err = suite.StorageDriver.PutContent(destPath, destContents) 559 c.Assert(err, check.IsNil) 560 561 err = suite.StorageDriver.Move(sourcePath, destPath) 562 c.Assert(err, check.IsNil) 563 564 received, err := suite.StorageDriver.GetContent(destPath) 565 c.Assert(err, check.IsNil) 566 c.Assert(received, check.DeepEquals, sourceContents) 567 568 _, err = suite.StorageDriver.GetContent(sourcePath) 569 c.Assert(err, check.NotNil) 570 c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) 571 } 572 573 // TestMoveNonexistent checks that moving a nonexistent key fails and does not 574 // delete the data at the destination path. 575 func (suite *DriverSuite) TestMoveNonexistent(c *check.C) { 576 contents := randomContents(32) 577 sourcePath := randomPath(32) 578 destPath := randomPath(32) 579 580 defer suite.StorageDriver.Delete(firstPart(destPath)) 581 582 err := suite.StorageDriver.PutContent(destPath, contents) 583 c.Assert(err, check.IsNil) 584 585 err = suite.StorageDriver.Move(sourcePath, destPath) 586 c.Assert(err, check.NotNil) 587 c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) 588 589 received, err := suite.StorageDriver.GetContent(destPath) 590 c.Assert(err, check.IsNil) 591 c.Assert(received, check.DeepEquals, contents) 592 } 593 594 // TestMoveInvalid provides various checks for invalid moves. 595 func (suite *DriverSuite) TestMoveInvalid(c *check.C) { 596 contents := randomContents(32) 597 598 // Create a regular file. 599 err := suite.StorageDriver.PutContent("/notadir", contents) 600 c.Assert(err, check.IsNil) 601 defer suite.StorageDriver.Delete("/notadir") 602 603 // Now try to move a non-existent file under it. 604 err = suite.StorageDriver.Move("/notadir/foo", "/notadir/bar") 605 c.Assert(err, check.NotNil) // non-nil error 606 } 607 608 // TestDelete checks that the delete operation removes data from the storage 609 // driver 610 func (suite *DriverSuite) TestDelete(c *check.C) { 611 filename := randomPath(32) 612 contents := randomContents(32) 613 614 defer suite.StorageDriver.Delete(firstPart(filename)) 615 616 err := suite.StorageDriver.PutContent(filename, contents) 617 c.Assert(err, check.IsNil) 618 619 err = suite.StorageDriver.Delete(filename) 620 c.Assert(err, check.IsNil) 621 622 _, err = suite.StorageDriver.GetContent(filename) 623 c.Assert(err, check.NotNil) 624 c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) 625 } 626 627 // TestURLFor checks that the URLFor method functions properly, but only if it 628 // is implemented 629 func (suite *DriverSuite) TestURLFor(c *check.C) { 630 filename := randomPath(32) 631 contents := randomContents(32) 632 633 defer suite.StorageDriver.Delete(firstPart(filename)) 634 635 err := suite.StorageDriver.PutContent(filename, contents) 636 c.Assert(err, check.IsNil) 637 638 url, err := suite.StorageDriver.URLFor(filename, nil) 639 if err == storagedriver.ErrUnsupportedMethod { 640 return 641 } 642 c.Assert(err, check.IsNil) 643 644 response, err := http.Get(url) 645 c.Assert(err, check.IsNil) 646 defer response.Body.Close() 647 648 read, err := ioutil.ReadAll(response.Body) 649 c.Assert(err, check.IsNil) 650 c.Assert(read, check.DeepEquals, contents) 651 652 url, err = suite.StorageDriver.URLFor(filename, map[string]interface{}{"method": "HEAD"}) 653 if err == storagedriver.ErrUnsupportedMethod { 654 return 655 } 656 c.Assert(err, check.IsNil) 657 658 response, err = http.Head(url) 659 c.Assert(response.StatusCode, check.Equals, 200) 660 c.Assert(response.ContentLength, check.Equals, int64(32)) 661 } 662 663 // TestDeleteNonexistent checks that removing a nonexistent key fails. 664 func (suite *DriverSuite) TestDeleteNonexistent(c *check.C) { 665 filename := randomPath(32) 666 err := suite.StorageDriver.Delete(filename) 667 c.Assert(err, check.NotNil) 668 c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) 669 } 670 671 // TestDeleteFolder checks that deleting a folder removes all child elements. 672 func (suite *DriverSuite) TestDeleteFolder(c *check.C) { 673 dirname := randomPath(32) 674 filename1 := randomPath(32) 675 filename2 := randomPath(32) 676 filename3 := randomPath(32) 677 contents := randomContents(32) 678 679 defer suite.StorageDriver.Delete(firstPart(dirname)) 680 681 err := suite.StorageDriver.PutContent(path.Join(dirname, filename1), contents) 682 c.Assert(err, check.IsNil) 683 684 err = suite.StorageDriver.PutContent(path.Join(dirname, filename2), contents) 685 c.Assert(err, check.IsNil) 686 687 err = suite.StorageDriver.PutContent(path.Join(dirname, filename3), contents) 688 c.Assert(err, check.IsNil) 689 690 err = suite.StorageDriver.Delete(path.Join(dirname, filename1)) 691 c.Assert(err, check.IsNil) 692 693 _, err = suite.StorageDriver.GetContent(path.Join(dirname, filename1)) 694 c.Assert(err, check.NotNil) 695 c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) 696 697 _, err = suite.StorageDriver.GetContent(path.Join(dirname, filename2)) 698 c.Assert(err, check.IsNil) 699 700 _, err = suite.StorageDriver.GetContent(path.Join(dirname, filename3)) 701 c.Assert(err, check.IsNil) 702 703 err = suite.StorageDriver.Delete(dirname) 704 c.Assert(err, check.IsNil) 705 706 _, err = suite.StorageDriver.GetContent(path.Join(dirname, filename1)) 707 c.Assert(err, check.NotNil) 708 c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) 709 710 _, err = suite.StorageDriver.GetContent(path.Join(dirname, filename2)) 711 c.Assert(err, check.NotNil) 712 c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) 713 714 _, err = suite.StorageDriver.GetContent(path.Join(dirname, filename3)) 715 c.Assert(err, check.NotNil) 716 c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) 717 } 718 719 // TestStatCall runs verifies the implementation of the storagedriver's Stat call. 720 func (suite *DriverSuite) TestStatCall(c *check.C) { 721 content := randomContents(4096) 722 dirPath := randomPath(32) 723 fileName := randomFilename(32) 724 filePath := path.Join(dirPath, fileName) 725 726 defer suite.StorageDriver.Delete(firstPart(dirPath)) 727 728 // Call on non-existent file/dir, check error. 729 fi, err := suite.StorageDriver.Stat(dirPath) 730 c.Assert(err, check.NotNil) 731 c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) 732 c.Assert(fi, check.IsNil) 733 734 fi, err = suite.StorageDriver.Stat(filePath) 735 c.Assert(err, check.NotNil) 736 c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) 737 c.Assert(fi, check.IsNil) 738 739 err = suite.StorageDriver.PutContent(filePath, content) 740 c.Assert(err, check.IsNil) 741 742 // Call on regular file, check results 743 fi, err = suite.StorageDriver.Stat(filePath) 744 c.Assert(err, check.IsNil) 745 c.Assert(fi, check.NotNil) 746 c.Assert(fi.Path(), check.Equals, filePath) 747 c.Assert(fi.Size(), check.Equals, int64(len(content))) 748 c.Assert(fi.IsDir(), check.Equals, false) 749 createdTime := fi.ModTime() 750 751 // Sleep and modify the file 752 time.Sleep(time.Second * 10) 753 content = randomContents(4096) 754 err = suite.StorageDriver.PutContent(filePath, content) 755 c.Assert(err, check.IsNil) 756 fi, err = suite.StorageDriver.Stat(filePath) 757 c.Assert(err, check.IsNil) 758 c.Assert(fi, check.NotNil) 759 time.Sleep(time.Second * 5) // allow changes to propagate (eventual consistency) 760 761 // Check if the modification time is after the creation time. 762 // In case of cloud storage services, storage frontend nodes might have 763 // time drift between them, however that should be solved with sleeping 764 // before update. 765 modTime := fi.ModTime() 766 if !modTime.After(createdTime) { 767 c.Errorf("modtime (%s) is before the creation time (%s)", modTime, createdTime) 768 } 769 770 // Call on directory (do not check ModTime as dirs don't need to support it) 771 fi, err = suite.StorageDriver.Stat(dirPath) 772 c.Assert(err, check.IsNil) 773 c.Assert(fi, check.NotNil) 774 c.Assert(fi.Path(), check.Equals, dirPath) 775 c.Assert(fi.Size(), check.Equals, int64(0)) 776 c.Assert(fi.IsDir(), check.Equals, true) 777 } 778 779 // TestPutContentMultipleTimes checks that if storage driver can overwrite the content 780 // in the subsequent puts. Validates that PutContent does not have to work 781 // with an offset like WriteStream does and overwrites the file entirely 782 // rather than writing the data to the [0,len(data)) of the file. 783 func (suite *DriverSuite) TestPutContentMultipleTimes(c *check.C) { 784 filename := randomPath(32) 785 contents := randomContents(4096) 786 787 defer suite.StorageDriver.Delete(firstPart(filename)) 788 err := suite.StorageDriver.PutContent(filename, contents) 789 c.Assert(err, check.IsNil) 790 791 contents = randomContents(2048) // upload a different, smaller file 792 err = suite.StorageDriver.PutContent(filename, contents) 793 c.Assert(err, check.IsNil) 794 795 readContents, err := suite.StorageDriver.GetContent(filename) 796 c.Assert(err, check.IsNil) 797 c.Assert(readContents, check.DeepEquals, contents) 798 } 799 800 // TestConcurrentStreamReads checks that multiple clients can safely read from 801 // the same file simultaneously with various offsets. 802 func (suite *DriverSuite) TestConcurrentStreamReads(c *check.C) { 803 var filesize int64 = 128 * 1024 * 1024 804 805 if testing.Short() { 806 filesize = 10 * 1024 * 1024 807 c.Log("Reducing file size to 10MB for short mode") 808 } 809 810 filename := randomPath(32) 811 contents := randomContents(filesize) 812 813 defer suite.StorageDriver.Delete(firstPart(filename)) 814 815 err := suite.StorageDriver.PutContent(filename, contents) 816 c.Assert(err, check.IsNil) 817 818 var wg sync.WaitGroup 819 820 readContents := func() { 821 defer wg.Done() 822 offset := rand.Int63n(int64(len(contents))) 823 reader, err := suite.StorageDriver.ReadStream(filename, offset) 824 c.Assert(err, check.IsNil) 825 826 readContents, err := ioutil.ReadAll(reader) 827 c.Assert(err, check.IsNil) 828 c.Assert(readContents, check.DeepEquals, contents[offset:]) 829 } 830 831 wg.Add(10) 832 for i := 0; i < 10; i++ { 833 go readContents() 834 } 835 wg.Wait() 836 } 837 838 // TestConcurrentFileStreams checks that multiple *os.File objects can be passed 839 // in to WriteStream concurrently without hanging. 840 func (suite *DriverSuite) TestConcurrentFileStreams(c *check.C) { 841 // if _, isIPC := suite.StorageDriver.(*ipc.StorageDriverClient); isIPC { 842 // c.Skip("Need to fix out-of-process concurrency") 843 // } 844 845 numStreams := 32 846 847 if testing.Short() { 848 numStreams = 8 849 c.Log("Reducing number of streams to 8 for short mode") 850 } 851 852 var wg sync.WaitGroup 853 854 testStream := func(size int64) { 855 defer wg.Done() 856 suite.testFileStreams(c, size) 857 } 858 859 wg.Add(numStreams) 860 for i := numStreams; i > 0; i-- { 861 go testStream(int64(numStreams) * 1024 * 1024) 862 } 863 864 wg.Wait() 865 } 866 867 // TestEventualConsistency checks that if stat says that a file is a certain size, then 868 // you can freely read from the file (this is the only guarantee that the driver needs to provide) 869 func (suite *DriverSuite) TestEventualConsistency(c *check.C) { 870 if testing.Short() { 871 c.Skip("Skipping test in short mode") 872 } 873 874 filename := randomPath(32) 875 defer suite.StorageDriver.Delete(firstPart(filename)) 876 877 var offset int64 878 var misswrites int 879 var chunkSize int64 = 32 880 881 for i := 0; i < 1024; i++ { 882 contents := randomContents(chunkSize) 883 read, err := suite.StorageDriver.WriteStream(filename, offset, bytes.NewReader(contents)) 884 c.Assert(err, check.IsNil) 885 886 fi, err := suite.StorageDriver.Stat(filename) 887 c.Assert(err, check.IsNil) 888 889 // We are most concerned with being able to read data as soon as Stat declares 890 // it is uploaded. This is the strongest guarantee that some drivers (that guarantee 891 // at best eventual consistency) absolutely need to provide. 892 if fi.Size() == offset+chunkSize { 893 reader, err := suite.StorageDriver.ReadStream(filename, offset) 894 c.Assert(err, check.IsNil) 895 896 readContents, err := ioutil.ReadAll(reader) 897 c.Assert(err, check.IsNil) 898 899 c.Assert(readContents, check.DeepEquals, contents) 900 901 reader.Close() 902 offset += read 903 } else { 904 misswrites++ 905 } 906 } 907 908 if misswrites > 0 { 909 c.Log("There were " + string(misswrites) + " occurences of a write not being instantly available.") 910 } 911 912 c.Assert(misswrites, check.Not(check.Equals), 1024) 913 } 914 915 // BenchmarkPutGetEmptyFiles benchmarks PutContent/GetContent for 0B files 916 func (suite *DriverSuite) BenchmarkPutGetEmptyFiles(c *check.C) { 917 suite.benchmarkPutGetFiles(c, 0) 918 } 919 920 // BenchmarkPutGet1KBFiles benchmarks PutContent/GetContent for 1KB files 921 func (suite *DriverSuite) BenchmarkPutGet1KBFiles(c *check.C) { 922 suite.benchmarkPutGetFiles(c, 1024) 923 } 924 925 // BenchmarkPutGet1MBFiles benchmarks PutContent/GetContent for 1MB files 926 func (suite *DriverSuite) BenchmarkPutGet1MBFiles(c *check.C) { 927 suite.benchmarkPutGetFiles(c, 1024*1024) 928 } 929 930 // BenchmarkPutGet1GBFiles benchmarks PutContent/GetContent for 1GB files 931 func (suite *DriverSuite) BenchmarkPutGet1GBFiles(c *check.C) { 932 suite.benchmarkPutGetFiles(c, 1024*1024*1024) 933 } 934 935 func (suite *DriverSuite) benchmarkPutGetFiles(c *check.C, size int64) { 936 c.SetBytes(size) 937 parentDir := randomPath(8) 938 defer func() { 939 c.StopTimer() 940 suite.StorageDriver.Delete(firstPart(parentDir)) 941 }() 942 943 for i := 0; i < c.N; i++ { 944 filename := path.Join(parentDir, randomPath(32)) 945 err := suite.StorageDriver.PutContent(filename, randomContents(size)) 946 c.Assert(err, check.IsNil) 947 948 _, err = suite.StorageDriver.GetContent(filename) 949 c.Assert(err, check.IsNil) 950 } 951 } 952 953 // BenchmarkStreamEmptyFiles benchmarks WriteStream/ReadStream for 0B files 954 func (suite *DriverSuite) BenchmarkStreamEmptyFiles(c *check.C) { 955 suite.benchmarkStreamFiles(c, 0) 956 } 957 958 // BenchmarkStream1KBFiles benchmarks WriteStream/ReadStream for 1KB files 959 func (suite *DriverSuite) BenchmarkStream1KBFiles(c *check.C) { 960 suite.benchmarkStreamFiles(c, 1024) 961 } 962 963 // BenchmarkStream1MBFiles benchmarks WriteStream/ReadStream for 1MB files 964 func (suite *DriverSuite) BenchmarkStream1MBFiles(c *check.C) { 965 suite.benchmarkStreamFiles(c, 1024*1024) 966 } 967 968 // BenchmarkStream1GBFiles benchmarks WriteStream/ReadStream for 1GB files 969 func (suite *DriverSuite) BenchmarkStream1GBFiles(c *check.C) { 970 suite.benchmarkStreamFiles(c, 1024*1024*1024) 971 } 972 973 func (suite *DriverSuite) benchmarkStreamFiles(c *check.C, size int64) { 974 c.SetBytes(size) 975 parentDir := randomPath(8) 976 defer func() { 977 c.StopTimer() 978 suite.StorageDriver.Delete(firstPart(parentDir)) 979 }() 980 981 for i := 0; i < c.N; i++ { 982 filename := path.Join(parentDir, randomPath(32)) 983 written, err := suite.StorageDriver.WriteStream(filename, 0, bytes.NewReader(randomContents(size))) 984 c.Assert(err, check.IsNil) 985 c.Assert(written, check.Equals, size) 986 987 rc, err := suite.StorageDriver.ReadStream(filename, 0) 988 c.Assert(err, check.IsNil) 989 rc.Close() 990 } 991 } 992 993 // BenchmarkList5Files benchmarks List for 5 small files 994 func (suite *DriverSuite) BenchmarkList5Files(c *check.C) { 995 suite.benchmarkListFiles(c, 5) 996 } 997 998 // BenchmarkList50Files benchmarks List for 50 small files 999 func (suite *DriverSuite) BenchmarkList50Files(c *check.C) { 1000 suite.benchmarkListFiles(c, 50) 1001 } 1002 1003 func (suite *DriverSuite) benchmarkListFiles(c *check.C, numFiles int64) { 1004 parentDir := randomPath(8) 1005 defer func() { 1006 c.StopTimer() 1007 suite.StorageDriver.Delete(firstPart(parentDir)) 1008 }() 1009 1010 for i := int64(0); i < numFiles; i++ { 1011 err := suite.StorageDriver.PutContent(path.Join(parentDir, randomPath(32)), nil) 1012 c.Assert(err, check.IsNil) 1013 } 1014 1015 c.ResetTimer() 1016 for i := 0; i < c.N; i++ { 1017 files, err := suite.StorageDriver.List(parentDir) 1018 c.Assert(err, check.IsNil) 1019 c.Assert(int64(len(files)), check.Equals, numFiles) 1020 } 1021 } 1022 1023 // BenchmarkDelete5Files benchmarks Delete for 5 small files 1024 func (suite *DriverSuite) BenchmarkDelete5Files(c *check.C) { 1025 suite.benchmarkDeleteFiles(c, 5) 1026 } 1027 1028 // BenchmarkDelete50Files benchmarks Delete for 50 small files 1029 func (suite *DriverSuite) BenchmarkDelete50Files(c *check.C) { 1030 suite.benchmarkDeleteFiles(c, 50) 1031 } 1032 1033 func (suite *DriverSuite) benchmarkDeleteFiles(c *check.C, numFiles int64) { 1034 for i := 0; i < c.N; i++ { 1035 parentDir := randomPath(8) 1036 defer suite.StorageDriver.Delete(firstPart(parentDir)) 1037 1038 c.StopTimer() 1039 for j := int64(0); j < numFiles; j++ { 1040 err := suite.StorageDriver.PutContent(path.Join(parentDir, randomPath(32)), nil) 1041 c.Assert(err, check.IsNil) 1042 } 1043 c.StartTimer() 1044 1045 // This is the operation we're benchmarking 1046 err := suite.StorageDriver.Delete(firstPart(parentDir)) 1047 c.Assert(err, check.IsNil) 1048 } 1049 } 1050 1051 func (suite *DriverSuite) testFileStreams(c *check.C, size int64) { 1052 tf, err := ioutil.TempFile("", "tf") 1053 c.Assert(err, check.IsNil) 1054 defer os.Remove(tf.Name()) 1055 defer tf.Close() 1056 1057 filename := randomPath(32) 1058 defer suite.StorageDriver.Delete(firstPart(filename)) 1059 1060 contents := randomContents(size) 1061 1062 _, err = tf.Write(contents) 1063 c.Assert(err, check.IsNil) 1064 1065 tf.Sync() 1066 tf.Seek(0, os.SEEK_SET) 1067 1068 nn, err := suite.StorageDriver.WriteStream(filename, 0, tf) 1069 c.Assert(err, check.IsNil) 1070 c.Assert(nn, check.Equals, size) 1071 1072 reader, err := suite.StorageDriver.ReadStream(filename, 0) 1073 c.Assert(err, check.IsNil) 1074 defer reader.Close() 1075 1076 readContents, err := ioutil.ReadAll(reader) 1077 c.Assert(err, check.IsNil) 1078 1079 c.Assert(readContents, check.DeepEquals, contents) 1080 } 1081 1082 func (suite *DriverSuite) writeReadCompare(c *check.C, filename string, contents []byte) { 1083 defer suite.StorageDriver.Delete(firstPart(filename)) 1084 1085 err := suite.StorageDriver.PutContent(filename, contents) 1086 c.Assert(err, check.IsNil) 1087 1088 readContents, err := suite.StorageDriver.GetContent(filename) 1089 c.Assert(err, check.IsNil) 1090 1091 c.Assert(readContents, check.DeepEquals, contents) 1092 } 1093 1094 func (suite *DriverSuite) writeReadCompareStreams(c *check.C, filename string, contents []byte) { 1095 defer suite.StorageDriver.Delete(firstPart(filename)) 1096 1097 nn, err := suite.StorageDriver.WriteStream(filename, 0, bytes.NewReader(contents)) 1098 c.Assert(err, check.IsNil) 1099 c.Assert(nn, check.Equals, int64(len(contents))) 1100 1101 reader, err := suite.StorageDriver.ReadStream(filename, 0) 1102 c.Assert(err, check.IsNil) 1103 defer reader.Close() 1104 1105 readContents, err := ioutil.ReadAll(reader) 1106 c.Assert(err, check.IsNil) 1107 1108 c.Assert(readContents, check.DeepEquals, contents) 1109 } 1110 1111 var filenameChars = []byte("abcdefghijklmnopqrstuvwxyz0123456789") 1112 var separatorChars = []byte("._-") 1113 1114 func randomPath(length int64) string { 1115 path := "/" 1116 for int64(len(path)) < length { 1117 chunkLength := rand.Int63n(length-int64(len(path))) + 1 1118 chunk := randomFilename(chunkLength) 1119 path += chunk 1120 remaining := length - int64(len(path)) 1121 if remaining == 1 { 1122 path += randomFilename(1) 1123 } else if remaining > 1 { 1124 path += "/" 1125 } 1126 } 1127 return path 1128 } 1129 1130 func randomFilename(length int64) string { 1131 b := make([]byte, length) 1132 wasSeparator := true 1133 for i := range b { 1134 if !wasSeparator && i < len(b)-1 && rand.Intn(4) == 0 { 1135 b[i] = separatorChars[rand.Intn(len(separatorChars))] 1136 wasSeparator = true 1137 } else { 1138 b[i] = filenameChars[rand.Intn(len(filenameChars))] 1139 wasSeparator = false 1140 } 1141 } 1142 return string(b) 1143 } 1144 1145 func randomContents(length int64) []byte { 1146 b := make([]byte, length) 1147 for i := range b { 1148 b[i] = byte(rand.Intn(2 << 8)) 1149 } 1150 return b 1151 } 1152 1153 type randReader struct { 1154 r int64 1155 m sync.Mutex 1156 } 1157 1158 func (rr *randReader) Read(p []byte) (n int, err error) { 1159 rr.m.Lock() 1160 defer rr.m.Unlock() 1161 for i := 0; i < len(p) && rr.r > 0; i++ { 1162 p[i] = byte(rand.Intn(255)) 1163 n++ 1164 rr.r-- 1165 } 1166 if rr.r == 0 { 1167 err = io.EOF 1168 } 1169 return 1170 } 1171 1172 func newRandReader(n int64) *randReader { 1173 return &randReader{r: n} 1174 } 1175 1176 func firstPart(filePath string) string { 1177 if filePath == "" { 1178 return "/" 1179 } 1180 for { 1181 if filePath[len(filePath)-1] == '/' { 1182 filePath = filePath[:len(filePath)-1] 1183 } 1184 1185 dir, file := path.Split(filePath) 1186 if dir == "" && file == "" { 1187 return "/" 1188 } 1189 if dir == "/" || dir == "" { 1190 return "/" + file 1191 } 1192 if file == "" { 1193 return dir 1194 } 1195 filePath = dir 1196 } 1197 }