github.com/minio/minio@v0.0.0-20240328213742-3f72439b8a27/cmd/object-api-listobjects_test.go (about) 1 // Copyright (c) 2015-2021 MinIO, Inc. 2 // 3 // This file is part of MinIO Object Storage stack 4 // 5 // This program is free software: you can redistribute it and/or modify 6 // it under the terms of the GNU Affero General Public License as published by 7 // the Free Software Foundation, either version 3 of the License, or 8 // (at your option) any later version. 9 // 10 // This program is distributed in the hope that it will be useful 11 // but WITHOUT ANY WARRANTY; without even the implied warranty of 12 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 // GNU Affero General Public License for more details. 14 // 15 // You should have received a copy of the GNU Affero General Public License 16 // along with this program. If not, see <http://www.gnu.org/licenses/>. 17 18 package cmd 19 20 import ( 21 "bytes" 22 "context" 23 "crypto/md5" 24 "encoding/hex" 25 "fmt" 26 "strconv" 27 "strings" 28 "testing" 29 ) 30 31 func TestListObjectsVersionedFolders(t *testing.T) { 32 ExecObjectLayerTest(t, testListObjectsVersionedFolders) 33 } 34 35 func testListObjectsVersionedFolders(obj ObjectLayer, instanceType string, t1 TestErrHandler) { 36 t, _ := t1.(*testing.T) 37 testBuckets := []string{ 38 // This bucket is used for testing ListObject operations. 39 "test-bucket-folders", 40 // This bucket has file delete marker. 41 "test-bucket-files", 42 } 43 for _, bucket := range testBuckets { 44 err := obj.MakeBucket(context.Background(), bucket, MakeBucketOptions{ 45 VersioningEnabled: true, 46 }) 47 if err != nil { 48 t.Fatalf("%s : %s", instanceType, err.Error()) 49 } 50 } 51 52 var err error 53 testObjects := []struct { 54 parentBucket string 55 name string 56 content string 57 meta map[string]string 58 addDeleteMarker bool 59 }{ 60 {testBuckets[0], "unique/folder/", "", nil, true}, 61 {testBuckets[0], "unique/folder/1.txt", "content", nil, false}, 62 {testBuckets[1], "unique/folder/1.txt", "content", nil, true}, 63 } 64 for _, object := range testObjects { 65 md5Bytes := md5.Sum([]byte(object.content)) 66 _, err = obj.PutObject(context.Background(), object.parentBucket, object.name, mustGetPutObjReader(t, bytes.NewBufferString(object.content), 67 int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), ObjectOptions{ 68 Versioned: globalBucketVersioningSys.PrefixEnabled(object.parentBucket, object.name), 69 UserDefined: object.meta, 70 }) 71 if err != nil { 72 t.Fatalf("%s : %s", instanceType, err.Error()) 73 } 74 if object.addDeleteMarker { 75 oi, err := obj.DeleteObject(context.Background(), object.parentBucket, object.name, ObjectOptions{ 76 Versioned: globalBucketVersioningSys.PrefixEnabled(object.parentBucket, object.name), 77 }) 78 if err != nil { 79 t.Fatalf("%s : %s", instanceType, err.Error()) 80 } 81 if oi.DeleteMarker != object.addDeleteMarker { 82 t.Fatalf("Expected, marker %t : got %t", object.addDeleteMarker, oi.DeleteMarker) 83 } 84 } 85 } 86 87 // Formulating the result data set to be expected from ListObjects call inside the tests, 88 // This will be used in testCases and used for asserting the correctness of ListObjects output in the tests. 89 90 resultCases := []ListObjectsInfo{ 91 { 92 IsTruncated: false, 93 Prefixes: []string{"unique/folder/"}, 94 }, 95 { 96 IsTruncated: false, 97 Objects: []ObjectInfo{ 98 {Name: "unique/folder/1.txt"}, 99 }, 100 }, 101 { 102 IsTruncated: false, 103 Objects: []ObjectInfo{}, 104 }, 105 } 106 107 resultCasesV := []ListObjectVersionsInfo{ 108 { 109 IsTruncated: false, 110 Prefixes: []string{"unique/folder/"}, 111 }, 112 { 113 IsTruncated: false, 114 Objects: []ObjectInfo{ 115 { 116 Name: "unique/folder/", 117 DeleteMarker: true, 118 }, 119 { 120 Name: "unique/folder/", 121 DeleteMarker: false, 122 }, 123 { 124 Name: "unique/folder/1.txt", 125 DeleteMarker: false, 126 }, 127 }, 128 }, 129 } 130 131 testCases := []struct { 132 // Inputs to ListObjects. 133 bucketName string 134 prefix string 135 marker string 136 delimiter string 137 maxKeys int 138 versioned bool 139 // Expected output of ListObjects. 140 resultL ListObjectsInfo 141 resultV ListObjectVersionsInfo 142 err error 143 // Flag indicating whether the test is expected to pass or not. 144 shouldPass bool 145 }{ 146 {testBuckets[0], "unique/", "", "/", 1000, false, resultCases[0], ListObjectVersionsInfo{}, nil, true}, 147 {testBuckets[0], "unique/folder", "", "/", 1000, false, resultCases[0], ListObjectVersionsInfo{}, nil, true}, 148 {testBuckets[0], "unique/", "", "", 1000, false, resultCases[1], ListObjectVersionsInfo{}, nil, true}, 149 {testBuckets[1], "unique/", "", "/", 1000, false, resultCases[0], ListObjectVersionsInfo{}, nil, true}, 150 {testBuckets[1], "unique/folder/", "", "/", 1000, false, resultCases[2], ListObjectVersionsInfo{}, nil, true}, 151 {testBuckets[0], "unique/", "", "/", 1000, true, ListObjectsInfo{}, resultCasesV[0], nil, true}, 152 {testBuckets[0], "unique/", "", "", 1000, true, ListObjectsInfo{}, resultCasesV[1], nil, true}, 153 } 154 155 for i, testCase := range testCases { 156 testCase := testCase 157 t.Run(fmt.Sprintf("%s-Test%d", instanceType, i+1), func(t *testing.T) { 158 var err error 159 var resultL ListObjectsInfo 160 var resultV ListObjectVersionsInfo 161 if testCase.versioned { 162 t.Log("ListObjectVersions, bucket:", testCase.bucketName, "prefix:", 163 testCase.prefix, "marker:", testCase.marker, "delimiter:", 164 testCase.delimiter, "maxkeys:", testCase.maxKeys) 165 166 resultV, err = obj.ListObjectVersions(context.Background(), testCase.bucketName, 167 testCase.prefix, testCase.marker, "", testCase.delimiter, testCase.maxKeys) 168 } else { 169 t.Log("ListObjects, bucket:", testCase.bucketName, "prefix:", 170 testCase.prefix, "marker:", testCase.marker, "delimiter:", 171 testCase.delimiter, "maxkeys:", testCase.maxKeys) 172 173 resultL, err = obj.ListObjects(context.Background(), testCase.bucketName, 174 testCase.prefix, testCase.marker, testCase.delimiter, testCase.maxKeys) 175 } 176 if err != nil && testCase.shouldPass { 177 t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, err.Error()) 178 } 179 if err == nil && !testCase.shouldPass { 180 t.Errorf("Test %d: %s: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, instanceType, testCase.err.Error()) 181 } 182 // Failed as expected, but does it fail for the expected reason. 183 if err != nil && !testCase.shouldPass { 184 if !strings.Contains(err.Error(), testCase.err.Error()) { 185 t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, instanceType, testCase.err.Error(), err.Error()) 186 } 187 } 188 // Since there are cases for which ListObjects fails, this is 189 // necessary. Test passes as expected, but the output values 190 // are verified for correctness here. 191 if err == nil && testCase.shouldPass { 192 // The length of the expected ListObjectsResult.Objects 193 // should match in both expected result from test cases 194 // and in the output. On failure calling t.Fatalf, 195 // otherwise it may lead to index out of range error in 196 // assertion following this. 197 if !testCase.versioned { 198 if len(testCase.resultL.Objects) != len(resultL.Objects) { 199 t.Logf("want: %v", objInfoNames(testCase.resultL.Objects)) 200 t.Logf("got: %v", objInfoNames(resultL.Objects)) 201 t.Errorf("Test %d: %s: Expected number of object in the result to be '%d', but found '%d' objects instead", i+1, instanceType, len(testCase.resultL.Objects), len(resultL.Objects)) 202 } 203 for j := 0; j < len(testCase.resultL.Objects); j++ { 204 if j >= len(resultL.Objects) { 205 t.Errorf("Test %d: %s: Expected object name to be \"%s\", but not nothing instead", i+1, instanceType, testCase.resultL.Objects[j].Name) 206 continue 207 } 208 if testCase.resultL.Objects[j].Name != resultL.Objects[j].Name { 209 t.Errorf("Test %d: %s: Expected object name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.resultL.Objects[j].Name, resultL.Objects[j].Name) 210 } 211 } 212 213 if len(testCase.resultL.Prefixes) != len(resultL.Prefixes) { 214 t.Logf("want: %v", testCase.resultL.Prefixes) 215 t.Logf("got: %v", resultL.Prefixes) 216 t.Errorf("Test %d: %s: Expected number of prefixes in the result to be '%d', but found '%d' prefixes instead", i+1, instanceType, len(testCase.resultL.Prefixes), len(resultL.Prefixes)) 217 } 218 for j := 0; j < len(testCase.resultL.Prefixes); j++ { 219 if j >= len(resultL.Prefixes) { 220 t.Errorf("Test %d: %s: Expected prefix name to be \"%s\", but found no result", i+1, instanceType, testCase.resultL.Prefixes[j]) 221 continue 222 } 223 if testCase.resultL.Prefixes[j] != resultL.Prefixes[j] { 224 t.Errorf("Test %d: %s: Expected prefix name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.resultL.Prefixes[j], resultL.Prefixes[j]) 225 } 226 } 227 228 if testCase.resultL.IsTruncated != resultL.IsTruncated { 229 // Allow an extra continuation token. 230 if !resultL.IsTruncated || len(resultL.Objects) == 0 { 231 t.Errorf("Test %d: %s: Expected IsTruncated flag to be %v, but instead found it to be %v", i+1, instanceType, testCase.resultL.IsTruncated, resultL.IsTruncated) 232 } 233 } 234 235 if testCase.resultL.IsTruncated && resultL.NextMarker == "" { 236 t.Errorf("Test %d: %s: Expected NextMarker to contain a string since listing is truncated, but instead found it to be empty", i+1, instanceType) 237 } 238 239 if !testCase.resultL.IsTruncated && resultL.NextMarker != "" { 240 if !resultL.IsTruncated || len(resultL.Objects) == 0 { 241 t.Errorf("Test %d: %s: Expected NextMarker to be empty since listing is not truncated, but instead found `%v`", i+1, instanceType, resultL.NextMarker) 242 } 243 } 244 } else { 245 if len(testCase.resultV.Objects) != len(resultV.Objects) { 246 t.Logf("want: %v", objInfoNames(testCase.resultV.Objects)) 247 t.Logf("got: %v", objInfoNames(resultV.Objects)) 248 t.Errorf("Test %d: %s: Expected number of object in the result to be '%d', but found '%d' objects instead", i+1, instanceType, len(testCase.resultV.Objects), len(resultV.Objects)) 249 } 250 for j := 0; j < len(testCase.resultV.Objects); j++ { 251 if j >= len(resultV.Objects) { 252 t.Errorf("Test %d: %s: Expected object name to be \"%s\", but not nothing instead", i+1, instanceType, testCase.resultV.Objects[j].Name) 253 continue 254 } 255 if testCase.resultV.Objects[j].Name != resultV.Objects[j].Name { 256 t.Errorf("Test %d: %s: Expected object name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.resultV.Objects[j].Name, resultV.Objects[j].Name) 257 } 258 } 259 260 if len(testCase.resultV.Prefixes) != len(resultV.Prefixes) { 261 t.Logf("want: %v", testCase.resultV.Prefixes) 262 t.Logf("got: %v", resultV.Prefixes) 263 t.Errorf("Test %d: %s: Expected number of prefixes in the result to be '%d', but found '%d' prefixes instead", i+1, instanceType, len(testCase.resultV.Prefixes), len(resultV.Prefixes)) 264 } 265 for j := 0; j < len(testCase.resultV.Prefixes); j++ { 266 if j >= len(resultV.Prefixes) { 267 t.Errorf("Test %d: %s: Expected prefix name to be \"%s\", but found no result", i+1, instanceType, testCase.resultV.Prefixes[j]) 268 continue 269 } 270 if testCase.resultV.Prefixes[j] != resultV.Prefixes[j] { 271 t.Errorf("Test %d: %s: Expected prefix name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.resultV.Prefixes[j], resultV.Prefixes[j]) 272 } 273 } 274 275 if testCase.resultV.IsTruncated != resultV.IsTruncated { 276 // Allow an extra continuation token. 277 if !resultV.IsTruncated || len(resultV.Objects) == 0 { 278 t.Errorf("Test %d: %s: Expected IsTruncated flag to be %v, but instead found it to be %v", i+1, instanceType, testCase.resultV.IsTruncated, resultV.IsTruncated) 279 } 280 } 281 282 if testCase.resultV.IsTruncated && resultV.NextMarker == "" { 283 t.Errorf("Test %d: %s: Expected NextMarker to contain a string since listing is truncated, but instead found it to be empty", i+1, instanceType) 284 } 285 286 if !testCase.resultV.IsTruncated && resultV.NextMarker != "" { 287 if !resultV.IsTruncated || len(resultV.Objects) == 0 { 288 t.Errorf("Test %d: %s: Expected NextMarker to be empty since listing is not truncated, but instead found `%v`", i+1, instanceType, resultV.NextMarker) 289 } 290 } 291 } 292 } 293 }) 294 } 295 } 296 297 // Wrapper for calling ListObjectsOnVersionedBuckets tests for both 298 // Erasure multiple disks and single node setup. 299 func TestListObjectsOnVersionedBuckets(t *testing.T) { 300 ExecObjectLayerTest(t, testListObjectsOnVersionedBuckets) 301 } 302 303 // Wrapper for calling ListObjects tests for both Erasure multiple 304 // disks and single node setup. 305 func TestListObjects(t *testing.T) { 306 ExecObjectLayerTest(t, testListObjects) 307 } 308 309 // Unit test for ListObjects on VersionedBucket. 310 func testListObjectsOnVersionedBuckets(obj ObjectLayer, instanceType string, t1 TestErrHandler) { 311 _testListObjects(obj, instanceType, t1, true) 312 } 313 314 // Unit test for ListObjects. 315 func testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler) { 316 _testListObjects(obj, instanceType, t1, false) 317 } 318 319 func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, versioned bool) { 320 t, _ := t1.(*testing.T) 321 testBuckets := []string{ 322 // This bucket is used for testing ListObject operations. 323 0: "test-bucket-list-object", 324 // This bucket will be tested with empty directories 325 1: "test-bucket-empty-dir", 326 // Will not store any objects in this bucket, 327 // Its to test ListObjects on an empty bucket. 328 2: "empty-bucket", 329 // Listing the case where the marker > last object. 330 3: "test-bucket-single-object", 331 // Listing uncommon delimiter. 332 4: "test-bucket-delimiter", 333 // Listing prefixes > maxKeys 334 5: "test-bucket-max-keys-prefixes", 335 // Listing custom delimiters 336 6: "test-bucket-custom-delimiter", 337 } 338 for _, bucket := range testBuckets { 339 err := obj.MakeBucket(context.Background(), bucket, MakeBucketOptions{ 340 VersioningEnabled: versioned, 341 }) 342 if err != nil { 343 t.Fatalf("%s : %s", instanceType, err.Error()) 344 } 345 } 346 347 var err error 348 testObjects := []struct { 349 parentBucket string 350 name string 351 content string 352 meta map[string]string 353 }{ 354 {testBuckets[0], "Asia-maps.png", "asis-maps", map[string]string{"content-type": "image/png"}}, 355 {testBuckets[0], "Asia/India/India-summer-photos-1", "contentstring", nil}, 356 {testBuckets[0], "Asia/India/Karnataka/Bangalore/Koramangala/pics", "contentstring", nil}, 357 {testBuckets[0], "newPrefix0", "newPrefix0", nil}, 358 {testBuckets[0], "newPrefix1", "newPrefix1", nil}, 359 {testBuckets[0], "newzen/zen/recurse/again/again/again/pics", "recurse", nil}, 360 {testBuckets[0], "obj0", "obj0", nil}, 361 {testBuckets[0], "obj1", "obj1", nil}, 362 {testBuckets[0], "obj2", "obj2", nil}, 363 {testBuckets[1], "obj1", "obj1", nil}, 364 {testBuckets[1], "obj2", "obj2", nil}, 365 {testBuckets[1], "temporary/0/", "", nil}, 366 {testBuckets[3], "A/B", "contentstring", nil}, 367 {testBuckets[4], "file1/receipt.json", "content", nil}, 368 {testBuckets[4], "file1/guidSplunk-aaaa/file", "content", nil}, 369 {testBuckets[5], "dir/day_id=2017-10-10/issue", "content", nil}, 370 {testBuckets[5], "dir/day_id=2017-10-11/issue", "content", nil}, 371 {testBuckets[5], "foo/201910/1122", "content", nil}, 372 {testBuckets[5], "foo/201910/1112", "content", nil}, 373 {testBuckets[5], "foo/201910/2112", "content", nil}, 374 {testBuckets[5], "foo/201910_txt", "content", nil}, 375 {testBuckets[5], "201910/foo/bar/xl.meta/1.txt", "content", nil}, 376 {testBuckets[6], "aaa", "content", nil}, 377 {testBuckets[6], "bbb_aaa", "content", nil}, 378 {testBuckets[6], "bbb_aaa", "content", nil}, 379 {testBuckets[6], "ccc", "content", nil}, 380 } 381 for _, object := range testObjects { 382 md5Bytes := md5.Sum([]byte(object.content)) 383 _, err = obj.PutObject(context.Background(), object.parentBucket, object.name, 384 mustGetPutObjReader(t, bytes.NewBufferString(object.content), 385 int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), ObjectOptions{ 386 Versioned: globalBucketVersioningSys.PrefixEnabled(object.parentBucket, object.name), 387 UserDefined: object.meta, 388 }) 389 if err != nil { 390 t.Fatalf("%s : %s", instanceType, err.Error()) 391 } 392 393 } 394 395 // Formulating the result data set to be expected from ListObjects call inside the tests, 396 // This will be used in testCases and used for asserting the correctness of ListObjects output in the tests. 397 398 resultCases := []ListObjectsInfo{ 399 // ListObjectsResult-0. 400 // Testing for listing all objects in the bucket, (testCase 20,21,22). 401 0: { 402 IsTruncated: false, 403 Objects: []ObjectInfo{ 404 {Name: "Asia-maps.png"}, 405 {Name: "Asia/India/India-summer-photos-1"}, 406 {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, 407 {Name: "newPrefix0"}, 408 {Name: "newPrefix1"}, 409 {Name: "newzen/zen/recurse/again/again/again/pics"}, 410 {Name: "obj0"}, 411 {Name: "obj1"}, 412 {Name: "obj2"}, 413 }, 414 }, 415 // ListObjectsResult-1. 416 // Used for asserting the truncated case, (testCase 23). 417 1: { 418 IsTruncated: true, 419 Objects: []ObjectInfo{ 420 {Name: "Asia-maps.png"}, 421 {Name: "Asia/India/India-summer-photos-1"}, 422 {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, 423 {Name: "newPrefix0"}, 424 {Name: "newPrefix1"}, 425 }, 426 }, 427 // ListObjectsResult-2. 428 // (TestCase 24). 429 2: { 430 IsTruncated: true, 431 Objects: []ObjectInfo{ 432 {Name: "Asia-maps.png"}, 433 {Name: "Asia/India/India-summer-photos-1"}, 434 {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, 435 {Name: "newPrefix0"}, 436 }, 437 }, 438 // ListObjectsResult-3. 439 // (TestCase 25). 440 3: { 441 IsTruncated: true, 442 Objects: []ObjectInfo{ 443 {Name: "Asia-maps.png"}, 444 {Name: "Asia/India/India-summer-photos-1"}, 445 {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, 446 }, 447 }, 448 // ListObjectsResult-4. 449 // Again used for truncated case. 450 // (TestCase 26). 451 4: { 452 IsTruncated: true, 453 Objects: []ObjectInfo{ 454 {Name: "Asia-maps.png"}, 455 }, 456 }, 457 // ListObjectsResult-5. 458 // Used for Asserting prefixes. 459 // Used for test case with prefix "new", (testCase 27-29). 460 5: { 461 IsTruncated: false, 462 Objects: []ObjectInfo{ 463 {Name: "newPrefix0"}, 464 {Name: "newPrefix1"}, 465 {Name: "newzen/zen/recurse/again/again/again/pics"}, 466 }, 467 }, 468 // ListObjectsResult-6. 469 // Used for Asserting prefixes. 470 // Used for test case with prefix = "obj", (testCase 30). 471 6: { 472 IsTruncated: false, 473 Objects: []ObjectInfo{ 474 {Name: "obj0"}, 475 {Name: "obj1"}, 476 {Name: "obj2"}, 477 }, 478 }, 479 // ListObjectsResult-7. 480 // Used for Asserting prefixes and truncation. 481 // Used for test case with prefix = "new" and maxKeys = 1, (testCase 31). 482 7: { 483 IsTruncated: true, 484 Objects: []ObjectInfo{ 485 {Name: "newPrefix0"}, 486 }, 487 }, 488 // ListObjectsResult-8. 489 // Used for Asserting prefixes. 490 // Used for test case with prefix = "obj" and maxKeys = 2, (testCase 32). 491 8: { 492 IsTruncated: true, 493 Objects: []ObjectInfo{ 494 {Name: "obj0"}, 495 {Name: "obj1"}, 496 }, 497 }, 498 // ListObjectsResult-9. 499 // Used for asserting the case with marker, but without prefix. 500 // marker is set to "newPrefix0" in the testCase, (testCase 33). 501 9: { 502 IsTruncated: false, 503 Objects: []ObjectInfo{ 504 {Name: "newPrefix1"}, 505 {Name: "newzen/zen/recurse/again/again/again/pics"}, 506 {Name: "obj0"}, 507 {Name: "obj1"}, 508 {Name: "obj2"}, 509 }, 510 }, 511 // ListObjectsResult-10. 512 // marker is set to "newPrefix1" in the testCase, (testCase 34). 513 10: { 514 IsTruncated: false, 515 Objects: []ObjectInfo{ 516 {Name: "newzen/zen/recurse/again/again/again/pics"}, 517 {Name: "obj0"}, 518 {Name: "obj1"}, 519 {Name: "obj2"}, 520 }, 521 }, 522 // ListObjectsResult-11. 523 // marker is set to "obj0" in the testCase, (testCase 35). 524 11: { 525 IsTruncated: false, 526 Objects: []ObjectInfo{ 527 {Name: "obj1"}, 528 {Name: "obj2"}, 529 }, 530 }, 531 // ListObjectsResult-12. 532 // Marker is set to "obj1" in the testCase, (testCase 36). 533 12: { 534 IsTruncated: false, 535 Objects: []ObjectInfo{ 536 {Name: "obj2"}, 537 }, 538 }, 539 // ListObjectsResult-13. 540 // Marker is set to "man" in the testCase, (testCase37). 541 13: { 542 IsTruncated: false, 543 Objects: []ObjectInfo{ 544 {Name: "newPrefix0"}, 545 {Name: "newPrefix1"}, 546 {Name: "newzen/zen/recurse/again/again/again/pics"}, 547 {Name: "obj0"}, 548 {Name: "obj1"}, 549 {Name: "obj2"}, 550 }, 551 }, 552 // ListObjectsResult-14. 553 // Marker is set to "Abc" in the testCase, (testCase 39). 554 14: { 555 IsTruncated: false, 556 Objects: []ObjectInfo{ 557 {Name: "Asia-maps.png"}, 558 {Name: "Asia/India/India-summer-photos-1"}, 559 {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, 560 {Name: "newPrefix0"}, 561 {Name: "newPrefix1"}, 562 {Name: "newzen/zen/recurse/again/again/again/pics"}, 563 {Name: "obj0"}, 564 {Name: "obj1"}, 565 {Name: "obj2"}, 566 }, 567 }, 568 // ListObjectsResult-15. 569 // Marker is set to "Asia/India/India-summer-photos-1" in the testCase, (testCase 40). 570 15: { 571 IsTruncated: false, 572 Objects: []ObjectInfo{ 573 {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, 574 {Name: "newPrefix0"}, 575 {Name: "newPrefix1"}, 576 {Name: "newzen/zen/recurse/again/again/again/pics"}, 577 {Name: "obj0"}, 578 {Name: "obj1"}, 579 {Name: "obj2"}, 580 }, 581 }, 582 // ListObjectsResult-16. 583 // Marker is set to "Asia/India/Karnataka/Bangalore/Koramangala/pics" in the testCase, (testCase 41). 584 16: { 585 IsTruncated: false, 586 Objects: []ObjectInfo{ 587 {Name: "newPrefix0"}, 588 {Name: "newPrefix1"}, 589 {Name: "newzen/zen/recurse/again/again/again/pics"}, 590 {Name: "obj0"}, 591 {Name: "obj1"}, 592 {Name: "obj2"}, 593 }, 594 }, 595 // ListObjectsResult-17. 596 // Used for asserting the case with marker, without prefix but with truncation. 597 // Marker = "newPrefix0" & maxKeys = 3 in the testCase, (testCase42). 598 // Output truncated to 3 values. 599 17: { 600 IsTruncated: true, 601 Objects: []ObjectInfo{ 602 {Name: "newPrefix1"}, 603 {Name: "newzen/zen/recurse/again/again/again/pics"}, 604 {Name: "obj0"}, 605 }, 606 }, 607 // ListObjectsResult-18. 608 // Marker = "newPrefix1" & maxkeys = 1 in the testCase, (testCase43). 609 // Output truncated to 1 value. 610 18: { 611 IsTruncated: true, 612 Objects: []ObjectInfo{ 613 {Name: "newzen/zen/recurse/again/again/again/pics"}, 614 }, 615 }, 616 // ListObjectsResult-19. 617 // Marker = "obj0" & maxKeys = 1 in the testCase, (testCase44). 618 // Output truncated to 1 value. 619 19: { 620 IsTruncated: true, 621 Objects: []ObjectInfo{ 622 {Name: "obj1"}, 623 }, 624 }, 625 // ListObjectsResult-20. 626 // Marker = "obj0" & prefix = "obj" in the testCase, (testCase 45). 627 20: { 628 IsTruncated: false, 629 Objects: []ObjectInfo{ 630 {Name: "obj1"}, 631 {Name: "obj2"}, 632 }, 633 }, 634 // ListObjectsResult-21. 635 // Marker = "obj1" & prefix = "obj" in the testCase, (testCase 46). 636 21: { 637 IsTruncated: false, 638 Objects: []ObjectInfo{ 639 {Name: "obj2"}, 640 }, 641 }, 642 // ListObjectsResult-22. 643 // Marker = "newPrefix0" & prefix = "new" in the testCase,, (testCase 47). 644 22: { 645 IsTruncated: false, 646 Objects: []ObjectInfo{ 647 {Name: "newPrefix1"}, 648 {Name: "newzen/zen/recurse/again/again/again/pics"}, 649 }, 650 }, 651 // ListObjectsResult-23. 652 // Prefix is set to "Asia/India/" in the testCase, and delimiter is not set (testCase 55). 653 23: { 654 IsTruncated: false, 655 Objects: []ObjectInfo{ 656 {Name: "Asia/India/India-summer-photos-1"}, 657 {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, 658 }, 659 }, 660 661 // ListObjectsResult-24. 662 // Prefix is set to "Asia" in the testCase, and delimiter is not set (testCase 56). 663 24: { 664 IsTruncated: false, 665 Objects: []ObjectInfo{ 666 {Name: "Asia-maps.png"}, 667 {Name: "Asia/India/India-summer-photos-1"}, 668 {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, 669 }, 670 }, 671 672 // ListObjectsResult-25. 673 // Prefix is set to "Asia" in the testCase, and delimiter is set (testCase 57). 674 25: { 675 IsTruncated: false, 676 Objects: []ObjectInfo{ 677 {Name: "Asia-maps.png"}, 678 }, 679 Prefixes: []string{"Asia/"}, 680 }, 681 // ListObjectsResult-26. 682 // prefix = "new" and delimiter is set in the testCase.(testCase 58). 683 26: { 684 IsTruncated: false, 685 Objects: []ObjectInfo{ 686 {Name: "newPrefix0"}, 687 {Name: "newPrefix1"}, 688 }, 689 Prefixes: []string{"newzen/"}, 690 }, 691 // ListObjectsResult-27. 692 // Prefix is set to "Asia/India/" in the testCase, and delimiter is set to forward slash '/' (testCase 59). 693 27: { 694 IsTruncated: false, 695 Objects: []ObjectInfo{ 696 {Name: "Asia/India/India-summer-photos-1"}, 697 }, 698 Prefixes: []string{"Asia/India/Karnataka/"}, 699 }, 700 // ListObjectsResult-28. 701 // Marker is set to "Asia/India/India-summer-photos-1" and delimiter set in the testCase, (testCase 60). 702 28: { 703 IsTruncated: false, 704 Objects: []ObjectInfo{ 705 {Name: "newPrefix0"}, 706 {Name: "newPrefix1"}, 707 {Name: "obj0"}, 708 {Name: "obj1"}, 709 {Name: "obj2"}, 710 }, 711 Prefixes: []string{"newzen/"}, 712 }, 713 // ListObjectsResult-29. 714 // Marker is set to "Asia/India/Karnataka/Bangalore/Koramangala/pics" in the testCase and delimiter set, (testCase 61). 715 29: { 716 IsTruncated: false, 717 Objects: []ObjectInfo{ 718 {Name: "newPrefix0"}, 719 {Name: "newPrefix1"}, 720 {Name: "obj0"}, 721 {Name: "obj1"}, 722 {Name: "obj2"}, 723 }, 724 Prefixes: []string{"newzen/"}, 725 }, 726 // ListObjectsResult-30. 727 // Prefix and Delimiter is set to '/', (testCase 62). 728 30: { 729 IsTruncated: false, 730 Objects: []ObjectInfo{}, 731 }, 732 // ListObjectsResult-31 Empty directory, recursive listing 733 31: { 734 IsTruncated: false, 735 Objects: []ObjectInfo{ 736 {Name: "obj1"}, 737 {Name: "obj2"}, 738 {Name: "temporary/0/"}, 739 }, 740 }, 741 // ListObjectsResult-32 Empty directory, non recursive listing 742 32: { 743 IsTruncated: false, 744 Objects: []ObjectInfo{ 745 {Name: "obj1"}, 746 {Name: "obj2"}, 747 }, 748 Prefixes: []string{"temporary/"}, 749 }, 750 // ListObjectsResult-33 Listing empty directory only 751 33: { 752 IsTruncated: false, 753 Objects: []ObjectInfo{ 754 {Name: "temporary/0/"}, 755 }, 756 }, 757 // ListObjectsResult-34: 758 // * Listing with marker > last object should return empty 759 // * Listing an object with a trailing slash and '/' delimiter 760 34: { 761 IsTruncated: false, 762 Objects: []ObjectInfo{}, 763 }, 764 // ListObjectsResult-35 list with custom uncommon delimiter 765 35: { 766 IsTruncated: false, 767 Objects: []ObjectInfo{ 768 {Name: "file1/receipt.json"}, 769 }, 770 Prefixes: []string{"file1/guidSplunk"}, 771 }, 772 // ListObjectsResult-36 list with nextmarker prefix and maxKeys set to 1. 773 36: { 774 IsTruncated: true, 775 Prefixes: []string{"dir/day_id=2017-10-10/"}, 776 }, 777 // ListObjectsResult-37 list with prefix match 2 levels deep 778 37: { 779 IsTruncated: false, 780 Objects: []ObjectInfo{ 781 {Name: "foo/201910/1112"}, 782 {Name: "foo/201910/1122"}, 783 }, 784 }, 785 // ListObjectsResult-38 list with prefix match 1 level deep 786 38: { 787 IsTruncated: false, 788 Objects: []ObjectInfo{ 789 {Name: "foo/201910/1112"}, 790 {Name: "foo/201910/1122"}, 791 {Name: "foo/201910/2112"}, 792 {Name: "foo/201910_txt"}, 793 }, 794 }, 795 // ListObjectsResult-39 list with prefix match 1 level deep 796 39: { 797 IsTruncated: false, 798 Objects: []ObjectInfo{ 799 {Name: "201910/foo/bar/xl.meta/1.txt"}, 800 }, 801 }, 802 // ListObjectsResult-40 803 40: { 804 IsTruncated: false, 805 Objects: []ObjectInfo{ 806 {Name: "aaa"}, 807 {Name: "ccc"}, 808 }, 809 Prefixes: []string{"bbb_"}, 810 }, 811 } 812 813 testCases := []struct { 814 // Inputs to ListObjects. 815 bucketName string 816 prefix string 817 marker string 818 delimiter string 819 maxKeys int32 820 // Expected output of ListObjects. 821 result ListObjectsInfo 822 err error 823 // Flag indicating whether the test is expected to pass or not. 824 shouldPass bool 825 }{ 826 // Test cases with invalid bucket names ( Test number 1-4 ). 827 {".test", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: ".test"}, false}, 828 {"Test", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: "Test"}, false}, 829 {"---", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: "---"}, false}, 830 {"ad", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: "ad"}, false}, 831 // Valid bucket names, but they do not exist (6-8). 832 {"volatile-bucket-1", "", "", "", 1000, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false}, 833 {"volatile-bucket-2", "", "", "", 1000, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false}, 834 {"volatile-bucket-3", "", "", "", 1000, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false}, 835 // If marker is *after* the last possible object from the prefix it should return an empty list. 836 {"test-bucket-list-object", "Asia", "europe-object", "", 0, ListObjectsInfo{}, nil, true}, 837 // If the marker is *before* the first possible object from the prefix it should return the first object. 838 {"test-bucket-list-object", "Asia", "A", "", 1, resultCases[4], nil, true}, 839 // Setting a non-existing directory to be prefix (12-13). 840 {"empty-bucket", "europe/france/", "", "", 1, ListObjectsInfo{}, nil, true}, 841 {"empty-bucket", "africa/tunisia/", "", "", 1, ListObjectsInfo{}, nil, true}, 842 // Testing on empty bucket, that is, bucket without any objects in it (14). 843 {"empty-bucket", "", "", "", 0, ListObjectsInfo{}, nil, true}, 844 // Setting maxKeys to negative value (15-16). 845 {"empty-bucket", "", "", "", -1, ListObjectsInfo{}, nil, true}, 846 {"empty-bucket", "", "", "", 1, ListObjectsInfo{}, nil, true}, 847 // Setting maxKeys to a very large value (17). 848 {"empty-bucket", "", "", "", 111100000, ListObjectsInfo{}, nil, true}, 849 // Testing for all 10 objects in the bucket (18). 850 {"test-bucket-list-object", "", "", "", 10, resultCases[0], nil, true}, 851 // Testing for negative value of maxKey, this should set maxKeys to listObjectsLimit (19). 852 {"test-bucket-list-object", "", "", "", -1, resultCases[0], nil, true}, 853 // Testing for very large value of maxKey, this should set maxKeys to listObjectsLimit (20). 854 {"test-bucket-list-object", "", "", "", 1234567890, resultCases[0], nil, true}, 855 // Testing for trancated value (21-24). 856 {"test-bucket-list-object", "", "", "", 5, resultCases[1], nil, true}, 857 {"test-bucket-list-object", "", "", "", 4, resultCases[2], nil, true}, 858 {"test-bucket-list-object", "", "", "", 3, resultCases[3], nil, true}, 859 {"test-bucket-list-object", "", "", "", 1, resultCases[4], nil, true}, 860 // Testing with prefix (25-28). 861 {"test-bucket-list-object", "new", "", "", 3, resultCases[5], nil, true}, 862 {"test-bucket-list-object", "new", "", "", 4, resultCases[5], nil, true}, 863 {"test-bucket-list-object", "new", "", "", 5, resultCases[5], nil, true}, 864 {"test-bucket-list-object", "obj", "", "", 3, resultCases[6], nil, true}, 865 {"test-bucket-list-object", "/obj", "", "", 0, ListObjectsInfo{}, nil, true}, 866 // Testing with prefix and truncation (29-30). 867 {"test-bucket-list-object", "new", "", "", 1, resultCases[7], nil, true}, 868 {"test-bucket-list-object", "obj", "", "", 2, resultCases[8], nil, true}, 869 // Testing with marker, but without prefix and truncation (31-35). 870 {"test-bucket-list-object", "", "newPrefix0", "", 6, resultCases[9], nil, true}, 871 {"test-bucket-list-object", "", "newPrefix1", "", 5, resultCases[10], nil, true}, 872 {"test-bucket-list-object", "", "obj0", "", 4, resultCases[11], nil, true}, 873 {"test-bucket-list-object", "", "obj1", "", 2, resultCases[12], nil, true}, 874 {"test-bucket-list-object", "", "man", "", 11, resultCases[13], nil, true}, 875 // Marker being set to a value which is greater than and all object names when sorted (36). 876 // Expected to send an empty response in this case. 877 {"test-bucket-list-object", "", "zen", "", 10, ListObjectsInfo{}, nil, true}, 878 // Marker being set to a value which is lesser than and all object names when sorted (37). 879 // Expected to send all the objects in the bucket in this case. 880 {"test-bucket-list-object", "", "Abc", "", 10, resultCases[14], nil, true}, 881 // Marker is to a hierarhical value (38-39). 882 {"test-bucket-list-object", "", "Asia/India/India-summer-photos-1", "", 10, resultCases[15], nil, true}, 883 {"test-bucket-list-object", "", "Asia/India/Karnataka/Bangalore/Koramangala/pics", "", 10, resultCases[16], nil, true}, 884 // Testing with marker and truncation, but no prefix (40-42). 885 {"test-bucket-list-object", "", "newPrefix0", "", 3, resultCases[17], nil, true}, 886 {"test-bucket-list-object", "", "newPrefix1", "", 1, resultCases[18], nil, true}, 887 {"test-bucket-list-object", "", "obj0", "", 1, resultCases[19], nil, true}, 888 // Testing with both marker and prefix, but without truncation (43-45). 889 // The valid combination of marker and prefix should satisfy strings.HasPrefix(marker, prefix). 890 {"test-bucket-list-object", "obj", "obj0", "", 2, resultCases[20], nil, true}, 891 {"test-bucket-list-object", "obj", "obj1", "", 1, resultCases[21], nil, true}, 892 {"test-bucket-list-object", "new", "newPrefix0", "", 2, resultCases[22], nil, true}, 893 // Testing with maxKeys set to 0 (46-52). 894 // The parameters have to valid. 895 {"test-bucket-list-object", "", "obj1", "", 0, ListObjectsInfo{}, nil, true}, 896 {"test-bucket-list-object", "", "obj0", "", 0, ListObjectsInfo{}, nil, true}, 897 {"test-bucket-list-object", "new", "", "", 0, ListObjectsInfo{}, nil, true}, 898 {"test-bucket-list-object", "obj", "", "", 0, ListObjectsInfo{}, nil, true}, 899 {"test-bucket-list-object", "obj", "obj0", "", 0, ListObjectsInfo{}, nil, true}, 900 {"test-bucket-list-object", "obj", "obj1", "", 0, ListObjectsInfo{}, nil, true}, 901 {"test-bucket-list-object", "new", "newPrefix0", "", 0, ListObjectsInfo{}, nil, true}, 902 // Tests on hierarchical key names as prefix. 903 // Without delimteter the code should recurse into the prefix Dir. 904 // Tests with prefix, but without delimiter (53-54). 905 {"test-bucket-list-object", "Asia/India/", "", "", 10, resultCases[23], nil, true}, 906 {"test-bucket-list-object", "Asia", "", "", 10, resultCases[24], nil, true}, 907 // Tests with prefix and delimiter (55-57). 908 // With delimiter the code should not recurse into the sub-directories of prefix Dir. 909 {"test-bucket-list-object", "Asia", "", SlashSeparator, 10, resultCases[25], nil, true}, 910 {"test-bucket-list-object", "new", "", SlashSeparator, 10, resultCases[26], nil, true}, 911 {"test-bucket-list-object", "Asia/India/", "", SlashSeparator, 10, resultCases[27], nil, true}, 912 // Test with marker set as hierarhical value and with delimiter. (58-59) 913 {"test-bucket-list-object", "", "Asia/India/India-summer-photos-1", SlashSeparator, 10, resultCases[28], nil, true}, 914 {"test-bucket-list-object", "", "Asia/India/Karnataka/Bangalore/Koramangala/pics", SlashSeparator, 10, resultCases[29], nil, true}, 915 // Test with prefix and delimiter set to '/'. (60) 916 {"test-bucket-list-object", SlashSeparator, "", SlashSeparator, 10, resultCases[30], nil, true}, 917 // Test with invalid prefix (61) 918 {"test-bucket-list-object", "\\", "", SlashSeparator, 10, ListObjectsInfo{}, nil, true}, 919 // Test listing an empty directory in recursive mode (62) 920 {"test-bucket-empty-dir", "", "", "", 10, resultCases[31], nil, true}, 921 // Test listing an empty directory in a non recursive mode (63) 922 {"test-bucket-empty-dir", "", "", SlashSeparator, 10, resultCases[32], nil, true}, 923 // Test listing a directory which contains an empty directory (64) 924 {"test-bucket-empty-dir", "", "temporary/", "", 10, resultCases[33], nil, true}, 925 // Test listing with marker > last object such that response should be empty (65) 926 {"test-bucket-single-object", "", "A/C", "", 1000, resultCases[34], nil, true}, 927 // Test listing an object with a trailing slash and a slash delimiter (66) 928 {"test-bucket-list-object", "Asia-maps.png/", "", "/", 1000, resultCases[34], nil, true}, 929 // Test listing an object with uncommon delimiter 930 {testBuckets[4], "", "", "guidSplunk", 1000, resultCases[35], nil, true}, 931 // Test listing an object with uncommon delimiter and matching prefix 932 {testBuckets[4], "file1/", "", "guidSplunk", 1000, resultCases[35], nil, true}, 933 // Test listing at prefix with expected prefix markers 934 {testBuckets[5], "dir/", "", SlashSeparator, 1, resultCases[36], nil, true}, 935 // Test listing with prefix match 936 {testBuckets[5], "foo/201910/11", "", "", 1000, resultCases[37], nil, true}, 937 {testBuckets[5], "foo/201910", "", "", 1000, resultCases[38], nil, true}, 938 // Test listing with prefix match with 'xl.meta' 939 {testBuckets[5], "201910/foo/bar", "", "", 1000, resultCases[39], nil, true}, 940 // Test listing with custom prefix 941 {testBuckets[6], "", "", "_", 1000, resultCases[40], nil, true}, 942 } 943 944 for i, testCase := range testCases { 945 testCase := testCase 946 t.Run(fmt.Sprintf("%s-Test%d", instanceType, i+1), func(t *testing.T) { 947 t.Log("ListObjects, bucket:", testCase.bucketName, "prefix:", testCase.prefix, "marker:", testCase.marker, "delimiter:", testCase.delimiter, "maxkeys:", testCase.maxKeys) 948 result, err := obj.ListObjects(context.Background(), testCase.bucketName, 949 testCase.prefix, testCase.marker, testCase.delimiter, int(testCase.maxKeys)) 950 if err != nil && testCase.shouldPass { 951 t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, err.Error()) 952 } 953 if err == nil && !testCase.shouldPass { 954 t.Errorf("Test %d: %s: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, instanceType, testCase.err.Error()) 955 } 956 // Failed as expected, but does it fail for the expected reason. 957 if err != nil && !testCase.shouldPass { 958 if !strings.Contains(err.Error(), testCase.err.Error()) { 959 t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, instanceType, testCase.err.Error(), err.Error()) 960 } 961 } 962 // Since there are cases for which ListObjects fails, this is 963 // necessary. Test passes as expected, but the output values 964 // are verified for correctness here. 965 if err == nil && testCase.shouldPass { 966 // The length of the expected ListObjectsResult.Objects 967 // should match in both expected result from test cases 968 // and in the output. On failure calling t.Fatalf, 969 // otherwise it may lead to index out of range error in 970 // assertion following this. 971 if len(testCase.result.Objects) != len(result.Objects) { 972 t.Logf("want: %v", objInfoNames(testCase.result.Objects)) 973 t.Logf("got: %v", objInfoNames(result.Objects)) 974 t.Errorf("Test %d: %s: Expected number of object in the result to be '%d', but found '%d' objects instead", i+1, instanceType, len(testCase.result.Objects), len(result.Objects)) 975 } 976 for j := 0; j < len(testCase.result.Objects); j++ { 977 if j >= len(result.Objects) { 978 t.Errorf("Test %d: %s: Expected object name to be \"%s\", but not nothing instead", i+1, instanceType, testCase.result.Objects[j].Name) 979 continue 980 } 981 if testCase.result.Objects[j].Name != result.Objects[j].Name { 982 t.Errorf("Test %d: %s: Expected object name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.result.Objects[j].Name, result.Objects[j].Name) 983 } 984 } 985 986 if len(testCase.result.Prefixes) != len(result.Prefixes) { 987 t.Logf("want: %v", testCase.result.Prefixes) 988 t.Logf("got: %v", result.Prefixes) 989 t.Errorf("Test %d: %s: Expected number of prefixes in the result to be '%d', but found '%d' prefixes instead", i+1, instanceType, len(testCase.result.Prefixes), len(result.Prefixes)) 990 } 991 for j := 0; j < len(testCase.result.Prefixes); j++ { 992 if j >= len(result.Prefixes) { 993 t.Errorf("Test %d: %s: Expected prefix name to be \"%s\", but found no result", i+1, instanceType, testCase.result.Prefixes[j]) 994 continue 995 } 996 if testCase.result.Prefixes[j] != result.Prefixes[j] { 997 t.Errorf("Test %d: %s: Expected prefix name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.result.Prefixes[j], result.Prefixes[j]) 998 } 999 } 1000 1001 if testCase.result.IsTruncated != result.IsTruncated { 1002 // Allow an extra continuation token. 1003 if !result.IsTruncated || len(result.Objects) == 0 { 1004 t.Errorf("Test %d: %s: Expected IsTruncated flag to be %v, but instead found it to be %v", i+1, instanceType, testCase.result.IsTruncated, result.IsTruncated) 1005 } 1006 } 1007 1008 if testCase.result.IsTruncated && result.NextMarker == "" { 1009 t.Errorf("Test %d: %s: Expected NextMarker to contain a string since listing is truncated, but instead found it to be empty", i+1, instanceType) 1010 } 1011 1012 if !testCase.result.IsTruncated && result.NextMarker != "" { 1013 if !result.IsTruncated || len(result.Objects) == 0 { 1014 t.Errorf("Test %d: %s: Expected NextMarker to be empty since listing is not truncated, but instead found `%v`", i+1, instanceType, result.NextMarker) 1015 } 1016 } 1017 1018 } 1019 }) 1020 } 1021 } 1022 1023 func objInfoNames(o []ObjectInfo) []string { 1024 res := make([]string, len(o)) 1025 for i := range o { 1026 res[i] = o[i].Name 1027 } 1028 return res 1029 } 1030 1031 func TestDeleteObjectVersionMarker(t *testing.T) { 1032 ExecObjectLayerTest(t, testDeleteObjectVersion) 1033 } 1034 1035 func testDeleteObjectVersion(obj ObjectLayer, instanceType string, t1 TestErrHandler) { 1036 t, _ := t1.(*testing.T) 1037 1038 testBuckets := []string{ 1039 "bucket-suspended-version", 1040 "bucket-suspended-version-id", 1041 } 1042 for _, bucket := range testBuckets { 1043 err := obj.MakeBucket(context.Background(), bucket, MakeBucketOptions{ 1044 VersioningEnabled: true, 1045 }) 1046 if err != nil { 1047 t.Fatalf("%s : %s", instanceType, err) 1048 } 1049 meta, err := loadBucketMetadata(context.Background(), obj, bucket) 1050 if err != nil { 1051 t.Fatalf("%s : %s", instanceType, err) 1052 } 1053 meta.VersioningConfigXML = []byte(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Suspended</Status></VersioningConfiguration>`) 1054 if err := meta.Save(context.Background(), obj); err != nil { 1055 t.Fatalf("%s : %s", instanceType, err) 1056 } 1057 globalBucketMetadataSys.Set(bucket, meta) 1058 globalNotificationSys.LoadBucketMetadata(context.Background(), bucket) 1059 } 1060 1061 testObjects := []struct { 1062 parentBucket string 1063 name string 1064 content string 1065 meta map[string]string 1066 versionID string 1067 expectDelMarker bool 1068 }{ 1069 {testBuckets[0], "delete-file", "contentstring", nil, "", true}, 1070 {testBuckets[1], "delete-file", "contentstring", nil, "null", false}, 1071 } 1072 for _, object := range testObjects { 1073 md5Bytes := md5.Sum([]byte(object.content)) 1074 _, err := obj.PutObject(context.Background(), object.parentBucket, object.name, 1075 mustGetPutObjReader(t, bytes.NewBufferString(object.content), 1076 int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), ObjectOptions{ 1077 Versioned: globalBucketVersioningSys.PrefixEnabled(object.parentBucket, object.name), 1078 VersionSuspended: globalBucketVersioningSys.PrefixSuspended(object.parentBucket, object.name), 1079 UserDefined: object.meta, 1080 }) 1081 if err != nil { 1082 t.Fatalf("%s : %s", instanceType, err) 1083 } 1084 obj, err := obj.DeleteObject(context.Background(), object.parentBucket, object.name, ObjectOptions{ 1085 Versioned: globalBucketVersioningSys.PrefixEnabled(object.parentBucket, object.name), 1086 VersionSuspended: globalBucketVersioningSys.PrefixSuspended(object.parentBucket, object.name), 1087 VersionID: object.versionID, 1088 }) 1089 if err != nil { 1090 if object.versionID != "" { 1091 if !isErrVersionNotFound(err) { 1092 t.Fatalf("%s : %s", instanceType, err) 1093 } 1094 } else { 1095 if !isErrObjectNotFound(err) { 1096 t.Fatalf("%s : %s", instanceType, err) 1097 } 1098 } 1099 } 1100 if obj.DeleteMarker != object.expectDelMarker { 1101 t.Fatalf("%s : expected deleted marker %t, found %t", instanceType, object.expectDelMarker, obj.DeleteMarker) 1102 } 1103 } 1104 } 1105 1106 // Wrapper for calling ListObjectVersions tests for both Erasure multiple disks and single node setup. 1107 func TestListObjectVersions(t *testing.T) { 1108 ExecObjectLayerTest(t, testListObjectVersions) 1109 } 1110 1111 // Unit test for ListObjectVersions 1112 func testListObjectVersions(obj ObjectLayer, instanceType string, t1 TestErrHandler) { 1113 t, _ := t1.(*testing.T) 1114 testBuckets := []string{ 1115 // This bucket is used for testing ListObject operations. 1116 "test-bucket-list-object", 1117 // This bucket will be tested with empty directories 1118 "test-bucket-empty-dir", 1119 // Will not store any objects in this bucket, 1120 // Its to test ListObjects on an empty bucket. 1121 "empty-bucket", 1122 // Listing the case where the marker > last object. 1123 "test-bucket-single-object", 1124 // Listing uncommon delimiter. 1125 "test-bucket-delimiter", 1126 // Listing prefixes > maxKeys 1127 "test-bucket-max-keys-prefixes", 1128 } 1129 for _, bucket := range testBuckets { 1130 err := obj.MakeBucket(context.Background(), bucket, MakeBucketOptions{VersioningEnabled: true}) 1131 if err != nil { 1132 t.Fatalf("%s : %s", instanceType, err.Error()) 1133 } 1134 } 1135 1136 var err error 1137 testObjects := []struct { 1138 parentBucket string 1139 name string 1140 content string 1141 meta map[string]string 1142 }{ 1143 {testBuckets[0], "Asia-maps.png", "asis-maps", map[string]string{"content-type": "image/png"}}, 1144 {testBuckets[0], "Asia/India/India-summer-photos-1", "contentstring", nil}, 1145 {testBuckets[0], "Asia/India/Karnataka/Bangalore/Koramangala/pics", "contentstring", nil}, 1146 {testBuckets[0], "newPrefix0", "newPrefix0", nil}, 1147 {testBuckets[0], "newPrefix1", "newPrefix1", nil}, 1148 {testBuckets[0], "newzen/zen/recurse/again/again/again/pics", "recurse", nil}, 1149 {testBuckets[0], "obj0", "obj0", nil}, 1150 {testBuckets[0], "obj1", "obj1", nil}, 1151 {testBuckets[0], "obj2", "obj2", nil}, 1152 {testBuckets[1], "obj1", "obj1", nil}, 1153 {testBuckets[1], "obj2", "obj2", nil}, 1154 {testBuckets[1], "temporary/0/", "", nil}, 1155 {testBuckets[3], "A/B", "contentstring", nil}, 1156 {testBuckets[4], "file1/receipt.json", "content", nil}, 1157 {testBuckets[4], "file1/guidSplunk-aaaa/file", "content", nil}, 1158 {testBuckets[5], "dir/day_id=2017-10-10/issue", "content", nil}, 1159 {testBuckets[5], "dir/day_id=2017-10-11/issue", "content", nil}, 1160 } 1161 1162 for _, object := range testObjects { 1163 md5Bytes := md5.Sum([]byte(object.content)) 1164 _, err = obj.PutObject(context.Background(), object.parentBucket, object.name, mustGetPutObjReader(t, bytes.NewBufferString(object.content), 1165 int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), ObjectOptions{UserDefined: object.meta}) 1166 if err != nil { 1167 t.Fatalf("%s : %s", instanceType, err.Error()) 1168 } 1169 1170 } 1171 1172 // Formualting the result data set to be expected from ListObjects call inside the tests, 1173 // This will be used in testCases and used for asserting the correctness of ListObjects output in the tests. 1174 1175 resultCases := []ListObjectsInfo{ 1176 // ListObjectsResult-0. 1177 // Testing for listing all objects in the bucket, (testCase 20,21,22). 1178 { 1179 IsTruncated: false, 1180 Objects: []ObjectInfo{ 1181 {Name: "Asia-maps.png"}, 1182 {Name: "Asia/India/India-summer-photos-1"}, 1183 {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, 1184 {Name: "newPrefix0"}, 1185 {Name: "newPrefix1"}, 1186 {Name: "newzen/zen/recurse/again/again/again/pics"}, 1187 {Name: "obj0"}, 1188 {Name: "obj1"}, 1189 {Name: "obj2"}, 1190 }, 1191 }, 1192 // ListObjectsResult-1. 1193 // Used for asserting the truncated case, (testCase 23). 1194 { 1195 IsTruncated: true, 1196 Objects: []ObjectInfo{ 1197 {Name: "Asia-maps.png"}, 1198 {Name: "Asia/India/India-summer-photos-1"}, 1199 {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, 1200 {Name: "newPrefix0"}, 1201 {Name: "newPrefix1"}, 1202 }, 1203 }, 1204 // ListObjectsResult-2. 1205 // (TestCase 24). 1206 { 1207 IsTruncated: true, 1208 Objects: []ObjectInfo{ 1209 {Name: "Asia-maps.png"}, 1210 {Name: "Asia/India/India-summer-photos-1"}, 1211 {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, 1212 {Name: "newPrefix0"}, 1213 }, 1214 }, 1215 // ListObjectsResult-3. 1216 // (TestCase 25). 1217 { 1218 IsTruncated: true, 1219 Objects: []ObjectInfo{ 1220 {Name: "Asia-maps.png"}, 1221 {Name: "Asia/India/India-summer-photos-1"}, 1222 {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, 1223 }, 1224 }, 1225 // ListObjectsResult-4. 1226 // Again used for truncated case. 1227 // (TestCase 26). 1228 { 1229 IsTruncated: true, 1230 Objects: []ObjectInfo{ 1231 {Name: "Asia-maps.png"}, 1232 }, 1233 }, 1234 // ListObjectsResult-5. 1235 // Used for Asserting prefixes. 1236 // Used for test case with prefix "new", (testCase 27-29). 1237 { 1238 IsTruncated: false, 1239 Objects: []ObjectInfo{ 1240 {Name: "newPrefix0"}, 1241 {Name: "newPrefix1"}, 1242 {Name: "newzen/zen/recurse/again/again/again/pics"}, 1243 }, 1244 }, 1245 // ListObjectsResult-6. 1246 // Used for Asserting prefixes. 1247 // Used for test case with prefix = "obj", (testCase 30). 1248 { 1249 IsTruncated: false, 1250 Objects: []ObjectInfo{ 1251 {Name: "obj0"}, 1252 {Name: "obj1"}, 1253 {Name: "obj2"}, 1254 }, 1255 }, 1256 // ListObjectsResult-7. 1257 // Used for Asserting prefixes and truncation. 1258 // Used for test case with prefix = "new" and maxKeys = 1, (testCase 31). 1259 { 1260 IsTruncated: true, 1261 Objects: []ObjectInfo{ 1262 {Name: "newPrefix0"}, 1263 }, 1264 }, 1265 // ListObjectsResult-8. 1266 // Used for Asserting prefixes. 1267 // Used for test case with prefix = "obj" and maxKeys = 2, (testCase 32). 1268 { 1269 IsTruncated: true, 1270 Objects: []ObjectInfo{ 1271 {Name: "obj0"}, 1272 {Name: "obj1"}, 1273 }, 1274 }, 1275 // ListObjectsResult-9. 1276 // Used for asserting the case with marker, but without prefix. 1277 // marker is set to "newPrefix0" in the testCase, (testCase 33). 1278 { 1279 IsTruncated: false, 1280 Objects: []ObjectInfo{ 1281 {Name: "newPrefix1"}, 1282 {Name: "newzen/zen/recurse/again/again/again/pics"}, 1283 {Name: "obj0"}, 1284 {Name: "obj1"}, 1285 {Name: "obj2"}, 1286 }, 1287 }, 1288 // ListObjectsResult-10. 1289 // marker is set to "newPrefix1" in the testCase, (testCase 34). 1290 { 1291 IsTruncated: false, 1292 Objects: []ObjectInfo{ 1293 {Name: "newzen/zen/recurse/again/again/again/pics"}, 1294 {Name: "obj0"}, 1295 {Name: "obj1"}, 1296 {Name: "obj2"}, 1297 }, 1298 }, 1299 // ListObjectsResult-11. 1300 // marker is set to "obj0" in the testCase, (testCase 35). 1301 { 1302 IsTruncated: false, 1303 Objects: []ObjectInfo{ 1304 {Name: "obj1"}, 1305 {Name: "obj2"}, 1306 }, 1307 }, 1308 // ListObjectsResult-12. 1309 // Marker is set to "obj1" in the testCase, (testCase 36). 1310 { 1311 IsTruncated: false, 1312 Objects: []ObjectInfo{ 1313 {Name: "obj2"}, 1314 }, 1315 }, 1316 // ListObjectsResult-13. 1317 // Marker is set to "man" in the testCase, (testCase37). 1318 { 1319 IsTruncated: false, 1320 Objects: []ObjectInfo{ 1321 {Name: "newPrefix0"}, 1322 {Name: "newPrefix1"}, 1323 {Name: "newzen/zen/recurse/again/again/again/pics"}, 1324 {Name: "obj0"}, 1325 {Name: "obj1"}, 1326 {Name: "obj2"}, 1327 }, 1328 }, 1329 // ListObjectsResult-14. 1330 // Marker is set to "Abc" in the testCase, (testCase 39). 1331 { 1332 IsTruncated: false, 1333 Objects: []ObjectInfo{ 1334 {Name: "Asia-maps.png"}, 1335 {Name: "Asia/India/India-summer-photos-1"}, 1336 {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, 1337 {Name: "newPrefix0"}, 1338 {Name: "newPrefix1"}, 1339 {Name: "newzen/zen/recurse/again/again/again/pics"}, 1340 {Name: "obj0"}, 1341 {Name: "obj1"}, 1342 {Name: "obj2"}, 1343 }, 1344 }, 1345 // ListObjectsResult-15. 1346 // Marker is set to "Asia/India/India-summer-photos-1" in the testCase, (testCase 40). 1347 { 1348 IsTruncated: false, 1349 Objects: []ObjectInfo{ 1350 {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, 1351 {Name: "newPrefix0"}, 1352 {Name: "newPrefix1"}, 1353 {Name: "newzen/zen/recurse/again/again/again/pics"}, 1354 {Name: "obj0"}, 1355 {Name: "obj1"}, 1356 {Name: "obj2"}, 1357 }, 1358 }, 1359 // ListObjectsResult-16. 1360 // Marker is set to "Asia/India/Karnataka/Bangalore/Koramangala/pics" in the testCase, (testCase 41). 1361 { 1362 IsTruncated: false, 1363 Objects: []ObjectInfo{ 1364 {Name: "newPrefix0"}, 1365 {Name: "newPrefix1"}, 1366 {Name: "newzen/zen/recurse/again/again/again/pics"}, 1367 {Name: "obj0"}, 1368 {Name: "obj1"}, 1369 {Name: "obj2"}, 1370 }, 1371 }, 1372 // ListObjectsResult-17. 1373 // Used for asserting the case with marker, without prefix but with truncation. 1374 // Marker = "newPrefix0" & maxKeys = 3 in the testCase, (testCase42). 1375 // Output truncated to 3 values. 1376 { 1377 IsTruncated: true, 1378 Objects: []ObjectInfo{ 1379 {Name: "newPrefix1"}, 1380 {Name: "newzen/zen/recurse/again/again/again/pics"}, 1381 {Name: "obj0"}, 1382 }, 1383 }, 1384 // ListObjectsResult-18. 1385 // Marker = "newPrefix1" & maxkeys = 1 in the testCase, (testCase43). 1386 // Output truncated to 1 value. 1387 { 1388 IsTruncated: true, 1389 Objects: []ObjectInfo{ 1390 {Name: "newzen/zen/recurse/again/again/again/pics"}, 1391 }, 1392 }, 1393 // ListObjectsResult-19. 1394 // Marker = "obj0" & maxKeys = 1 in the testCase, (testCase44). 1395 // Output truncated to 1 value. 1396 { 1397 IsTruncated: true, 1398 Objects: []ObjectInfo{ 1399 {Name: "obj1"}, 1400 }, 1401 }, 1402 // ListObjectsResult-20. 1403 // Marker = "obj0" & prefix = "obj" in the testCase, (testCase 45). 1404 { 1405 IsTruncated: false, 1406 Objects: []ObjectInfo{ 1407 {Name: "obj1"}, 1408 {Name: "obj2"}, 1409 }, 1410 }, 1411 // ListObjectsResult-21. 1412 // Marker = "obj1" & prefix = "obj" in the testCase, (testCase 46). 1413 { 1414 IsTruncated: false, 1415 Objects: []ObjectInfo{ 1416 {Name: "obj2"}, 1417 }, 1418 }, 1419 // ListObjectsResult-22. 1420 // Marker = "newPrefix0" & prefix = "new" in the testCase,, (testCase 47). 1421 { 1422 IsTruncated: false, 1423 Objects: []ObjectInfo{ 1424 {Name: "newPrefix1"}, 1425 {Name: "newzen/zen/recurse/again/again/again/pics"}, 1426 }, 1427 }, 1428 // ListObjectsResult-23. 1429 // Prefix is set to "Asia/India/" in the testCase, and delimiter is not set (testCase 55). 1430 { 1431 IsTruncated: false, 1432 Objects: []ObjectInfo{ 1433 {Name: "Asia/India/India-summer-photos-1"}, 1434 {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, 1435 }, 1436 }, 1437 1438 // ListObjectsResult-24. 1439 // Prefix is set to "Asia" in the testCase, and delimiter is not set (testCase 56). 1440 { 1441 IsTruncated: false, 1442 Objects: []ObjectInfo{ 1443 {Name: "Asia-maps.png"}, 1444 {Name: "Asia/India/India-summer-photos-1"}, 1445 {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, 1446 }, 1447 }, 1448 1449 // ListObjectsResult-25. 1450 // Prefix is set to "Asia" in the testCase, and delimiter is set (testCase 57). 1451 { 1452 IsTruncated: false, 1453 Objects: []ObjectInfo{ 1454 {Name: "Asia-maps.png"}, 1455 }, 1456 Prefixes: []string{"Asia/"}, 1457 }, 1458 // ListObjectsResult-26. 1459 // prefix = "new" and delimiter is set in the testCase.(testCase 58). 1460 { 1461 IsTruncated: false, 1462 Objects: []ObjectInfo{ 1463 {Name: "newPrefix0"}, 1464 {Name: "newPrefix1"}, 1465 }, 1466 Prefixes: []string{"newzen/"}, 1467 }, 1468 // ListObjectsResult-27. 1469 // Prefix is set to "Asia/India/" in the testCase, and delimiter is set to forward slash '/' (testCase 59). 1470 { 1471 IsTruncated: false, 1472 Objects: []ObjectInfo{ 1473 {Name: "Asia/India/India-summer-photos-1"}, 1474 }, 1475 Prefixes: []string{"Asia/India/Karnataka/"}, 1476 }, 1477 // ListObjectsResult-28. 1478 // Marker is set to "Asia/India/India-summer-photos-1" and delimiter set in the testCase, (testCase 60). 1479 { 1480 IsTruncated: false, 1481 Objects: []ObjectInfo{ 1482 {Name: "newPrefix0"}, 1483 {Name: "newPrefix1"}, 1484 {Name: "obj0"}, 1485 {Name: "obj1"}, 1486 {Name: "obj2"}, 1487 }, 1488 Prefixes: []string{"newzen/"}, 1489 }, 1490 // ListObjectsResult-29. 1491 // Marker is set to "Asia/India/Karnataka/Bangalore/Koramangala/pics" in the testCase and delimiter set, (testCase 61). 1492 { 1493 IsTruncated: false, 1494 Objects: []ObjectInfo{ 1495 {Name: "newPrefix0"}, 1496 {Name: "newPrefix1"}, 1497 {Name: "obj0"}, 1498 {Name: "obj1"}, 1499 {Name: "obj2"}, 1500 }, 1501 Prefixes: []string{"newzen/"}, 1502 }, 1503 // ListObjectsResult-30. 1504 // Prefix and Delimiter is set to '/', (testCase 62). 1505 { 1506 IsTruncated: false, 1507 Objects: []ObjectInfo{}, 1508 }, 1509 // ListObjectsResult-31 Empty directory, recursive listing 1510 { 1511 IsTruncated: false, 1512 Objects: []ObjectInfo{ 1513 {Name: "obj1"}, 1514 {Name: "obj2"}, 1515 {Name: "temporary/0/"}, 1516 }, 1517 }, 1518 // ListObjectsResult-32 Empty directory, non recursive listing 1519 { 1520 IsTruncated: false, 1521 Objects: []ObjectInfo{ 1522 {Name: "obj1"}, 1523 {Name: "obj2"}, 1524 }, 1525 Prefixes: []string{"temporary/"}, 1526 }, 1527 // ListObjectsResult-33 Listing empty directory only 1528 { 1529 IsTruncated: false, 1530 Objects: []ObjectInfo{ 1531 {Name: "temporary/0/"}, 1532 }, 1533 }, 1534 // ListObjectsResult-34: 1535 // * Listing with marker > last object should return empty 1536 // * Listing an object with a trailing slash and '/' delimiter 1537 { 1538 IsTruncated: false, 1539 Objects: []ObjectInfo{}, 1540 }, 1541 // ListObjectsResult-35 list with custom uncommon delimiter 1542 { 1543 IsTruncated: false, 1544 Objects: []ObjectInfo{ 1545 {Name: "file1/receipt.json"}, 1546 }, 1547 Prefixes: []string{"file1/guidSplunk"}, 1548 }, 1549 // ListObjectsResult-36 list with nextmarker prefix and maxKeys set to 1. 1550 { 1551 IsTruncated: true, 1552 Prefixes: []string{"dir/day_id=2017-10-10/"}, 1553 }, 1554 } 1555 1556 testCases := []struct { 1557 // Inputs to ListObjects. 1558 bucketName string 1559 prefix string 1560 marker string 1561 delimiter string 1562 maxKeys int32 1563 // Expected output of ListObjects. 1564 result ListObjectsInfo 1565 err error 1566 // Flag indicating whether the test is expected to pass or not. 1567 shouldPass bool 1568 }{ 1569 // Test cases with invalid bucket names ( Test number 1-4). 1570 {".test", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: ".test"}, false}, 1571 {"Test", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: "Test"}, false}, 1572 {"---", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: "---"}, false}, 1573 {"ad", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: "ad"}, false}, 1574 // Valid bucket names, but they do not exist (6-8). 1575 {"volatile-bucket-1", "", "", "", 1000, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false}, 1576 {"volatile-bucket-2", "", "", "", 1000, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false}, 1577 {"volatile-bucket-3", "", "", "", 1000, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false}, 1578 // If marker is *after* the last possible object from the prefix it should return an empty list. 1579 {"test-bucket-list-object", "Asia", "europe-object", "", 0, ListObjectsInfo{}, nil, true}, 1580 // Setting a non-existing directory to be prefix (10-11). 1581 {"empty-bucket", "europe/france/", "", "", 1, ListObjectsInfo{}, nil, true}, 1582 {"empty-bucket", "africa/tunisia/", "", "", 1, ListObjectsInfo{}, nil, true}, 1583 // Testing on empty bucket, that is, bucket without any objects in it (12). 1584 {"empty-bucket", "", "", "", 0, ListObjectsInfo{}, nil, true}, 1585 // Setting maxKeys to negative value (13-14). 1586 {"empty-bucket", "", "", "", -1, ListObjectsInfo{}, nil, true}, 1587 {"empty-bucket", "", "", "", 1, ListObjectsInfo{}, nil, true}, 1588 // Setting maxKeys to a very large value (15). 1589 {"empty-bucket", "", "", "", 111100000, ListObjectsInfo{}, nil, true}, 1590 // Testing for all 10 objects in the bucket (16). 1591 {"test-bucket-list-object", "", "", "", 10, resultCases[0], nil, true}, 1592 // Testing for negative value of maxKey, this should set maxKeys to listObjectsLimit (17). 1593 {"test-bucket-list-object", "", "", "", -1, resultCases[0], nil, true}, 1594 // Testing for very large value of maxKey, this should set maxKeys to listObjectsLimit (18). 1595 {"test-bucket-list-object", "", "", "", 1234567890, resultCases[0], nil, true}, 1596 // Testing for trancated value (19-22). 1597 {"test-bucket-list-object", "", "", "", 5, resultCases[1], nil, true}, 1598 {"test-bucket-list-object", "", "", "", 4, resultCases[2], nil, true}, 1599 {"test-bucket-list-object", "", "", "", 3, resultCases[3], nil, true}, 1600 {"test-bucket-list-object", "", "", "", 1, resultCases[4], nil, true}, 1601 // Testing with prefix (23-26). 1602 {"test-bucket-list-object", "new", "", "", 3, resultCases[5], nil, true}, 1603 {"test-bucket-list-object", "new", "", "", 4, resultCases[5], nil, true}, 1604 {"test-bucket-list-object", "new", "", "", 5, resultCases[5], nil, true}, 1605 {"test-bucket-list-object", "obj", "", "", 3, resultCases[6], nil, true}, 1606 // Testing with prefix and truncation (27-28). 1607 {"test-bucket-list-object", "new", "", "", 1, resultCases[7], nil, true}, 1608 {"test-bucket-list-object", "obj", "", "", 2, resultCases[8], nil, true}, 1609 // Testing with marker, but without prefix and truncation (29-33). 1610 {"test-bucket-list-object", "", "newPrefix0", "", 6, resultCases[9], nil, true}, 1611 {"test-bucket-list-object", "", "newPrefix1", "", 5, resultCases[10], nil, true}, 1612 {"test-bucket-list-object", "", "obj0", "", 4, resultCases[11], nil, true}, 1613 {"test-bucket-list-object", "", "obj1", "", 2, resultCases[12], nil, true}, 1614 {"test-bucket-list-object", "", "man", "", 11, resultCases[13], nil, true}, 1615 // Marker being set to a value which is greater than and all object names when sorted (34). 1616 // Expected to send an empty response in this case. 1617 {"test-bucket-list-object", "", "zen", "", 10, ListObjectsInfo{}, nil, true}, 1618 // Marker being set to a value which is lesser than and all object names when sorted (35). 1619 // Expected to send all the objects in the bucket in this case. 1620 {"test-bucket-list-object", "", "Abc", "", 10, resultCases[14], nil, true}, 1621 // Marker is to a hierarhical value (36-37). 1622 {"test-bucket-list-object", "", "Asia/India/India-summer-photos-1", "", 10, resultCases[15], nil, true}, 1623 {"test-bucket-list-object", "", "Asia/India/Karnataka/Bangalore/Koramangala/pics", "", 10, resultCases[16], nil, true}, 1624 // Testing with marker and truncation, but no prefix (38-40). 1625 {"test-bucket-list-object", "", "newPrefix0", "", 3, resultCases[17], nil, true}, 1626 {"test-bucket-list-object", "", "newPrefix1", "", 1, resultCases[18], nil, true}, 1627 {"test-bucket-list-object", "", "obj0", "", 1, resultCases[19], nil, true}, 1628 // Testing with both marker and prefix, but without truncation (41-43). 1629 // The valid combination of marker and prefix should satisfy strings.HasPrefix(marker, prefix). 1630 {"test-bucket-list-object", "obj", "obj0", "", 2, resultCases[20], nil, true}, 1631 {"test-bucket-list-object", "obj", "obj1", "", 1, resultCases[21], nil, true}, 1632 {"test-bucket-list-object", "new", "newPrefix0", "", 2, resultCases[22], nil, true}, 1633 // Testing with maxKeys set to 0 (44-50). 1634 // The parameters have to valid. 1635 {"test-bucket-list-object", "", "obj1", "", 0, ListObjectsInfo{}, nil, true}, 1636 {"test-bucket-list-object", "", "obj0", "", 0, ListObjectsInfo{}, nil, true}, 1637 {"test-bucket-list-object", "new", "", "", 0, ListObjectsInfo{}, nil, true}, 1638 {"test-bucket-list-object", "obj", "", "", 0, ListObjectsInfo{}, nil, true}, 1639 {"test-bucket-list-object", "obj", "obj0", "", 0, ListObjectsInfo{}, nil, true}, 1640 {"test-bucket-list-object", "obj", "obj1", "", 0, ListObjectsInfo{}, nil, true}, 1641 {"test-bucket-list-object", "new", "newPrefix0", "", 0, ListObjectsInfo{}, nil, true}, 1642 // Tests on hierarchical key names as prefix. 1643 // Without delimteter the code should recurse into the prefix Dir. 1644 // Tests with prefix, but without delimiter (51-52). 1645 {"test-bucket-list-object", "Asia/India/", "", "", 10, resultCases[23], nil, true}, 1646 {"test-bucket-list-object", "Asia", "", "", 10, resultCases[24], nil, true}, 1647 // Tests with prefix and delimiter (53-55). 1648 // With delimiter the code should not recurse into the sub-directories of prefix Dir. 1649 {"test-bucket-list-object", "Asia", "", SlashSeparator, 10, resultCases[25], nil, true}, 1650 {"test-bucket-list-object", "new", "", SlashSeparator, 10, resultCases[26], nil, true}, 1651 {"test-bucket-list-object", "Asia/India/", "", SlashSeparator, 10, resultCases[27], nil, true}, 1652 // Test with marker set as hierarhical value and with delimiter. (56-57) 1653 {"test-bucket-list-object", "", "Asia/India/India-summer-photos-1", SlashSeparator, 10, resultCases[28], nil, true}, 1654 {"test-bucket-list-object", "", "Asia/India/Karnataka/Bangalore/Koramangala/pics", SlashSeparator, 10, resultCases[29], nil, true}, 1655 // Test with prefix and delimiter set to '/'. (58) 1656 {"test-bucket-list-object", SlashSeparator, "", SlashSeparator, 10, resultCases[30], nil, true}, 1657 // Test with invalid prefix (59) 1658 {"test-bucket-list-object", "\\", "", SlashSeparator, 10, ListObjectsInfo{}, nil, true}, 1659 // Test listing an empty directory in recursive mode (60) 1660 {"test-bucket-empty-dir", "", "", "", 10, resultCases[31], nil, true}, 1661 // Test listing an empty directory in a non recursive mode (61) 1662 {"test-bucket-empty-dir", "", "", SlashSeparator, 10, resultCases[32], nil, true}, 1663 // Test listing a directory which contains an empty directory (62) 1664 {"test-bucket-empty-dir", "", "temporary/", "", 10, resultCases[33], nil, true}, 1665 // Test listing with marker > last object such that response should be empty (63) 1666 {"test-bucket-single-object", "", "A/C", "", 1000, resultCases[34], nil, true}, 1667 // Test listing an object with a trailing slash and a slash delimiter (64) 1668 {"test-bucket-list-object", "Asia-maps.png/", "", "/", 1000, resultCases[34], nil, true}, 1669 // Test listing an object with uncommon delimiter 1670 {testBuckets[4], "", "", "guidSplunk", 1000, resultCases[35], nil, true}, 1671 // Test listing an object with uncommon delimiter and matching prefix 1672 {testBuckets[4], "file1/", "", "guidSplunk", 1000, resultCases[35], nil, true}, 1673 // Test listing at prefix with expected prefix markers 1674 {testBuckets[5], "dir/", "", SlashSeparator, 1, resultCases[36], nil, true}, 1675 {"test-bucket-list-object", "Asia", "A", "", 1, resultCases[4], nil, true}, 1676 } 1677 1678 for i, testCase := range testCases { 1679 testCase := testCase 1680 t.Run(fmt.Sprintf("%s-Test%d", instanceType, i+1), func(t *testing.T) { 1681 result, err := obj.ListObjectVersions(context.Background(), testCase.bucketName, 1682 testCase.prefix, testCase.marker, "", testCase.delimiter, int(testCase.maxKeys)) 1683 if err != nil && testCase.shouldPass { 1684 t.Errorf("%s: Expected to pass, but failed with: <ERROR> %s", instanceType, err.Error()) 1685 } 1686 if err == nil && !testCase.shouldPass { 1687 t.Errorf("%s: Expected to fail with <ERROR> \"%s\", but passed instead", instanceType, testCase.err.Error()) 1688 } 1689 // Failed as expected, but does it fail for the expected reason. 1690 if err != nil && !testCase.shouldPass { 1691 if !strings.Contains(err.Error(), testCase.err.Error()) { 1692 t.Errorf("%s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", instanceType, testCase.err.Error(), err.Error()) 1693 } 1694 } 1695 // Since there are cases for which ListObjects fails, this is 1696 // necessary. Test passes as expected, but the output values 1697 // are verified for correctness here. 1698 if err == nil && testCase.shouldPass { 1699 // The length of the expected ListObjectsResult.Objects 1700 // should match in both expected result from test cases 1701 // and in the output. On failure calling t.Fatalf, 1702 // otherwise it may lead to index out of range error in 1703 // assertion following this. 1704 if len(testCase.result.Objects) != len(result.Objects) { 1705 t.Fatalf("%s: Expected number of object in the result to be '%d', but found '%d' objects instead", instanceType, len(testCase.result.Objects), len(result.Objects)) 1706 } 1707 for j := 0; j < len(testCase.result.Objects); j++ { 1708 if testCase.result.Objects[j].Name != result.Objects[j].Name { 1709 t.Errorf("%s: Expected object name to be \"%s\", but found \"%s\" instead", instanceType, testCase.result.Objects[j].Name, result.Objects[j].Name) 1710 } 1711 } 1712 1713 if len(testCase.result.Prefixes) != len(result.Prefixes) { 1714 t.Log(testCase, testCase.result.Prefixes, result.Prefixes) 1715 t.Fatalf("%s: Expected number of prefixes in the result to be '%d', but found '%d' prefixes instead", instanceType, len(testCase.result.Prefixes), len(result.Prefixes)) 1716 } 1717 for j := 0; j < len(testCase.result.Prefixes); j++ { 1718 if testCase.result.Prefixes[j] != result.Prefixes[j] { 1719 t.Errorf("%s: Expected prefix name to be \"%s\", but found \"%s\" instead", instanceType, testCase.result.Prefixes[j], result.Prefixes[j]) 1720 } 1721 } 1722 1723 if testCase.result.IsTruncated != result.IsTruncated { 1724 // Allow an extra continuation token. 1725 if !result.IsTruncated || len(result.Objects) == 0 { 1726 t.Errorf("%s: Expected IsTruncated flag to be %v, but instead found it to be %v", instanceType, testCase.result.IsTruncated, result.IsTruncated) 1727 } 1728 } 1729 1730 if testCase.result.IsTruncated && result.NextMarker == "" { 1731 t.Errorf("%s: Expected NextMarker to contain a string since listing is truncated, but instead found it to be empty", instanceType) 1732 } 1733 1734 if !testCase.result.IsTruncated && result.NextMarker != "" { 1735 if !result.IsTruncated || len(result.Objects) == 0 { 1736 t.Errorf("%s: Expected NextMarker to be empty since listing is not truncated, but instead found `%v`", instanceType, result.NextMarker) 1737 } 1738 } 1739 } 1740 }) 1741 } 1742 } 1743 1744 // Wrapper for calling ListObjects continuation tests for both Erasure multiple disks and single node setup. 1745 func TestListObjectsContinuation(t *testing.T) { 1746 ExecObjectLayerTest(t, testListObjectsContinuation) 1747 } 1748 1749 // Unit test for ListObjects in general. 1750 func testListObjectsContinuation(obj ObjectLayer, instanceType string, t1 TestErrHandler) { 1751 t, _ := t1.(*testing.T) 1752 testBuckets := []string{ 1753 // This bucket is used for testing ListObject operations. 1754 "test-bucket-list-object-continuation-1", 1755 "test-bucket-list-object-continuation-2", 1756 } 1757 for _, bucket := range testBuckets { 1758 err := obj.MakeBucket(context.Background(), bucket, MakeBucketOptions{}) 1759 if err != nil { 1760 t.Fatalf("%s : %s", instanceType, err.Error()) 1761 } 1762 } 1763 1764 var err error 1765 testObjects := []struct { 1766 parentBucket string 1767 name string 1768 content string 1769 meta map[string]string 1770 }{ 1771 {testBuckets[0], "a/1.txt", "contentstring", nil}, 1772 {testBuckets[0], "a-1.txt", "contentstring", nil}, 1773 {testBuckets[0], "a.txt", "contentstring", nil}, 1774 {testBuckets[0], "apache2-doc/1.txt", "contentstring", nil}, 1775 {testBuckets[0], "apache2/1.txt", "contentstring", nil}, 1776 {testBuckets[0], "apache2/-sub/2.txt", "contentstring", nil}, 1777 {testBuckets[1], "azerty/1.txt", "contentstring", nil}, 1778 {testBuckets[1], "apache2-doc/1.txt", "contentstring", nil}, 1779 {testBuckets[1], "apache2/1.txt", "contentstring", nil}, 1780 } 1781 for _, object := range testObjects { 1782 md5Bytes := md5.Sum([]byte(object.content)) 1783 _, err = obj.PutObject(context.Background(), object.parentBucket, object.name, mustGetPutObjReader(t, bytes.NewBufferString(object.content), 1784 int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), ObjectOptions{UserDefined: object.meta}) 1785 if err != nil { 1786 t.Fatalf("%s : %s", instanceType, err.Error()) 1787 } 1788 1789 } 1790 1791 // Formulating the result data set to be expected from ListObjects call inside the tests, 1792 // This will be used in testCases and used for asserting the correctness of ListObjects output in the tests. 1793 1794 resultCases := []ListObjectsInfo{ 1795 { 1796 Objects: []ObjectInfo{ 1797 {Name: "a-1.txt"}, 1798 {Name: "a.txt"}, 1799 {Name: "a/1.txt"}, 1800 {Name: "apache2-doc/1.txt"}, 1801 {Name: "apache2/-sub/2.txt"}, 1802 {Name: "apache2/1.txt"}, 1803 }, 1804 }, 1805 { 1806 Objects: []ObjectInfo{ 1807 {Name: "apache2-doc/1.txt"}, 1808 {Name: "apache2/1.txt"}, 1809 }, 1810 }, 1811 { 1812 Prefixes: []string{"apache2-doc/", "apache2/", "azerty/"}, 1813 }, 1814 } 1815 1816 testCases := []struct { 1817 // Inputs to ListObjects. 1818 bucketName string 1819 prefix string 1820 delimiter string 1821 page int 1822 // Expected output of ListObjects. 1823 result ListObjectsInfo 1824 }{ 1825 {testBuckets[0], "", "", 1, resultCases[0]}, 1826 {testBuckets[0], "a", "", 1, resultCases[0]}, 1827 {testBuckets[1], "apache", "", 1, resultCases[1]}, 1828 {testBuckets[1], "", "/", 1, resultCases[2]}, 1829 } 1830 1831 for i, testCase := range testCases { 1832 testCase := testCase 1833 t.Run(fmt.Sprintf("%s-Test%d", instanceType, i+1), func(t *testing.T) { 1834 var foundObjects []ObjectInfo 1835 var foundPrefixes []string 1836 marker := "" 1837 for { 1838 result, err := obj.ListObjects(context.Background(), testCase.bucketName, 1839 testCase.prefix, marker, testCase.delimiter, testCase.page) 1840 if err != nil { 1841 t.Fatalf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, err.Error()) 1842 } 1843 foundObjects = append(foundObjects, result.Objects...) 1844 foundPrefixes = append(foundPrefixes, result.Prefixes...) 1845 if !result.IsTruncated { 1846 break 1847 } 1848 marker = result.NextMarker 1849 if len(result.Objects) > 0 { 1850 // Discard marker, so it cannot resume listing. 1851 marker = result.Objects[len(result.Objects)-1].Name 1852 } 1853 } 1854 1855 if len(testCase.result.Objects) != len(foundObjects) { 1856 t.Logf("want: %v", objInfoNames(testCase.result.Objects)) 1857 t.Logf("got: %v", objInfoNames(foundObjects)) 1858 t.Errorf("Test %d: %s: Expected number of objects in the result to be '%d', but found '%d' objects instead", 1859 i+1, instanceType, len(testCase.result.Objects), len(foundObjects)) 1860 } 1861 for j := 0; j < len(testCase.result.Objects); j++ { 1862 if j >= len(foundObjects) { 1863 t.Errorf("Test %d: %s: Expected object name to be \"%s\", but not nothing instead", i+1, instanceType, testCase.result.Objects[j].Name) 1864 continue 1865 } 1866 if testCase.result.Objects[j].Name != foundObjects[j].Name { 1867 t.Errorf("Test %d: %s: Expected object name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.result.Objects[j].Name, foundObjects[j].Name) 1868 } 1869 } 1870 1871 if len(testCase.result.Prefixes) != len(foundPrefixes) { 1872 t.Logf("want: %v", testCase.result.Prefixes) 1873 t.Logf("got: %v", foundPrefixes) 1874 t.Errorf("Test %d: %s: Expected number of prefixes in the result to be '%d', but found '%d' prefixes instead", 1875 i+1, instanceType, len(testCase.result.Prefixes), len(foundPrefixes)) 1876 } 1877 for j := 0; j < len(testCase.result.Prefixes); j++ { 1878 if j >= len(foundPrefixes) { 1879 t.Errorf("Test %d: %s: Expected prefix name to be \"%s\", but found no result", i+1, instanceType, testCase.result.Prefixes[j]) 1880 continue 1881 } 1882 if testCase.result.Prefixes[j] != foundPrefixes[j] { 1883 t.Errorf("Test %d: %s: Expected prefix name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.result.Prefixes[j], foundPrefixes[j]) 1884 } 1885 } 1886 }) 1887 } 1888 } 1889 1890 // Initialize FS backend for the benchmark. 1891 func initFSObjectsB(disk string, t *testing.B) (obj ObjectLayer) { 1892 obj, _, err := initObjectLayer(context.Background(), mustGetPoolEndpoints(0, disk)) 1893 if err != nil { 1894 t.Fatal(err) 1895 } 1896 1897 newTestConfig(globalMinioDefaultRegion, obj) 1898 1899 initAllSubsystems(GlobalContext) 1900 return obj 1901 } 1902 1903 // BenchmarkListObjects - Run ListObject Repeatedly and benchmark. 1904 func BenchmarkListObjects(b *testing.B) { 1905 // Make a temporary directory to use as the obj. 1906 directory := b.TempDir() 1907 1908 // Create the obj. 1909 obj := initFSObjectsB(directory, b) 1910 1911 bucket := "ls-benchmark-bucket" 1912 // Create a bucket. 1913 err := obj.MakeBucket(context.Background(), bucket, MakeBucketOptions{}) 1914 if err != nil { 1915 b.Fatal(err) 1916 } 1917 1918 // Insert objects to be listed and benchmarked later. 1919 for i := 0; i < 20000; i++ { 1920 key := "obj" + strconv.Itoa(i) 1921 _, err = obj.PutObject(context.Background(), bucket, key, mustGetPutObjReader(b, bytes.NewBufferString(key), int64(len(key)), "", ""), ObjectOptions{}) 1922 if err != nil { 1923 b.Fatal(err) 1924 } 1925 } 1926 1927 b.ResetTimer() 1928 1929 // List the buckets over and over and over. 1930 for i := 0; i < b.N; i++ { 1931 _, err = obj.ListObjects(context.Background(), bucket, "", "obj9000", "", -1) 1932 if err != nil { 1933 b.Fatal(err) 1934 } 1935 } 1936 }