storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/object-api-utils_test.go (about) 1 /* 2 * MinIO Cloud Storage, (C) 2016-2019 MinIO, Inc. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package cmd 18 19 import ( 20 "bytes" 21 "fmt" 22 "io" 23 "net/http" 24 "reflect" 25 "strconv" 26 "testing" 27 28 "github.com/klauspost/compress/s2" 29 30 "storj.io/minio/cmd/config/compress" 31 "storj.io/minio/cmd/crypto" 32 "storj.io/minio/pkg/trie" 33 ) 34 35 // Tests validate bucket name. 36 func TestIsValidBucketName(t *testing.T) { 37 testCases := []struct { 38 bucketName string 39 shouldPass bool 40 }{ 41 // cases which should pass the test. 42 // passing in valid bucket names. 43 {"lol", true}, 44 {"1-this-is-valid", true}, 45 {"1-this-too-is-valid-1", true}, 46 {"this.works.too.1", true}, 47 {"1234567", true}, 48 {"123", true}, 49 {"s3-eu-west-1.amazonaws.com", true}, 50 {"ideas-are-more-powerful-than-guns", true}, 51 {"testbucket", true}, 52 {"1bucket", true}, 53 {"bucket1", true}, 54 {"a.b", true}, 55 {"ab.a.bc", true}, 56 // cases for which test should fail. 57 // passing invalid bucket names. 58 {"------", false}, 59 {"my..bucket", false}, 60 {"192.168.1.1", false}, 61 {"$this-is-not-valid-too", false}, 62 {"contains-$-dollar", false}, 63 {"contains-^-carret", false}, 64 {"contains-$-dollar", false}, 65 {"contains-$-dollar", false}, 66 {"......", false}, 67 {"", false}, 68 {"a", false}, 69 {"ab", false}, 70 {".starts-with-a-dot", false}, 71 {"ends-with-a-dot.", false}, 72 {"ends-with-a-dash-", false}, 73 {"-starts-with-a-dash", false}, 74 {"THIS-BEGINS-WITH-UPPERCASe", false}, 75 {"tHIS-ENDS-WITH-UPPERCASE", false}, 76 {"ThisBeginsAndEndsWithUpperCasE", false}, 77 {"una ñina", false}, 78 {"dash-.may-not-appear-next-to-dot", false}, 79 {"dash.-may-not-appear-next-to-dot", false}, 80 {"dash-.-may-not-appear-next-to-dot", false}, 81 {"lalalallalallalalalallalallalala-thestring-size-is-greater-than-63", false}, 82 } 83 84 for i, testCase := range testCases { 85 isValidBucketName := IsValidBucketName(testCase.bucketName) 86 if testCase.shouldPass && !isValidBucketName { 87 t.Errorf("Test case %d: Expected \"%s\" to be a valid bucket name", i+1, testCase.bucketName) 88 } 89 if !testCase.shouldPass && isValidBucketName { 90 t.Errorf("Test case %d: Expected bucket name \"%s\" to be invalid", i+1, testCase.bucketName) 91 } 92 } 93 } 94 95 // Tests for validate object name. 96 func TestIsValidObjectName(t *testing.T) { 97 testCases := []struct { 98 objectName string 99 shouldPass bool 100 }{ 101 // cases which should pass the test. 102 // passing in valid object name. 103 {"object", true}, 104 {"The Shining Script <v1>.pdf", true}, 105 {"Cost Benefit Analysis (2009-2010).pptx", true}, 106 {"117Gn8rfHL2ACARPAhaFd0AGzic9pUbIA/5OCn5A", true}, 107 {"SHØRT", true}, 108 {"f*le", true}, 109 {"contains-^-carret", true}, 110 {"contains-|-pipe", true}, 111 {"contains-`-tick", true}, 112 {"..test", true}, 113 {".. test", true}, 114 {". test", true}, 115 {".test", true}, 116 {"There are far too many object names, and far too few bucket names!", true}, 117 {"!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~/!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~)", true}, 118 {"!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~", true}, 119 {"␀␁␂␃␄␅␆␇␈␉␊␋␌␍␎␏␐␑␒␓␔␕␖␗␘␙␚␛␜␝␞␟␡", true}, 120 {"trailing VT␋/trailing VT␋", true}, 121 {"␋leading VT/␋leading VT", true}, 122 {"~leading tilde", true}, 123 {"\rleading CR", true}, 124 {"\nleading LF", true}, 125 {"\tleading HT", true}, 126 {"trailing CR\r", true}, 127 {"trailing LF\n", true}, 128 {"trailing HT\t", true}, 129 // cases for which test should fail. 130 // passing invalid object names. 131 {"", false}, 132 {"a/b/c/", false}, 133 {"../../etc", false}, 134 {"../../", false}, 135 {"/../../etc", false}, 136 {" ../etc", false}, 137 {"./././", false}, 138 {"./etc", false}, 139 {`contains//double/forwardslash`, false}, 140 {`//contains/double-forwardslash-prefix`, false}, 141 {string([]byte{0xff, 0xfe, 0xfd}), false}, 142 } 143 144 for i, testCase := range testCases { 145 isValidObjectName := IsValidObjectName(testCase.objectName) 146 if testCase.shouldPass && !isValidObjectName { 147 t.Errorf("Test case %d: Expected \"%s\" to be a valid object name", i+1, testCase.objectName) 148 } 149 if !testCase.shouldPass && isValidObjectName { 150 t.Errorf("Test case %d: Expected object name \"%s\" to be invalid", i+1, testCase.objectName) 151 } 152 } 153 } 154 155 // Tests getCompleteMultipartMD5 156 func TestGetCompleteMultipartMD5(t *testing.T) { 157 testCases := []struct { 158 parts []CompletePart 159 expectedResult string 160 expectedErr string 161 }{ 162 // Wrong MD5 hash string, returns md5um of hash 163 {[]CompletePart{{ETag: "wrong-md5-hash-string"}}, "0deb8cb07527b4b2669c861cb9653607-1", ""}, 164 165 // Single CompletePart with valid MD5 hash string. 166 {[]CompletePart{{ETag: "cf1f738a5924e645913c984e0fe3d708"}}, "10dc1617fbcf0bd0858048cb96e6bd77-1", ""}, 167 168 // Multiple CompletePart with valid MD5 hash string. 169 {[]CompletePart{{ETag: "cf1f738a5924e645913c984e0fe3d708"}, {ETag: "9ccbc9a80eee7fb6fdd22441db2aedbd"}}, "0239a86b5266bb624f0ac60ba2aed6c8-2", ""}, 170 } 171 172 for i, test := range testCases { 173 result := getCompleteMultipartMD5(test.parts) 174 if result != test.expectedResult { 175 t.Fatalf("test %d failed: expected: result=%v, got=%v", i+1, test.expectedResult, result) 176 } 177 } 178 } 179 180 // TestIsMinioBucketName - Tests isMinioBucketName helper function. 181 func TestIsMinioMetaBucketName(t *testing.T) { 182 testCases := []struct { 183 bucket string 184 result bool 185 }{ 186 // MinIO meta bucket. 187 { 188 bucket: minioMetaBucket, 189 result: true, 190 }, 191 // MinIO meta bucket. 192 { 193 bucket: minioMetaMultipartBucket, 194 result: true, 195 }, 196 // MinIO meta bucket. 197 { 198 bucket: minioMetaTmpBucket, 199 result: true, 200 }, 201 // Normal bucket 202 { 203 bucket: "mybucket", 204 result: false, 205 }, 206 } 207 208 for i, test := range testCases { 209 actual := isMinioMetaBucketName(test.bucket) 210 if actual != test.result { 211 t.Errorf("Test %d - expected %v but received %v", 212 i+1, test.result, actual) 213 } 214 } 215 } 216 217 // Tests RemoveStandardStorageClass method. Expectation is metadata map 218 // should be cleared of x-amz-storage-class, if it is set to STANDARD 219 func TestRemoveStandardStorageClass(t *testing.T) { 220 tests := []struct { 221 name string 222 metadata map[string]string 223 want map[string]string 224 }{ 225 { 226 name: "1", 227 metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "STANDARD"}, 228 want: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86"}, 229 }, 230 { 231 name: "2", 232 metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "REDUCED_REDUNDANCY"}, 233 want: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "REDUCED_REDUNDANCY"}, 234 }, 235 { 236 name: "3", 237 metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86"}, 238 want: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86"}, 239 }, 240 } 241 for _, tt := range tests { 242 if got := removeStandardStorageClass(tt.metadata); !reflect.DeepEqual(got, tt.want) { 243 t.Errorf("Test %s failed, expected %v, got %v", tt.name, tt.want, got) 244 } 245 } 246 } 247 248 // Tests CleanMetadata method. Expectation is metadata map 249 // should be cleared of etag, md5Sum and x-amz-storage-class, if it is set to STANDARD 250 func TestCleanMetadata(t *testing.T) { 251 tests := []struct { 252 name string 253 metadata map[string]string 254 want map[string]string 255 }{ 256 { 257 name: "1", 258 metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "STANDARD"}, 259 want: map[string]string{"content-type": "application/octet-stream"}, 260 }, 261 { 262 name: "2", 263 metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "REDUCED_REDUNDANCY"}, 264 want: map[string]string{"content-type": "application/octet-stream", "x-amz-storage-class": "REDUCED_REDUNDANCY"}, 265 }, 266 { 267 name: "3", 268 metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "md5Sum": "abcde"}, 269 want: map[string]string{"content-type": "application/octet-stream"}, 270 }, 271 } 272 for _, tt := range tests { 273 if got := cleanMetadata(tt.metadata); !reflect.DeepEqual(got, tt.want) { 274 t.Errorf("Test %s failed, expected %v, got %v", tt.name, tt.want, got) 275 } 276 } 277 } 278 279 // Tests CleanMetadataKeys method. Expectation is metadata map 280 // should be cleared of keys passed to CleanMetadataKeys method 281 func TestCleanMetadataKeys(t *testing.T) { 282 tests := []struct { 283 name string 284 metadata map[string]string 285 keys []string 286 want map[string]string 287 }{ 288 { 289 name: "1", 290 metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "STANDARD", "md5": "abcde"}, 291 keys: []string{"etag", "md5"}, 292 want: map[string]string{"content-type": "application/octet-stream", "x-amz-storage-class": "STANDARD"}, 293 }, 294 { 295 name: "2", 296 metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "REDUCED_REDUNDANCY", "md5sum": "abcde"}, 297 keys: []string{"etag", "md5sum"}, 298 want: map[string]string{"content-type": "application/octet-stream", "x-amz-storage-class": "REDUCED_REDUNDANCY"}, 299 }, 300 { 301 name: "3", 302 metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "xyz": "abcde"}, 303 keys: []string{"etag", "xyz"}, 304 want: map[string]string{"content-type": "application/octet-stream"}, 305 }, 306 } 307 for _, tt := range tests { 308 if got := cleanMetadataKeys(tt.metadata, tt.keys...); !reflect.DeepEqual(got, tt.want) { 309 t.Errorf("Test %s failed, expected %v, got %v", tt.name, tt.want, got) 310 } 311 } 312 } 313 314 // Tests isCompressed method 315 func TestIsCompressed(t *testing.T) { 316 testCases := []struct { 317 objInfo ObjectInfo 318 result bool 319 err bool 320 }{ 321 0: { 322 objInfo: ObjectInfo{ 323 UserDefined: map[string]string{"X-Minio-Internal-compression": compressionAlgorithmV1, 324 "content-type": "application/octet-stream", 325 "etag": "b3ff3ef3789147152fbfbc50efba4bfd-2"}, 326 }, 327 result: true, 328 }, 329 1: { 330 objInfo: ObjectInfo{ 331 UserDefined: map[string]string{"X-Minio-Internal-compression": compressionAlgorithmV2, 332 "content-type": "application/octet-stream", 333 "etag": "b3ff3ef3789147152fbfbc50efba4bfd-2"}, 334 }, 335 result: true, 336 }, 337 2: { 338 objInfo: ObjectInfo{ 339 UserDefined: map[string]string{"X-Minio-Internal-compression": "unknown/compression/type", 340 "content-type": "application/octet-stream", 341 "etag": "b3ff3ef3789147152fbfbc50efba4bfd-2"}, 342 }, 343 result: true, 344 err: true, 345 }, 346 3: { 347 objInfo: ObjectInfo{ 348 UserDefined: map[string]string{"X-Minio-Internal-compression": compressionAlgorithmV2, 349 "content-type": "application/octet-stream", 350 "etag": "b3ff3ef3789147152fbfbc50efba4bfd-2", 351 crypto.MetaIV: "yes", 352 }, 353 }, 354 result: true, 355 err: false, 356 }, 357 4: { 358 objInfo: ObjectInfo{ 359 UserDefined: map[string]string{"X-Minio-Internal-XYZ": "klauspost/compress/s2", 360 "content-type": "application/octet-stream", 361 "etag": "b3ff3ef3789147152fbfbc50efba4bfd-2"}, 362 }, 363 result: false, 364 }, 365 5: { 366 objInfo: ObjectInfo{ 367 UserDefined: map[string]string{"content-type": "application/octet-stream", 368 "etag": "b3ff3ef3789147152fbfbc50efba4bfd-2"}, 369 }, 370 result: false, 371 }, 372 } 373 for i, test := range testCases { 374 t.Run(strconv.Itoa(i), func(t *testing.T) { 375 got := test.objInfo.IsCompressed() 376 if got != test.result { 377 t.Errorf("IsCompressed: Expected %v but received %v", 378 test.result, got) 379 } 380 got, gErr := test.objInfo.IsCompressedOK() 381 if got != test.result { 382 t.Errorf("IsCompressedOK: Expected %v but received %v", 383 test.result, got) 384 } 385 if gErr != nil != test.err { 386 t.Errorf("IsCompressedOK: want error: %t, got error: %v", test.err, gErr) 387 } 388 }) 389 } 390 } 391 392 // Tests excludeForCompression. 393 func TestExcludeForCompression(t *testing.T) { 394 testCases := []struct { 395 object string 396 header http.Header 397 result bool 398 }{ 399 { 400 object: "object.txt", 401 header: http.Header{ 402 "Content-Type": []string{"application/zip"}, 403 }, 404 result: true, 405 }, 406 { 407 object: "object.zip", 408 header: http.Header{ 409 "Content-Type": []string{"application/XYZ"}, 410 }, 411 result: true, 412 }, 413 { 414 object: "object.json", 415 header: http.Header{ 416 "Content-Type": []string{"application/json"}, 417 }, 418 result: false, 419 }, 420 { 421 object: "object.txt", 422 header: http.Header{ 423 "Content-Type": []string{"text/plain"}, 424 }, 425 result: false, 426 }, 427 { 428 object: "object", 429 header: http.Header{ 430 "Content-Type": []string{"text/something"}, 431 }, 432 result: false, 433 }, 434 } 435 for i, test := range testCases { 436 got := excludeForCompression(test.header, test.object, compress.Config{ 437 Enabled: true, 438 }) 439 if got != test.result { 440 t.Errorf("Test %d - expected %v but received %v", 441 i+1, test.result, got) 442 } 443 } 444 } 445 446 func BenchmarkGetPartFileWithTrie(b *testing.B) { 447 b.ResetTimer() 448 449 entriesTrie := trie.NewTrie() 450 for i := 1; i <= 10000; i++ { 451 entriesTrie.Insert(fmt.Sprintf("%.5d.8a034f82cb9cb31140d87d3ce2a9ede3.67108864", i)) 452 } 453 454 for i := 1; i <= 10000; i++ { 455 partFile := getPartFile(entriesTrie, i, "8a034f82cb9cb31140d87d3ce2a9ede3") 456 if partFile == "" { 457 b.Fatal("partFile returned is empty") 458 } 459 } 460 461 b.ReportAllocs() 462 } 463 464 func TestGetActualSize(t *testing.T) { 465 testCases := []struct { 466 objInfo ObjectInfo 467 result int64 468 }{ 469 { 470 objInfo: ObjectInfo{ 471 UserDefined: map[string]string{"X-Minio-Internal-compression": "klauspost/compress/s2", 472 "X-Minio-Internal-actual-size": "100000001", 473 "content-type": "application/octet-stream", 474 "etag": "b3ff3ef3789147152fbfbc50efba4bfd-2"}, 475 Parts: []ObjectPartInfo{ 476 { 477 Size: 39235668, 478 ActualSize: 67108864, 479 }, 480 { 481 Size: 19177372, 482 ActualSize: 32891137, 483 }, 484 }, 485 }, 486 result: 100000001, 487 }, 488 { 489 objInfo: ObjectInfo{ 490 UserDefined: map[string]string{"X-Minio-Internal-compression": "klauspost/compress/s2", 491 "X-Minio-Internal-actual-size": "841", 492 "content-type": "application/octet-stream", 493 "etag": "b3ff3ef3789147152fbfbc50efba4bfd-2"}, 494 Parts: []ObjectPartInfo{}, 495 }, 496 result: 841, 497 }, 498 { 499 objInfo: ObjectInfo{ 500 UserDefined: map[string]string{"X-Minio-Internal-compression": "klauspost/compress/s2", 501 "content-type": "application/octet-stream", 502 "etag": "b3ff3ef3789147152fbfbc50efba4bfd-2"}, 503 Parts: []ObjectPartInfo{}, 504 }, 505 result: -1, 506 }, 507 } 508 for i, test := range testCases { 509 got, _ := test.objInfo.GetActualSize() 510 if got != test.result { 511 t.Errorf("Test %d - expected %d but received %d", 512 i+1, test.result, got) 513 } 514 } 515 } 516 517 func TestGetCompressedOffsets(t *testing.T) { 518 testCases := []struct { 519 objInfo ObjectInfo 520 offset int64 521 startOffset int64 522 snappyStartOffset int64 523 firstPart int 524 }{ 525 0: { 526 objInfo: ObjectInfo{ 527 Parts: []ObjectPartInfo{ 528 { 529 Size: 39235668, 530 ActualSize: 67108864, 531 }, 532 { 533 Size: 19177372, 534 ActualSize: 32891137, 535 }, 536 }, 537 }, 538 offset: 79109865, 539 startOffset: 39235668, 540 snappyStartOffset: 12001001, 541 firstPart: 1, 542 }, 543 1: { 544 objInfo: ObjectInfo{ 545 Parts: []ObjectPartInfo{ 546 { 547 Size: 39235668, 548 ActualSize: 67108864, 549 }, 550 { 551 Size: 19177372, 552 ActualSize: 32891137, 553 }, 554 }, 555 }, 556 offset: 19109865, 557 startOffset: 0, 558 snappyStartOffset: 19109865, 559 }, 560 2: { 561 objInfo: ObjectInfo{ 562 Parts: []ObjectPartInfo{ 563 { 564 Size: 39235668, 565 ActualSize: 67108864, 566 }, 567 { 568 Size: 19177372, 569 ActualSize: 32891137, 570 }, 571 }, 572 }, 573 offset: 0, 574 startOffset: 0, 575 snappyStartOffset: 0, 576 }, 577 } 578 for i, test := range testCases { 579 startOffset, snappyStartOffset, firstPart := getCompressedOffsets(test.objInfo, test.offset) 580 if startOffset != test.startOffset { 581 t.Errorf("Test %d - expected startOffset %d but received %d", 582 i, test.startOffset, startOffset) 583 } 584 if snappyStartOffset != test.snappyStartOffset { 585 t.Errorf("Test %d - expected snappyOffset %d but received %d", 586 i, test.snappyStartOffset, snappyStartOffset) 587 } 588 if firstPart != test.firstPart { 589 t.Errorf("Test %d - expected firstPart %d but received %d", 590 i, test.firstPart, firstPart) 591 } 592 } 593 } 594 595 func TestS2CompressReader(t *testing.T) { 596 tests := []struct { 597 name string 598 data []byte 599 }{ 600 {name: "empty", data: nil}, 601 {name: "small", data: []byte("hello, world")}, 602 {name: "large", data: bytes.Repeat([]byte("hello, world"), 1000)}, 603 } 604 605 for _, tt := range tests { 606 t.Run(tt.name, func(t *testing.T) { 607 buf := make([]byte, 100) // make small buffer to ensure multiple reads are required for large case 608 609 r := newS2CompressReader(bytes.NewReader(tt.data), int64(len(tt.data))) 610 defer r.Close() 611 612 var rdrBuf bytes.Buffer 613 _, err := io.CopyBuffer(&rdrBuf, r, buf) 614 if err != nil { 615 t.Fatal(err) 616 } 617 618 var stdBuf bytes.Buffer 619 w := s2.NewWriter(&stdBuf) 620 _, err = io.CopyBuffer(w, bytes.NewReader(tt.data), buf) 621 if err != nil { 622 t.Fatal(err) 623 } 624 err = w.Close() 625 if err != nil { 626 t.Fatal(err) 627 } 628 629 var ( 630 got = rdrBuf.Bytes() 631 want = stdBuf.Bytes() 632 ) 633 if !bytes.Equal(got, want) { 634 t.Errorf("encoded data does not match\n\t%q\n\t%q", got, want) 635 } 636 637 var decBuf bytes.Buffer 638 decRdr := s2.NewReader(&rdrBuf) 639 _, err = io.Copy(&decBuf, decRdr) 640 if err != nil { 641 t.Fatal(err) 642 } 643 644 if !bytes.Equal(tt.data, decBuf.Bytes()) { 645 t.Errorf("roundtrip failed\n\t%q\n\t%q", tt.data, decBuf.Bytes()) 646 } 647 }) 648 } 649 }