github.com/as-polyakov/minio@v0.1.0-cvefix.3/cmd/object-api-utils_test.go (about) 1 /* 2 * MinIO Cloud Storage, (C) 2016-2019 MinIO, Inc. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package cmd 18 19 import ( 20 "bytes" 21 "fmt" 22 "io" 23 "net/http" 24 "reflect" 25 "strconv" 26 "testing" 27 28 "github.com/klauspost/compress/s2" 29 "github.com/as-polyakov/minio/cmd/config/compress" 30 "github.com/as-polyakov/minio/cmd/crypto" 31 "github.com/as-polyakov/minio/pkg/trie" 32 ) 33 34 // Tests validate bucket name. 35 func TestIsValidBucketName(t *testing.T) { 36 testCases := []struct { 37 bucketName string 38 shouldPass bool 39 }{ 40 // cases which should pass the test. 41 // passing in valid bucket names. 42 {"lol", true}, 43 {"1-this-is-valid", true}, 44 {"1-this-too-is-valid-1", true}, 45 {"this.works.too.1", true}, 46 {"1234567", true}, 47 {"123", true}, 48 {"s3-eu-west-1.amazonaws.com", true}, 49 {"ideas-are-more-powerful-than-guns", true}, 50 {"testbucket", true}, 51 {"1bucket", true}, 52 {"bucket1", true}, 53 {"a.b", true}, 54 {"ab.a.bc", true}, 55 // cases for which test should fail. 56 // passing invalid bucket names. 57 {"------", false}, 58 {"my..bucket", false}, 59 {"192.168.1.1", false}, 60 {"$this-is-not-valid-too", false}, 61 {"contains-$-dollar", false}, 62 {"contains-^-carret", false}, 63 {"contains-$-dollar", false}, 64 {"contains-$-dollar", false}, 65 {"......", false}, 66 {"", false}, 67 {"a", false}, 68 {"ab", false}, 69 {".starts-with-a-dot", false}, 70 {"ends-with-a-dot.", false}, 71 {"ends-with-a-dash-", false}, 72 {"-starts-with-a-dash", false}, 73 {"THIS-BEGINS-WITH-UPPERCASe", false}, 74 {"tHIS-ENDS-WITH-UPPERCASE", false}, 75 {"ThisBeginsAndEndsWithUpperCasE", false}, 76 {"una ñina", false}, 77 {"dash-.may-not-appear-next-to-dot", false}, 78 {"dash.-may-not-appear-next-to-dot", false}, 79 {"dash-.-may-not-appear-next-to-dot", false}, 80 {"lalalallalallalalalallalallalala-thestring-size-is-greater-than-63", false}, 81 } 82 83 for i, testCase := range testCases { 84 isValidBucketName := IsValidBucketName(testCase.bucketName) 85 if testCase.shouldPass && !isValidBucketName { 86 t.Errorf("Test case %d: Expected \"%s\" to be a valid bucket name", i+1, testCase.bucketName) 87 } 88 if !testCase.shouldPass && isValidBucketName { 89 t.Errorf("Test case %d: Expected bucket name \"%s\" to be invalid", i+1, testCase.bucketName) 90 } 91 } 92 } 93 94 // Tests for validate object name. 95 func TestIsValidObjectName(t *testing.T) { 96 testCases := []struct { 97 objectName string 98 shouldPass bool 99 }{ 100 // cases which should pass the test. 101 // passing in valid object name. 102 {"object", true}, 103 {"The Shining Script <v1>.pdf", true}, 104 {"Cost Benefit Analysis (2009-2010).pptx", true}, 105 {"117Gn8rfHL2ACARPAhaFd0AGzic9pUbIA/5OCn5A", true}, 106 {"SHØRT", true}, 107 {"f*le", true}, 108 {"contains-^-carret", true}, 109 {"contains-|-pipe", true}, 110 {"contains-`-tick", true}, 111 {"..test", true}, 112 {".. test", true}, 113 {". test", true}, 114 {".test", true}, 115 {"There are far too many object names, and far too few bucket names!", true}, 116 {"!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~/!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~)", true}, 117 {"!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~", true}, 118 {"␀␁␂␃␄␅␆␇␈␉␊␋␌␍␎␏␐␑␒␓␔␕␖␗␘␙␚␛␜␝␞␟␡", true}, 119 {"trailing VT␋/trailing VT␋", true}, 120 {"␋leading VT/␋leading VT", true}, 121 {"~leading tilde", true}, 122 {"\rleading CR", true}, 123 {"\nleading LF", true}, 124 {"\tleading HT", true}, 125 {"trailing CR\r", true}, 126 {"trailing LF\n", true}, 127 {"trailing HT\t", true}, 128 // cases for which test should fail. 129 // passing invalid object names. 130 {"", false}, 131 {"a/b/c/", false}, 132 {"../../etc", false}, 133 {"../../", false}, 134 {"/../../etc", false}, 135 {" ../etc", false}, 136 {"./././", false}, 137 {"./etc", false}, 138 {`contains//double/forwardslash`, false}, 139 {`//contains/double-forwardslash-prefix`, false}, 140 {string([]byte{0xff, 0xfe, 0xfd}), false}, 141 } 142 143 for i, testCase := range testCases { 144 isValidObjectName := IsValidObjectName(testCase.objectName) 145 if testCase.shouldPass && !isValidObjectName { 146 t.Errorf("Test case %d: Expected \"%s\" to be a valid object name", i+1, testCase.objectName) 147 } 148 if !testCase.shouldPass && isValidObjectName { 149 t.Errorf("Test case %d: Expected object name \"%s\" to be invalid", i+1, testCase.objectName) 150 } 151 } 152 } 153 154 // Tests getCompleteMultipartMD5 155 func TestGetCompleteMultipartMD5(t *testing.T) { 156 testCases := []struct { 157 parts []CompletePart 158 expectedResult string 159 expectedErr string 160 }{ 161 // Wrong MD5 hash string, returns md5um of hash 162 {[]CompletePart{{ETag: "wrong-md5-hash-string"}}, "0deb8cb07527b4b2669c861cb9653607-1", ""}, 163 164 // Single CompletePart with valid MD5 hash string. 165 {[]CompletePart{{ETag: "cf1f738a5924e645913c984e0fe3d708"}}, "10dc1617fbcf0bd0858048cb96e6bd77-1", ""}, 166 167 // Multiple CompletePart with valid MD5 hash string. 168 {[]CompletePart{{ETag: "cf1f738a5924e645913c984e0fe3d708"}, {ETag: "9ccbc9a80eee7fb6fdd22441db2aedbd"}}, "0239a86b5266bb624f0ac60ba2aed6c8-2", ""}, 169 } 170 171 for i, test := range testCases { 172 result := getCompleteMultipartMD5(test.parts) 173 if result != test.expectedResult { 174 t.Fatalf("test %d failed: expected: result=%v, got=%v", i+1, test.expectedResult, result) 175 } 176 } 177 } 178 179 // TestIsMinioBucketName - Tests isMinioBucketName helper function. 180 func TestIsMinioMetaBucketName(t *testing.T) { 181 testCases := []struct { 182 bucket string 183 result bool 184 }{ 185 // MinIO meta bucket. 186 { 187 bucket: minioMetaBucket, 188 result: true, 189 }, 190 // MinIO meta bucket. 191 { 192 bucket: minioMetaMultipartBucket, 193 result: true, 194 }, 195 // MinIO meta bucket. 196 { 197 bucket: minioMetaTmpBucket, 198 result: true, 199 }, 200 // Normal bucket 201 { 202 bucket: "mybucket", 203 result: false, 204 }, 205 } 206 207 for i, test := range testCases { 208 actual := isMinioMetaBucketName(test.bucket) 209 if actual != test.result { 210 t.Errorf("Test %d - expected %v but received %v", 211 i+1, test.result, actual) 212 } 213 } 214 } 215 216 // Tests RemoveStandardStorageClass method. Expectation is metadata map 217 // should be cleared of x-amz-storage-class, if it is set to STANDARD 218 func TestRemoveStandardStorageClass(t *testing.T) { 219 tests := []struct { 220 name string 221 metadata map[string]string 222 want map[string]string 223 }{ 224 { 225 name: "1", 226 metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "STANDARD"}, 227 want: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86"}, 228 }, 229 { 230 name: "2", 231 metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "REDUCED_REDUNDANCY"}, 232 want: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "REDUCED_REDUNDANCY"}, 233 }, 234 { 235 name: "3", 236 metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86"}, 237 want: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86"}, 238 }, 239 } 240 for _, tt := range tests { 241 if got := removeStandardStorageClass(tt.metadata); !reflect.DeepEqual(got, tt.want) { 242 t.Errorf("Test %s failed, expected %v, got %v", tt.name, tt.want, got) 243 } 244 } 245 } 246 247 // Tests CleanMetadata method. Expectation is metadata map 248 // should be cleared of etag, md5Sum and x-amz-storage-class, if it is set to STANDARD 249 func TestCleanMetadata(t *testing.T) { 250 tests := []struct { 251 name string 252 metadata map[string]string 253 want map[string]string 254 }{ 255 { 256 name: "1", 257 metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "STANDARD"}, 258 want: map[string]string{"content-type": "application/octet-stream"}, 259 }, 260 { 261 name: "2", 262 metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "REDUCED_REDUNDANCY"}, 263 want: map[string]string{"content-type": "application/octet-stream", "x-amz-storage-class": "REDUCED_REDUNDANCY"}, 264 }, 265 { 266 name: "3", 267 metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "md5Sum": "abcde"}, 268 want: map[string]string{"content-type": "application/octet-stream"}, 269 }, 270 } 271 for _, tt := range tests { 272 if got := cleanMetadata(tt.metadata); !reflect.DeepEqual(got, tt.want) { 273 t.Errorf("Test %s failed, expected %v, got %v", tt.name, tt.want, got) 274 } 275 } 276 } 277 278 // Tests CleanMetadataKeys method. Expectation is metadata map 279 // should be cleared of keys passed to CleanMetadataKeys method 280 func TestCleanMetadataKeys(t *testing.T) { 281 tests := []struct { 282 name string 283 metadata map[string]string 284 keys []string 285 want map[string]string 286 }{ 287 { 288 name: "1", 289 metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "STANDARD", "md5": "abcde"}, 290 keys: []string{"etag", "md5"}, 291 want: map[string]string{"content-type": "application/octet-stream", "x-amz-storage-class": "STANDARD"}, 292 }, 293 { 294 name: "2", 295 metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "REDUCED_REDUNDANCY", "md5sum": "abcde"}, 296 keys: []string{"etag", "md5sum"}, 297 want: map[string]string{"content-type": "application/octet-stream", "x-amz-storage-class": "REDUCED_REDUNDANCY"}, 298 }, 299 { 300 name: "3", 301 metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "xyz": "abcde"}, 302 keys: []string{"etag", "xyz"}, 303 want: map[string]string{"content-type": "application/octet-stream"}, 304 }, 305 } 306 for _, tt := range tests { 307 if got := cleanMetadataKeys(tt.metadata, tt.keys...); !reflect.DeepEqual(got, tt.want) { 308 t.Errorf("Test %s failed, expected %v, got %v", tt.name, tt.want, got) 309 } 310 } 311 } 312 313 // Tests isCompressed method 314 func TestIsCompressed(t *testing.T) { 315 testCases := []struct { 316 objInfo ObjectInfo 317 result bool 318 err bool 319 }{ 320 0: { 321 objInfo: ObjectInfo{ 322 UserDefined: map[string]string{"X-Minio-Internal-compression": compressionAlgorithmV1, 323 "content-type": "application/octet-stream", 324 "etag": "b3ff3ef3789147152fbfbc50efba4bfd-2"}, 325 }, 326 result: true, 327 }, 328 1: { 329 objInfo: ObjectInfo{ 330 UserDefined: map[string]string{"X-Minio-Internal-compression": compressionAlgorithmV2, 331 "content-type": "application/octet-stream", 332 "etag": "b3ff3ef3789147152fbfbc50efba4bfd-2"}, 333 }, 334 result: true, 335 }, 336 2: { 337 objInfo: ObjectInfo{ 338 UserDefined: map[string]string{"X-Minio-Internal-compression": "unknown/compression/type", 339 "content-type": "application/octet-stream", 340 "etag": "b3ff3ef3789147152fbfbc50efba4bfd-2"}, 341 }, 342 result: true, 343 err: true, 344 }, 345 3: { 346 objInfo: ObjectInfo{ 347 UserDefined: map[string]string{"X-Minio-Internal-compression": compressionAlgorithmV2, 348 "content-type": "application/octet-stream", 349 "etag": "b3ff3ef3789147152fbfbc50efba4bfd-2", 350 crypto.MetaIV: "yes", 351 }, 352 }, 353 result: true, 354 err: false, 355 }, 356 4: { 357 objInfo: ObjectInfo{ 358 UserDefined: map[string]string{"X-Minio-Internal-XYZ": "klauspost/compress/s2", 359 "content-type": "application/octet-stream", 360 "etag": "b3ff3ef3789147152fbfbc50efba4bfd-2"}, 361 }, 362 result: false, 363 }, 364 5: { 365 objInfo: ObjectInfo{ 366 UserDefined: map[string]string{"content-type": "application/octet-stream", 367 "etag": "b3ff3ef3789147152fbfbc50efba4bfd-2"}, 368 }, 369 result: false, 370 }, 371 } 372 for i, test := range testCases { 373 t.Run(strconv.Itoa(i), func(t *testing.T) { 374 got := test.objInfo.IsCompressed() 375 if got != test.result { 376 t.Errorf("IsCompressed: Expected %v but received %v", 377 test.result, got) 378 } 379 got, gErr := test.objInfo.IsCompressedOK() 380 if got != test.result { 381 t.Errorf("IsCompressedOK: Expected %v but received %v", 382 test.result, got) 383 } 384 if gErr != nil != test.err { 385 t.Errorf("IsCompressedOK: want error: %t, got error: %v", test.err, gErr) 386 } 387 }) 388 } 389 } 390 391 // Tests excludeForCompression. 392 func TestExcludeForCompression(t *testing.T) { 393 testCases := []struct { 394 object string 395 header http.Header 396 result bool 397 }{ 398 { 399 object: "object.txt", 400 header: http.Header{ 401 "Content-Type": []string{"application/zip"}, 402 }, 403 result: true, 404 }, 405 { 406 object: "object.zip", 407 header: http.Header{ 408 "Content-Type": []string{"application/XYZ"}, 409 }, 410 result: true, 411 }, 412 { 413 object: "object.json", 414 header: http.Header{ 415 "Content-Type": []string{"application/json"}, 416 }, 417 result: false, 418 }, 419 { 420 object: "object.txt", 421 header: http.Header{ 422 "Content-Type": []string{"text/plain"}, 423 }, 424 result: false, 425 }, 426 { 427 object: "object", 428 header: http.Header{ 429 "Content-Type": []string{"text/something"}, 430 }, 431 result: false, 432 }, 433 } 434 for i, test := range testCases { 435 got := excludeForCompression(test.header, test.object, compress.Config{ 436 Enabled: true, 437 }) 438 if got != test.result { 439 t.Errorf("Test %d - expected %v but received %v", 440 i+1, test.result, got) 441 } 442 } 443 } 444 445 func BenchmarkGetPartFileWithTrie(b *testing.B) { 446 b.ResetTimer() 447 448 entriesTrie := trie.NewTrie() 449 for i := 1; i <= 10000; i++ { 450 entriesTrie.Insert(fmt.Sprintf("%.5d.8a034f82cb9cb31140d87d3ce2a9ede3.67108864", i)) 451 } 452 453 for i := 1; i <= 10000; i++ { 454 partFile := getPartFile(entriesTrie, i, "8a034f82cb9cb31140d87d3ce2a9ede3") 455 if partFile == "" { 456 b.Fatal("partFile returned is empty") 457 } 458 } 459 460 b.ReportAllocs() 461 } 462 463 func TestGetActualSize(t *testing.T) { 464 testCases := []struct { 465 objInfo ObjectInfo 466 result int64 467 }{ 468 { 469 objInfo: ObjectInfo{ 470 UserDefined: map[string]string{"X-Minio-Internal-compression": "klauspost/compress/s2", 471 "X-Minio-Internal-actual-size": "100000001", 472 "content-type": "application/octet-stream", 473 "etag": "b3ff3ef3789147152fbfbc50efba4bfd-2"}, 474 Parts: []ObjectPartInfo{ 475 { 476 Size: 39235668, 477 ActualSize: 67108864, 478 }, 479 { 480 Size: 19177372, 481 ActualSize: 32891137, 482 }, 483 }, 484 }, 485 result: 100000001, 486 }, 487 { 488 objInfo: ObjectInfo{ 489 UserDefined: map[string]string{"X-Minio-Internal-compression": "klauspost/compress/s2", 490 "X-Minio-Internal-actual-size": "841", 491 "content-type": "application/octet-stream", 492 "etag": "b3ff3ef3789147152fbfbc50efba4bfd-2"}, 493 Parts: []ObjectPartInfo{}, 494 }, 495 result: 841, 496 }, 497 { 498 objInfo: ObjectInfo{ 499 UserDefined: map[string]string{"X-Minio-Internal-compression": "klauspost/compress/s2", 500 "content-type": "application/octet-stream", 501 "etag": "b3ff3ef3789147152fbfbc50efba4bfd-2"}, 502 Parts: []ObjectPartInfo{}, 503 }, 504 result: -1, 505 }, 506 } 507 for i, test := range testCases { 508 got, _ := test.objInfo.GetActualSize() 509 if got != test.result { 510 t.Errorf("Test %d - expected %d but received %d", 511 i+1, test.result, got) 512 } 513 } 514 } 515 516 func TestGetCompressedOffsets(t *testing.T) { 517 testCases := []struct { 518 objInfo ObjectInfo 519 offset int64 520 startOffset int64 521 snappyStartOffset int64 522 }{ 523 { 524 objInfo: ObjectInfo{ 525 Parts: []ObjectPartInfo{ 526 { 527 Size: 39235668, 528 ActualSize: 67108864, 529 }, 530 { 531 Size: 19177372, 532 ActualSize: 32891137, 533 }, 534 }, 535 }, 536 offset: 79109865, 537 startOffset: 39235668, 538 snappyStartOffset: 12001001, 539 }, 540 { 541 objInfo: ObjectInfo{ 542 Parts: []ObjectPartInfo{ 543 { 544 Size: 39235668, 545 ActualSize: 67108864, 546 }, 547 { 548 Size: 19177372, 549 ActualSize: 32891137, 550 }, 551 }, 552 }, 553 offset: 19109865, 554 startOffset: 0, 555 snappyStartOffset: 19109865, 556 }, 557 { 558 objInfo: ObjectInfo{ 559 Parts: []ObjectPartInfo{ 560 { 561 Size: 39235668, 562 ActualSize: 67108864, 563 }, 564 { 565 Size: 19177372, 566 ActualSize: 32891137, 567 }, 568 }, 569 }, 570 offset: 0, 571 startOffset: 0, 572 snappyStartOffset: 0, 573 }, 574 } 575 for i, test := range testCases { 576 startOffset, snappyStartOffset := getCompressedOffsets(test.objInfo, test.offset) 577 if startOffset != test.startOffset { 578 t.Errorf("Test %d - expected startOffset %d but received %d", 579 i+1, test.startOffset, startOffset) 580 } 581 if snappyStartOffset != test.snappyStartOffset { 582 t.Errorf("Test %d - expected snappyOffset %d but received %d", 583 i+1, test.snappyStartOffset, snappyStartOffset) 584 } 585 } 586 } 587 588 func TestS2CompressReader(t *testing.T) { 589 tests := []struct { 590 name string 591 data []byte 592 }{ 593 {name: "empty", data: nil}, 594 {name: "small", data: []byte("hello, world")}, 595 {name: "large", data: bytes.Repeat([]byte("hello, world"), 1000)}, 596 } 597 598 for _, tt := range tests { 599 t.Run(tt.name, func(t *testing.T) { 600 buf := make([]byte, 100) // make small buffer to ensure multiple reads are required for large case 601 602 r := newS2CompressReader(bytes.NewReader(tt.data)) 603 defer r.Close() 604 605 var rdrBuf bytes.Buffer 606 _, err := io.CopyBuffer(&rdrBuf, r, buf) 607 if err != nil { 608 t.Fatal(err) 609 } 610 611 var stdBuf bytes.Buffer 612 w := s2.NewWriter(&stdBuf) 613 _, err = io.CopyBuffer(w, bytes.NewReader(tt.data), buf) 614 if err != nil { 615 t.Fatal(err) 616 } 617 err = w.Close() 618 if err != nil { 619 t.Fatal(err) 620 } 621 622 var ( 623 got = rdrBuf.Bytes() 624 want = stdBuf.Bytes() 625 ) 626 if !bytes.Equal(got, want) { 627 t.Errorf("encoded data does not match\n\t%q\n\t%q", got, want) 628 } 629 630 var decBuf bytes.Buffer 631 decRdr := s2.NewReader(&rdrBuf) 632 _, err = io.Copy(&decBuf, decRdr) 633 if err != nil { 634 t.Fatal(err) 635 } 636 637 if !bytes.Equal(tt.data, decBuf.Bytes()) { 638 t.Errorf("roundtrip failed\n\t%q\n\t%q", tt.data, decBuf.Bytes()) 639 } 640 }) 641 } 642 }