github.com/minio/minio-go/v6@v6.0.57/functional_tests.go (about) 1 // +build ignore 2 3 /* 4 * MinIO Go Library for Amazon S3 Compatible Cloud Storage 5 * Copyright 2015-2017 MinIO, Inc. 6 * 7 * Licensed under the Apache License, Version 2.0 (the "License"); 8 * you may not use this file except in compliance with the License. 9 * You may obtain a copy of the License at 10 * 11 * http://www.apache.org/licenses/LICENSE-2.0 12 * 13 * Unless required by applicable law or agreed to in writing, software 14 * distributed under the License is distributed on an "AS IS" BASIS, 15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 * See the License for the specific language governing permissions and 17 * limitations under the License. 18 */ 19 20 package main 21 22 import ( 23 "bytes" 24 "context" 25 "errors" 26 "fmt" 27 "io" 28 "io/ioutil" 29 "math/rand" 30 "mime/multipart" 31 "net/http" 32 "net/url" 33 "os" 34 "path/filepath" 35 "reflect" 36 "runtime" 37 "strconv" 38 "strings" 39 "time" 40 41 "github.com/dustin/go-humanize" 42 jsoniter "github.com/json-iterator/go" 43 log "github.com/sirupsen/logrus" 44 45 "github.com/minio/minio-go/v6" 46 "github.com/minio/minio-go/v6/pkg/encrypt" 47 ) 48 49 const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" 50 const ( 51 letterIdxBits = 6 // 6 bits to represent a letter index 52 letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits 53 letterIdxMax = 63 / letterIdxBits // # of letter indices fitting in 63 bits 54 ) 55 const ( 56 serverEndpoint = "SERVER_ENDPOINT" 57 accessKey = "ACCESS_KEY" 58 secretKey = "SECRET_KEY" 59 enableHTTPS = "ENABLE_HTTPS" 60 enableKMS = "ENABLE_KMS" 61 ) 62 63 type mintJSONFormatter struct { 64 } 65 66 func (f *mintJSONFormatter) Format(entry *log.Entry) ([]byte, error) { 67 data := make(log.Fields, len(entry.Data)) 68 for k, v := range entry.Data { 69 switch v := v.(type) { 70 case error: 71 // Otherwise errors are ignored by `encoding/json` 72 // https://github.com/sirupsen/logrus/issues/137 73 data[k] = v.Error() 74 default: 75 data[k] = v 76 } 77 } 78 var json = jsoniter.ConfigCompatibleWithStandardLibrary 79 serialized, err := json.Marshal(data) 80 if err != nil { 81 return nil, fmt.Errorf("Failed to marshal fields to JSON, %v", err) 82 } 83 return append(serialized, '\n'), nil 84 } 85 86 func cleanEmptyEntries(fields log.Fields) log.Fields { 87 cleanFields := log.Fields{} 88 for k, v := range fields { 89 if v != "" { 90 cleanFields[k] = v 91 } 92 } 93 return cleanFields 94 } 95 96 // log successful test runs 97 func successLogger(testName string, function string, args map[string]interface{}, startTime time.Time) *log.Entry { 98 // calculate the test case duration 99 duration := time.Since(startTime) 100 // log with the fields as per mint 101 fields := log.Fields{"name": "minio-go: " + testName, "function": function, "args": args, "duration": duration.Nanoseconds() / 1000000, "status": "PASS"} 102 return log.WithFields(cleanEmptyEntries(fields)) 103 } 104 105 // As few of the features are not available in Gateway(s) currently, Check if err value is NotImplemented, 106 // and log as NA in that case and continue execution. Otherwise log as failure and return 107 func logError(testName string, function string, args map[string]interface{}, startTime time.Time, alert string, message string, err error) { 108 // If server returns NotImplemented we assume it is gateway mode and hence log it as info and move on to next tests 109 // Special case for ComposeObject API as it is implemented on client side and adds specific error details like `Error in upload-part-copy` in 110 // addition to NotImplemented error returned from server 111 if isErrNotImplemented(err) { 112 ignoredLog(testName, function, args, startTime, message).Info() 113 } else { 114 failureLog(testName, function, args, startTime, alert, message, err).Fatal() 115 } 116 } 117 118 // log failed test runs 119 func failureLog(testName string, function string, args map[string]interface{}, startTime time.Time, alert string, message string, err error) *log.Entry { 120 // calculate the test case duration 121 duration := time.Since(startTime) 122 var fields log.Fields 123 // log with the fields as per mint 124 if err != nil { 125 fields = log.Fields{"name": "minio-go: " + testName, "function": function, "args": args, 126 "duration": duration.Nanoseconds() / 1000000, "status": "FAIL", "alert": alert, "message": message, "error": err} 127 } else { 128 fields = log.Fields{"name": "minio-go: " + testName, "function": function, "args": args, 129 "duration": duration.Nanoseconds() / 1000000, "status": "FAIL", "alert": alert, "message": message} 130 } 131 return log.WithFields(cleanEmptyEntries(fields)) 132 } 133 134 // log not applicable test runs 135 func ignoredLog(testName string, function string, args map[string]interface{}, startTime time.Time, alert string) *log.Entry { 136 // calculate the test case duration 137 duration := time.Since(startTime) 138 // log with the fields as per mint 139 fields := log.Fields{"name": "minio-go: " + testName, "function": function, "args": args, 140 "duration": duration.Nanoseconds() / 1000000, "status": "NA", "alert": alert} 141 return log.WithFields(cleanEmptyEntries(fields)) 142 } 143 144 // Delete objects in given bucket, recursively 145 func cleanupBucket(bucketName string, c *minio.Client) error { 146 // Create a done channel to control 'ListObjectsV2' go routine. 147 doneCh := make(chan struct{}) 148 // Exit cleanly upon return. 149 defer close(doneCh) 150 // Iterate over all objects in the bucket via listObjectsV2 and delete 151 for objCh := range c.ListObjectsV2(bucketName, "", true, doneCh) { 152 if objCh.Err != nil { 153 return objCh.Err 154 } 155 if objCh.Key != "" { 156 err := c.RemoveObject(bucketName, objCh.Key) 157 if err != nil { 158 return err 159 } 160 } 161 } 162 for objPartInfo := range c.ListIncompleteUploads(bucketName, "", true, doneCh) { 163 if objPartInfo.Err != nil { 164 return objPartInfo.Err 165 } 166 if objPartInfo.Key != "" { 167 err := c.RemoveIncompleteUpload(bucketName, objPartInfo.Key) 168 if err != nil { 169 return err 170 } 171 } 172 } 173 // objects are already deleted, clear the buckets now 174 err := c.RemoveBucket(bucketName) 175 if err != nil { 176 return err 177 } 178 return err 179 } 180 181 func isErrNotImplemented(err error) bool { 182 return minio.ToErrorResponse(err).Code == "NotImplemented" 183 } 184 185 func init() { 186 // If server endpoint is not set, all tests default to 187 // using https://play.min.io 188 if os.Getenv(serverEndpoint) == "" { 189 os.Setenv(serverEndpoint, "play.min.io") 190 os.Setenv(accessKey, "Q3AM3UQ867SPQQA43P2F") 191 os.Setenv(secretKey, "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG") 192 os.Setenv(enableHTTPS, "1") 193 } 194 } 195 196 var mintDataDir = os.Getenv("MINT_DATA_DIR") 197 198 func getMintDataDirFilePath(filename string) (fp string) { 199 if mintDataDir == "" { 200 return 201 } 202 return filepath.Join(mintDataDir, filename) 203 } 204 205 type sizedReader struct { 206 io.Reader 207 size int 208 } 209 210 func (l *sizedReader) Size() int { 211 return l.size 212 } 213 214 func (l *sizedReader) Close() error { 215 return nil 216 } 217 218 type randomReader struct{ seed []byte } 219 220 func (r *randomReader) Read(b []byte) (int, error) { 221 return copy(b, bytes.Repeat(r.seed, len(b))), nil 222 } 223 224 // read data from file if it exists or optionally create a buffer of particular size 225 func getDataReader(fileName string) io.ReadCloser { 226 if mintDataDir == "" { 227 size := dataFileMap[fileName] 228 return &sizedReader{ 229 Reader: io.LimitReader(&randomReader{ 230 seed: []byte("a"), 231 }, int64(size)), 232 size: size, 233 } 234 } 235 reader, _ := os.Open(getMintDataDirFilePath(fileName)) 236 return reader 237 } 238 239 // randString generates random names and prepends them with a known prefix. 240 func randString(n int, src rand.Source, prefix string) string { 241 b := make([]byte, n) 242 // A rand.Int63() generates 63 random bits, enough for letterIdxMax letters! 243 for i, cache, remain := n-1, src.Int63(), letterIdxMax; i >= 0; { 244 if remain == 0 { 245 cache, remain = src.Int63(), letterIdxMax 246 } 247 if idx := int(cache & letterIdxMask); idx < len(letterBytes) { 248 b[i] = letterBytes[idx] 249 i-- 250 } 251 cache >>= letterIdxBits 252 remain-- 253 } 254 return prefix + string(b[0:30-len(prefix)]) 255 } 256 257 var dataFileMap = map[string]int{ 258 "datafile-1-b": 1, 259 "datafile-10-kB": 10 * humanize.KiByte, 260 "datafile-33-kB": 33 * humanize.KiByte, 261 "datafile-100-kB": 100 * humanize.KiByte, 262 "datafile-1.03-MB": 1056 * humanize.KiByte, 263 "datafile-1-MB": 1 * humanize.MiByte, 264 "datafile-5-MB": 5 * humanize.MiByte, 265 "datafile-6-MB": 6 * humanize.MiByte, 266 "datafile-11-MB": 11 * humanize.MiByte, 267 "datafile-129-MB": 129 * humanize.MiByte, 268 } 269 270 func isFullMode() bool { 271 return os.Getenv("MINT_MODE") == "full" 272 } 273 274 func getFuncName() string { 275 pc, _, _, _ := runtime.Caller(1) 276 return strings.TrimPrefix(runtime.FuncForPC(pc).Name(), "main.") 277 } 278 279 // Tests bucket re-create errors. 280 func testMakeBucketError() { 281 region := "eu-central-1" 282 283 // initialize logging params 284 startTime := time.Now() 285 testName := getFuncName() 286 function := "MakeBucket(bucketName, region)" 287 // initialize logging params 288 args := map[string]interface{}{ 289 "bucketName": "", 290 "region": region, 291 } 292 293 // skipping region functional tests for non s3 runs 294 if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { 295 ignoredLog(testName, function, args, startTime, "Skipped region functional tests for non s3 runs").Info() 296 return 297 } 298 299 // Seed random based on current time. 300 rand.Seed(time.Now().Unix()) 301 302 // Instantiate new minio client object. 303 c, err := minio.New( 304 os.Getenv(serverEndpoint), 305 os.Getenv(accessKey), 306 os.Getenv(secretKey), 307 mustParseBool(os.Getenv(enableHTTPS)), 308 ) 309 if err != nil { 310 logError(testName, function, args, startTime, "", "MinIO client creation failed", err) 311 return 312 } 313 314 // Enable tracing, write to stderr. 315 // c.TraceOn(os.Stderr) 316 317 // Set user agent. 318 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 319 320 // Generate a new random bucket name. 321 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 322 args["bucketName"] = bucketName 323 324 // Make a new bucket in 'eu-central-1'. 325 if err = c.MakeBucket(bucketName, region); err != nil { 326 logError(testName, function, args, startTime, "", "MakeBucket Failed", err) 327 return 328 } 329 if err = c.MakeBucket(bucketName, region); err == nil { 330 logError(testName, function, args, startTime, "", "Bucket already exists", err) 331 return 332 } 333 // Verify valid error response from server. 334 if minio.ToErrorResponse(err).Code != "BucketAlreadyExists" && 335 minio.ToErrorResponse(err).Code != "BucketAlreadyOwnedByYou" { 336 logError(testName, function, args, startTime, "", "Invalid error returned by server", err) 337 return 338 } 339 // Delete all objects and buckets 340 if err = cleanupBucket(bucketName, c); err != nil { 341 logError(testName, function, args, startTime, "", "Cleanup failed", err) 342 return 343 } 344 successLogger(testName, function, args, startTime).Info() 345 } 346 347 func testMetadataSizeLimit() { 348 startTime := time.Now() 349 testName := getFuncName() 350 function := "PutObject(bucketName, objectName, reader, objectSize, opts)" 351 args := map[string]interface{}{ 352 "bucketName": "", 353 "objectName": "", 354 "opts.UserMetadata": "", 355 } 356 rand.Seed(startTime.Unix()) 357 358 // Instantiate new minio client object. 359 c, err := minio.New( 360 os.Getenv(serverEndpoint), 361 os.Getenv(accessKey), 362 os.Getenv(secretKey), 363 mustParseBool(os.Getenv(enableHTTPS)), 364 ) 365 if err != nil { 366 logError(testName, function, args, startTime, "", "MinIO client creation failed", err) 367 return 368 } 369 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 370 371 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 372 args["bucketName"] = bucketName 373 374 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 375 args["objectName"] = objectName 376 377 err = c.MakeBucket(bucketName, "us-east-1") 378 if err != nil { 379 logError(testName, function, args, startTime, "", "Make bucket failed", err) 380 return 381 } 382 383 const HeaderSizeLimit = 8 * 1024 384 const UserMetadataLimit = 2 * 1024 385 386 // Meta-data greater than the 2 KB limit of AWS - PUT calls with this meta-data should fail 387 metadata := make(map[string]string) 388 metadata["X-Amz-Meta-Mint-Test"] = string(bytes.Repeat([]byte("m"), 1+UserMetadataLimit-len("X-Amz-Meta-Mint-Test"))) 389 args["metadata"] = fmt.Sprint(metadata) 390 391 _, err = c.PutObject(bucketName, objectName, bytes.NewReader(nil), 0, minio.PutObjectOptions{UserMetadata: metadata}) 392 if err == nil { 393 logError(testName, function, args, startTime, "", "Created object with user-defined metadata exceeding metadata size limits", nil) 394 return 395 } 396 397 // Meta-data (headers) greater than the 8 KB limit of AWS - PUT calls with this meta-data should fail 398 metadata = make(map[string]string) 399 metadata["X-Amz-Mint-Test"] = string(bytes.Repeat([]byte("m"), 1+HeaderSizeLimit-len("X-Amz-Mint-Test"))) 400 args["metadata"] = fmt.Sprint(metadata) 401 _, err = c.PutObject(bucketName, objectName, bytes.NewReader(nil), 0, minio.PutObjectOptions{UserMetadata: metadata}) 402 if err == nil { 403 logError(testName, function, args, startTime, "", "Created object with headers exceeding header size limits", nil) 404 return 405 } 406 407 // Delete all objects and buckets 408 if err = cleanupBucket(bucketName, c); err != nil { 409 logError(testName, function, args, startTime, "", "Cleanup failed", err) 410 return 411 } 412 413 successLogger(testName, function, args, startTime).Info() 414 } 415 416 // Tests various bucket supported formats. 417 func testMakeBucketRegions() { 418 region := "eu-central-1" 419 // initialize logging params 420 startTime := time.Now() 421 testName := getFuncName() 422 function := "MakeBucket(bucketName, region)" 423 // initialize logging params 424 args := map[string]interface{}{ 425 "bucketName": "", 426 "region": region, 427 } 428 429 // skipping region functional tests for non s3 runs 430 if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { 431 ignoredLog(testName, function, args, startTime, "Skipped region functional tests for non s3 runs").Info() 432 return 433 } 434 435 // Seed random based on current time. 436 rand.Seed(time.Now().Unix()) 437 438 // Instantiate new minio client object. 439 c, err := minio.New( 440 os.Getenv(serverEndpoint), 441 os.Getenv(accessKey), 442 os.Getenv(secretKey), 443 mustParseBool(os.Getenv(enableHTTPS)), 444 ) 445 if err != nil { 446 logError(testName, function, args, startTime, "", "MinIO client creation failed", err) 447 return 448 } 449 450 // Enable tracing, write to stderr. 451 // c.TraceOn(os.Stderr) 452 453 // Set user agent. 454 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 455 456 // Generate a new random bucket name. 457 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 458 args["bucketName"] = bucketName 459 460 // Make a new bucket in 'eu-central-1'. 461 if err = c.MakeBucket(bucketName, region); err != nil { 462 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 463 return 464 } 465 466 // Delete all objects and buckets 467 if err = cleanupBucket(bucketName, c); err != nil { 468 logError(testName, function, args, startTime, "", "Cleanup failed", err) 469 return 470 } 471 472 // Make a new bucket with '.' in its name, in 'us-west-2'. This 473 // request is internally staged into a path style instead of 474 // virtual host style. 475 region = "us-west-2" 476 args["region"] = region 477 if err = c.MakeBucket(bucketName+".withperiod", region); err != nil { 478 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 479 return 480 } 481 482 // Delete all objects and buckets 483 if err = cleanupBucket(bucketName+".withperiod", c); err != nil { 484 logError(testName, function, args, startTime, "", "Cleanup failed", err) 485 return 486 } 487 successLogger(testName, function, args, startTime).Info() 488 } 489 490 // Test PutObject using a large data to trigger multipart readat 491 func testPutObjectReadAt() { 492 // initialize logging params 493 startTime := time.Now() 494 testName := getFuncName() 495 function := "PutObject(bucketName, objectName, reader, opts)" 496 args := map[string]interface{}{ 497 "bucketName": "", 498 "objectName": "", 499 "opts": "objectContentType", 500 } 501 502 // Seed random based on current time. 503 rand.Seed(time.Now().Unix()) 504 505 // Instantiate new minio client object. 506 c, err := minio.New( 507 os.Getenv(serverEndpoint), 508 os.Getenv(accessKey), 509 os.Getenv(secretKey), 510 mustParseBool(os.Getenv(enableHTTPS)), 511 ) 512 if err != nil { 513 logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) 514 return 515 } 516 517 // Enable tracing, write to stderr. 518 // c.TraceOn(os.Stderr) 519 520 // Set user agent. 521 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 522 523 // Generate a new random bucket name. 524 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 525 args["bucketName"] = bucketName 526 527 // Make a new bucket. 528 err = c.MakeBucket(bucketName, "us-east-1") 529 if err != nil { 530 logError(testName, function, args, startTime, "", "Make bucket failed", err) 531 return 532 } 533 534 bufSize := dataFileMap["datafile-129-MB"] 535 var reader = getDataReader("datafile-129-MB") 536 defer reader.Close() 537 538 // Save the data 539 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 540 args["objectName"] = objectName 541 542 // Object content type 543 objectContentType := "binary/octet-stream" 544 args["objectContentType"] = objectContentType 545 546 n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: objectContentType}) 547 if err != nil { 548 logError(testName, function, args, startTime, "", "PutObject failed", err) 549 return 550 } 551 552 if n != int64(bufSize) { 553 logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match, expected "+string(bufSize)+" got "+string(n), err) 554 return 555 } 556 557 // Read the data back 558 r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) 559 if err != nil { 560 logError(testName, function, args, startTime, "", "Get Object failed", err) 561 return 562 } 563 564 st, err := r.Stat() 565 if err != nil { 566 logError(testName, function, args, startTime, "", "Stat Object failed", err) 567 return 568 } 569 if st.Size != int64(bufSize) { 570 logError(testName, function, args, startTime, "", fmt.Sprintf("Number of bytes in stat does not match, expected %d got %d", bufSize, st.Size), err) 571 return 572 } 573 if st.ContentType != objectContentType && st.ContentType != "application/octet-stream" { 574 logError(testName, function, args, startTime, "", "Content types don't match", err) 575 return 576 } 577 if err := r.Close(); err != nil { 578 logError(testName, function, args, startTime, "", "Object Close failed", err) 579 return 580 } 581 if err := r.Close(); err == nil { 582 logError(testName, function, args, startTime, "", "Object is already closed, didn't return error on Close", err) 583 return 584 } 585 586 // Delete all objects and buckets 587 if err = cleanupBucket(bucketName, c); err != nil { 588 logError(testName, function, args, startTime, "", "Cleanup failed", err) 589 return 590 } 591 592 successLogger(testName, function, args, startTime).Info() 593 } 594 595 // Test PutObject using a large data to trigger multipart readat 596 func testPutObjectWithMetadata() { 597 // initialize logging params 598 startTime := time.Now() 599 testName := getFuncName() 600 function := "PutObject(bucketName, objectName, reader,size, opts)" 601 args := map[string]interface{}{ 602 "bucketName": "", 603 "objectName": "", 604 "opts": "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress}", 605 } 606 607 if !isFullMode() { 608 ignoredLog(testName, function, args, startTime, "Skipping functional tests for short/quick runs").Info() 609 return 610 } 611 612 // Seed random based on current time. 613 rand.Seed(time.Now().Unix()) 614 615 // Instantiate new minio client object. 616 c, err := minio.New( 617 os.Getenv(serverEndpoint), 618 os.Getenv(accessKey), 619 os.Getenv(secretKey), 620 mustParseBool(os.Getenv(enableHTTPS)), 621 ) 622 if err != nil { 623 logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) 624 return 625 } 626 627 // Enable tracing, write to stderr. 628 // c.TraceOn(os.Stderr) 629 630 // Set user agent. 631 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 632 633 // Generate a new random bucket name. 634 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 635 args["bucketName"] = bucketName 636 637 // Make a new bucket. 638 err = c.MakeBucket(bucketName, "us-east-1") 639 if err != nil { 640 logError(testName, function, args, startTime, "", "Make bucket failed", err) 641 return 642 } 643 644 bufSize := dataFileMap["datafile-129-MB"] 645 var reader = getDataReader("datafile-129-MB") 646 defer reader.Close() 647 648 // Save the data 649 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 650 args["objectName"] = objectName 651 652 // Object custom metadata 653 customContentType := "custom/contenttype" 654 655 args["metadata"] = map[string][]string{ 656 "Content-Type": {customContentType}, 657 "X-Amz-Meta-CustomKey": {"extra spaces in value"}, 658 } 659 660 n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ 661 ContentType: customContentType}) 662 if err != nil { 663 logError(testName, function, args, startTime, "", "PutObject failed", err) 664 return 665 } 666 667 if n != int64(bufSize) { 668 logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match, expected "+string(bufSize)+" got "+string(n), err) 669 return 670 } 671 672 // Read the data back 673 r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) 674 if err != nil { 675 logError(testName, function, args, startTime, "", "GetObject failed", err) 676 return 677 } 678 679 st, err := r.Stat() 680 if err != nil { 681 logError(testName, function, args, startTime, "", "Stat failed", err) 682 return 683 } 684 if st.Size != int64(bufSize) { 685 logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match GetObject, expected "+string(bufSize)+" got "+string(st.Size), err) 686 return 687 } 688 if st.ContentType != customContentType && st.ContentType != "application/octet-stream" { 689 logError(testName, function, args, startTime, "", "ContentType does not match, expected "+customContentType+" got "+st.ContentType, err) 690 return 691 } 692 if err := r.Close(); err != nil { 693 logError(testName, function, args, startTime, "", "Object Close failed", err) 694 return 695 } 696 if err := r.Close(); err == nil { 697 logError(testName, function, args, startTime, "", "Object already closed, should respond with error", err) 698 return 699 } 700 701 // Delete all objects and buckets 702 if err = cleanupBucket(bucketName, c); err != nil { 703 logError(testName, function, args, startTime, "", "Cleanup failed", err) 704 return 705 } 706 707 successLogger(testName, function, args, startTime).Info() 708 } 709 710 func testPutObjectWithContentLanguage() { 711 // initialize logging params 712 objectName := "test-object" 713 startTime := time.Now() 714 testName := getFuncName() 715 function := "PutObject(bucketName, objectName, reader, size, opts)" 716 args := map[string]interface{}{ 717 "bucketName": "", 718 "objectName": objectName, 719 "size": -1, 720 "opts": "", 721 } 722 723 // Seed random based on current time. 724 rand.Seed(time.Now().Unix()) 725 726 // Instantiate new minio client object. 727 c, err := minio.NewV4( 728 os.Getenv(serverEndpoint), 729 os.Getenv(accessKey), 730 os.Getenv(secretKey), 731 mustParseBool(os.Getenv(enableHTTPS)), 732 ) 733 if err != nil { 734 logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) 735 return 736 } 737 738 // Enable tracing, write to stderr. 739 // c.TraceOn(os.Stderr) 740 741 // Set user agent. 742 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 743 744 // Generate a new random bucket name. 745 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 746 args["bucketName"] = bucketName 747 // Make a new bucket. 748 err = c.MakeBucket(bucketName, "us-east-1") 749 if err != nil { 750 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 751 return 752 } 753 754 data := bytes.Repeat([]byte("a"), int(0)) 755 n, err := c.PutObject(bucketName, objectName, bytes.NewReader(data), int64(0), minio.PutObjectOptions{ 756 ContentLanguage: "en", 757 }) 758 if err != nil { 759 logError(testName, function, args, startTime, "", "PutObject failed", err) 760 return 761 } 762 763 if n != 0 { 764 logError(testName, function, args, startTime, "", "Expected upload object '0' doesn't match with PutObject return value", err) 765 return 766 } 767 768 objInfo, err := c.StatObject(bucketName, objectName, minio.StatObjectOptions{}) 769 if err != nil { 770 logError(testName, function, args, startTime, "", "StatObject failed", err) 771 return 772 } 773 774 if objInfo.Metadata.Get("Content-Language") != "en" { 775 logError(testName, function, args, startTime, "", "Expected content-language 'en' doesn't match with StatObject return value", err) 776 return 777 } 778 779 // Delete all objects and buckets 780 if err = cleanupBucket(bucketName, c); err != nil { 781 logError(testName, function, args, startTime, "", "Cleanup failed", err) 782 return 783 } 784 785 successLogger(testName, function, args, startTime).Info() 786 } 787 788 // Test put object with streaming signature. 789 func testPutObjectStreaming() { 790 // initialize logging params 791 objectName := "test-object" 792 startTime := time.Now() 793 testName := getFuncName() 794 function := "PutObject(bucketName, objectName, reader,size,opts)" 795 args := map[string]interface{}{ 796 "bucketName": "", 797 "objectName": objectName, 798 "size": -1, 799 "opts": "", 800 } 801 802 // Seed random based on current time. 803 rand.Seed(time.Now().Unix()) 804 805 // Instantiate new minio client object. 806 c, err := minio.NewV4( 807 os.Getenv(serverEndpoint), 808 os.Getenv(accessKey), 809 os.Getenv(secretKey), 810 mustParseBool(os.Getenv(enableHTTPS)), 811 ) 812 if err != nil { 813 logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) 814 return 815 } 816 817 // Enable tracing, write to stderr. 818 // c.TraceOn(os.Stderr) 819 820 // Set user agent. 821 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 822 823 // Generate a new random bucket name. 824 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 825 args["bucketName"] = bucketName 826 // Make a new bucket. 827 err = c.MakeBucket(bucketName, "us-east-1") 828 if err != nil { 829 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 830 return 831 } 832 833 // Upload an object. 834 sizes := []int64{0, 64*1024 - 1, 64 * 1024} 835 836 for _, size := range sizes { 837 data := bytes.Repeat([]byte("a"), int(size)) 838 n, err := c.PutObject(bucketName, objectName, bytes.NewReader(data), int64(size), minio.PutObjectOptions{}) 839 if err != nil { 840 logError(testName, function, args, startTime, "", "PutObjectStreaming failed", err) 841 return 842 } 843 844 if n != size { 845 logError(testName, function, args, startTime, "", "Expected upload object size doesn't match with PutObjectStreaming return value", err) 846 return 847 } 848 } 849 850 // Delete all objects and buckets 851 if err = cleanupBucket(bucketName, c); err != nil { 852 logError(testName, function, args, startTime, "", "Cleanup failed", err) 853 return 854 } 855 856 successLogger(testName, function, args, startTime).Info() 857 } 858 859 // Test get object seeker from the end, using whence set to '2'. 860 func testGetObjectSeekEnd() { 861 // initialize logging params 862 startTime := time.Now() 863 testName := getFuncName() 864 function := "GetObject(bucketName, objectName)" 865 args := map[string]interface{}{} 866 867 // Seed random based on current time. 868 rand.Seed(time.Now().Unix()) 869 870 // Instantiate new minio client object. 871 c, err := minio.New( 872 os.Getenv(serverEndpoint), 873 os.Getenv(accessKey), 874 os.Getenv(secretKey), 875 mustParseBool(os.Getenv(enableHTTPS)), 876 ) 877 if err != nil { 878 logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) 879 return 880 } 881 882 // Enable tracing, write to stderr. 883 // c.TraceOn(os.Stderr) 884 885 // Set user agent. 886 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 887 888 // Generate a new random bucket name. 889 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 890 args["bucketName"] = bucketName 891 892 // Make a new bucket. 893 err = c.MakeBucket(bucketName, "us-east-1") 894 if err != nil { 895 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 896 return 897 } 898 899 // Generate 33K of data. 900 bufSize := dataFileMap["datafile-33-kB"] 901 var reader = getDataReader("datafile-33-kB") 902 defer reader.Close() 903 904 // Save the data 905 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 906 args["objectName"] = objectName 907 908 buf, err := ioutil.ReadAll(reader) 909 if err != nil { 910 logError(testName, function, args, startTime, "", "ReadAll failed", err) 911 return 912 } 913 914 n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) 915 if err != nil { 916 logError(testName, function, args, startTime, "", "PutObject failed", err) 917 return 918 } 919 920 if n != int64(bufSize) { 921 logError(testName, function, args, startTime, "", "Number of bytes read does not match, expected "+string(int64(bufSize))+" got "+string(n), err) 922 return 923 } 924 925 // Read the data back 926 r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) 927 if err != nil { 928 logError(testName, function, args, startTime, "", "GetObject failed", err) 929 return 930 } 931 932 st, err := r.Stat() 933 if err != nil { 934 logError(testName, function, args, startTime, "", "Stat failed", err) 935 return 936 } 937 938 if st.Size != int64(bufSize) { 939 logError(testName, function, args, startTime, "", "Number of bytes read does not match, expected "+string(int64(bufSize))+" got "+string(st.Size), err) 940 return 941 } 942 943 pos, err := r.Seek(-100, 2) 944 if err != nil { 945 logError(testName, function, args, startTime, "", "Object Seek failed", err) 946 return 947 } 948 if pos != st.Size-100 { 949 logError(testName, function, args, startTime, "", "Incorrect position", err) 950 return 951 } 952 buf2 := make([]byte, 100) 953 m, err := io.ReadFull(r, buf2) 954 if err != nil { 955 logError(testName, function, args, startTime, "", "Error reading through io.ReadFull", err) 956 return 957 } 958 if m != len(buf2) { 959 logError(testName, function, args, startTime, "", "Number of bytes dont match, expected "+string(len(buf2))+" got "+string(m), err) 960 return 961 } 962 hexBuf1 := fmt.Sprintf("%02x", buf[len(buf)-100:]) 963 hexBuf2 := fmt.Sprintf("%02x", buf2[:m]) 964 if hexBuf1 != hexBuf2 { 965 logError(testName, function, args, startTime, "", "Values at same index dont match", err) 966 return 967 } 968 pos, err = r.Seek(-100, 2) 969 if err != nil { 970 logError(testName, function, args, startTime, "", "Object Seek failed", err) 971 return 972 } 973 if pos != st.Size-100 { 974 logError(testName, function, args, startTime, "", "Incorrect position", err) 975 return 976 } 977 if err = r.Close(); err != nil { 978 logError(testName, function, args, startTime, "", "ObjectClose failed", err) 979 return 980 } 981 982 // Delete all objects and buckets 983 if err = cleanupBucket(bucketName, c); err != nil { 984 logError(testName, function, args, startTime, "", "Cleanup failed", err) 985 return 986 } 987 988 successLogger(testName, function, args, startTime).Info() 989 } 990 991 // Test get object reader to not throw error on being closed twice. 992 func testGetObjectClosedTwice() { 993 // initialize logging params 994 startTime := time.Now() 995 testName := getFuncName() 996 function := "GetObject(bucketName, objectName)" 997 args := map[string]interface{}{} 998 999 // Seed random based on current time. 1000 rand.Seed(time.Now().Unix()) 1001 1002 // Instantiate new minio client object. 1003 c, err := minio.New( 1004 os.Getenv(serverEndpoint), 1005 os.Getenv(accessKey), 1006 os.Getenv(secretKey), 1007 mustParseBool(os.Getenv(enableHTTPS)), 1008 ) 1009 if err != nil { 1010 logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) 1011 return 1012 } 1013 1014 // Enable tracing, write to stderr. 1015 // c.TraceOn(os.Stderr) 1016 1017 // Set user agent. 1018 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 1019 1020 // Generate a new random bucket name. 1021 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 1022 args["bucketName"] = bucketName 1023 1024 // Make a new bucket. 1025 err = c.MakeBucket(bucketName, "us-east-1") 1026 if err != nil { 1027 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 1028 return 1029 } 1030 1031 // Generate 33K of data. 1032 bufSize := dataFileMap["datafile-33-kB"] 1033 var reader = getDataReader("datafile-33-kB") 1034 defer reader.Close() 1035 1036 // Save the data 1037 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 1038 args["objectName"] = objectName 1039 1040 n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) 1041 if err != nil { 1042 logError(testName, function, args, startTime, "", "PutObject failed", err) 1043 return 1044 } 1045 1046 if n != int64(bufSize) { 1047 logError(testName, function, args, startTime, "", "PutObject response doesn't match sent bytes, expected "+string(int64(bufSize))+" got "+string(n), err) 1048 return 1049 } 1050 1051 // Read the data back 1052 r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) 1053 if err != nil { 1054 logError(testName, function, args, startTime, "", "GetObject failed", err) 1055 return 1056 } 1057 1058 st, err := r.Stat() 1059 if err != nil { 1060 logError(testName, function, args, startTime, "", "Stat failed", err) 1061 return 1062 } 1063 if st.Size != int64(bufSize) { 1064 logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+" got "+string(st.Size), err) 1065 return 1066 } 1067 if err := r.Close(); err != nil { 1068 logError(testName, function, args, startTime, "", "Object Close failed", err) 1069 return 1070 } 1071 if err := r.Close(); err == nil { 1072 logError(testName, function, args, startTime, "", "Already closed object. No error returned", err) 1073 return 1074 } 1075 1076 // Delete all objects and buckets 1077 if err = cleanupBucket(bucketName, c); err != nil { 1078 logError(testName, function, args, startTime, "", "Cleanup failed", err) 1079 return 1080 } 1081 1082 successLogger(testName, function, args, startTime).Info() 1083 } 1084 1085 // Test RemoveObjectsWithContext request context cancels after timeout 1086 func testRemoveObjectsWithContext() { 1087 // Initialize logging params. 1088 startTime := time.Now() 1089 testName := getFuncName() 1090 function := "RemoveObjectsWithContext(ctx, bucketName, objectsCh)" 1091 args := map[string]interface{}{ 1092 "bucketName": "", 1093 } 1094 1095 // Seed random based on current tie. 1096 rand.Seed(time.Now().Unix()) 1097 1098 // Instantiate new minio client. 1099 c, err := minio.New( 1100 os.Getenv(serverEndpoint), 1101 os.Getenv(accessKey), 1102 os.Getenv(secretKey), 1103 mustParseBool(os.Getenv(enableHTTPS)), 1104 ) 1105 if err != nil { 1106 logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) 1107 return 1108 } 1109 1110 // Set user agent. 1111 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 1112 // Enable tracing, write to stdout. 1113 // c.TraceOn(os.Stderr) 1114 1115 // Generate a new random bucket name. 1116 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 1117 args["bucketName"] = bucketName 1118 1119 // Make a new bucket. 1120 err = c.MakeBucket(bucketName, "us-east-1") 1121 if err != nil { 1122 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 1123 } 1124 1125 // Generate put data. 1126 r := bytes.NewReader(bytes.Repeat([]byte("a"), 8)) 1127 1128 // Multi remove of 20 objects. 1129 nrObjects := 20 1130 objectsCh := make(chan string) 1131 go func() { 1132 defer close(objectsCh) 1133 for i := 0; i < nrObjects; i++ { 1134 objectName := "sample" + strconv.Itoa(i) + ".txt" 1135 _, err = c.PutObject(bucketName, objectName, r, 8, minio.PutObjectOptions{ContentType: "application/octet-stream"}) 1136 if err != nil { 1137 logError(testName, function, args, startTime, "", "PutObject failed", err) 1138 continue 1139 } 1140 objectsCh <- objectName 1141 } 1142 }() 1143 // Set context to cancel in 1 nanosecond. 1144 ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) 1145 args["ctx"] = ctx 1146 defer cancel() 1147 1148 // Call RemoveObjectsWithContext API with short timeout. 1149 errorCh := c.RemoveObjectsWithContext(ctx, bucketName, objectsCh) 1150 // Check for error. 1151 select { 1152 case r := <-errorCh: 1153 if r.Err == nil { 1154 logError(testName, function, args, startTime, "", "RemoveObjectsWithContext should fail on short timeout", err) 1155 return 1156 } 1157 } 1158 // Set context with longer timeout. 1159 ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) 1160 args["ctx"] = ctx 1161 defer cancel() 1162 // Perform RemoveObjectsWithContext with the longer timeout. Expect the removals to succeed. 1163 errorCh = c.RemoveObjectsWithContext(ctx, bucketName, objectsCh) 1164 select { 1165 case r, more := <-errorCh: 1166 if more || r.Err != nil { 1167 logError(testName, function, args, startTime, "", "Unexpected error", r.Err) 1168 return 1169 } 1170 } 1171 1172 // Delete all objects and buckets. 1173 if err = cleanupBucket(bucketName, c); err != nil { 1174 logError(testName, function, args, startTime, "", "Cleanup failed", err) 1175 return 1176 } 1177 successLogger(testName, function, args, startTime).Info() 1178 } 1179 1180 // Test removing multiple objects with Remove API 1181 func testRemoveMultipleObjects() { 1182 // initialize logging params 1183 startTime := time.Now() 1184 testName := getFuncName() 1185 function := "RemoveObjects(bucketName, objectsCh)" 1186 args := map[string]interface{}{ 1187 "bucketName": "", 1188 } 1189 1190 // Seed random based on current time. 1191 rand.Seed(time.Now().Unix()) 1192 1193 // Instantiate new minio client object. 1194 c, err := minio.New( 1195 os.Getenv(serverEndpoint), 1196 os.Getenv(accessKey), 1197 os.Getenv(secretKey), 1198 mustParseBool(os.Getenv(enableHTTPS)), 1199 ) 1200 1201 if err != nil { 1202 logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) 1203 return 1204 } 1205 1206 // Set user agent. 1207 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 1208 1209 // Enable tracing, write to stdout. 1210 // c.TraceOn(os.Stderr) 1211 1212 // Generate a new random bucket name. 1213 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 1214 args["bucketName"] = bucketName 1215 1216 // Make a new bucket. 1217 err = c.MakeBucket(bucketName, "us-east-1") 1218 if err != nil { 1219 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 1220 return 1221 } 1222 1223 r := bytes.NewReader(bytes.Repeat([]byte("a"), 8)) 1224 1225 // Multi remove of 1100 objects 1226 nrObjects := 200 1227 1228 objectsCh := make(chan string) 1229 1230 go func() { 1231 defer close(objectsCh) 1232 // Upload objects and send them to objectsCh 1233 for i := 0; i < nrObjects; i++ { 1234 objectName := "sample" + strconv.Itoa(i) + ".txt" 1235 _, err = c.PutObject(bucketName, objectName, r, 8, minio.PutObjectOptions{ContentType: "application/octet-stream"}) 1236 if err != nil { 1237 logError(testName, function, args, startTime, "", "PutObject failed", err) 1238 continue 1239 } 1240 objectsCh <- objectName 1241 } 1242 }() 1243 1244 // Call RemoveObjects API 1245 errorCh := c.RemoveObjects(bucketName, objectsCh) 1246 1247 // Check if errorCh doesn't receive any error 1248 select { 1249 case r, more := <-errorCh: 1250 if more { 1251 logError(testName, function, args, startTime, "", "Unexpected error", r.Err) 1252 return 1253 } 1254 } 1255 1256 // Delete all objects and buckets 1257 if err = cleanupBucket(bucketName, c); err != nil { 1258 logError(testName, function, args, startTime, "", "Cleanup failed", err) 1259 return 1260 } 1261 1262 successLogger(testName, function, args, startTime).Info() 1263 } 1264 1265 // Tests FPutObject of a big file to trigger multipart 1266 func testFPutObjectMultipart() { 1267 // initialize logging params 1268 startTime := time.Now() 1269 testName := getFuncName() 1270 function := "FPutObject(bucketName, objectName, fileName, opts)" 1271 args := map[string]interface{}{ 1272 "bucketName": "", 1273 "objectName": "", 1274 "fileName": "", 1275 "opts": "", 1276 } 1277 1278 // Seed random based on current time. 1279 rand.Seed(time.Now().Unix()) 1280 1281 // Instantiate new minio client object. 1282 c, err := minio.New( 1283 os.Getenv(serverEndpoint), 1284 os.Getenv(accessKey), 1285 os.Getenv(secretKey), 1286 mustParseBool(os.Getenv(enableHTTPS)), 1287 ) 1288 if err != nil { 1289 logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) 1290 return 1291 } 1292 1293 // Enable tracing, write to stderr. 1294 // c.TraceOn(os.Stderr) 1295 1296 // Set user agent. 1297 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 1298 1299 // Generate a new random bucket name. 1300 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 1301 args["bucketName"] = bucketName 1302 1303 // Make a new bucket. 1304 err = c.MakeBucket(bucketName, "us-east-1") 1305 if err != nil { 1306 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 1307 return 1308 } 1309 1310 // Upload 4 parts to utilize all 3 'workers' in multipart and still have a part to upload. 1311 var fileName = getMintDataDirFilePath("datafile-129-MB") 1312 if fileName == "" { 1313 // Make a temp file with minPartSize bytes of data. 1314 file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") 1315 if err != nil { 1316 logError(testName, function, args, startTime, "", "TempFile creation failed", err) 1317 return 1318 } 1319 // Upload 2 parts to utilize all 3 'workers' in multipart and still have a part to upload. 1320 if _, err = io.Copy(file, getDataReader("datafile-129-MB")); err != nil { 1321 logError(testName, function, args, startTime, "", "Copy failed", err) 1322 return 1323 } 1324 if err = file.Close(); err != nil { 1325 logError(testName, function, args, startTime, "", "File Close failed", err) 1326 return 1327 } 1328 fileName = file.Name() 1329 args["fileName"] = fileName 1330 } 1331 totalSize := dataFileMap["datafile-129-MB"] 1332 // Set base object name 1333 objectName := bucketName + "FPutObject" + "-standard" 1334 args["objectName"] = objectName 1335 1336 objectContentType := "testapplication/octet-stream" 1337 args["objectContentType"] = objectContentType 1338 1339 // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) 1340 n, err := c.FPutObject(bucketName, objectName, fileName, minio.PutObjectOptions{ContentType: objectContentType}) 1341 if err != nil { 1342 logError(testName, function, args, startTime, "", "FPutObject failed", err) 1343 return 1344 } 1345 if n != int64(totalSize) { 1346 logError(testName, function, args, startTime, "", "FPutObject failed", err) 1347 return 1348 } 1349 1350 r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) 1351 if err != nil { 1352 logError(testName, function, args, startTime, "", "GetObject failed", err) 1353 return 1354 } 1355 objInfo, err := r.Stat() 1356 if err != nil { 1357 logError(testName, function, args, startTime, "", "Unexpected error", err) 1358 return 1359 } 1360 if objInfo.Size != int64(totalSize) { 1361 logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(totalSize))+" got "+string(objInfo.Size), err) 1362 return 1363 } 1364 if objInfo.ContentType != objectContentType && objInfo.ContentType != "application/octet-stream" { 1365 logError(testName, function, args, startTime, "", "ContentType doesn't match", err) 1366 return 1367 } 1368 1369 // Delete all objects and buckets 1370 if err = cleanupBucket(bucketName, c); err != nil { 1371 logError(testName, function, args, startTime, "", "Cleanup failed", err) 1372 return 1373 } 1374 1375 successLogger(testName, function, args, startTime).Info() 1376 } 1377 1378 // Tests FPutObject with null contentType (default = application/octet-stream) 1379 func testFPutObject() { 1380 // initialize logging params 1381 startTime := time.Now() 1382 testName := getFuncName() 1383 function := "FPutObject(bucketName, objectName, fileName, opts)" 1384 1385 args := map[string]interface{}{ 1386 "bucketName": "", 1387 "objectName": "", 1388 "fileName": "", 1389 "opts": "", 1390 } 1391 1392 // Seed random based on current time. 1393 rand.Seed(time.Now().Unix()) 1394 1395 // Instantiate new minio client object. 1396 c, err := minio.New( 1397 os.Getenv(serverEndpoint), 1398 os.Getenv(accessKey), 1399 os.Getenv(secretKey), 1400 mustParseBool(os.Getenv(enableHTTPS)), 1401 ) 1402 if err != nil { 1403 logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) 1404 return 1405 } 1406 1407 // Enable tracing, write to stderr. 1408 // c.TraceOn(os.Stderr) 1409 1410 // Set user agent. 1411 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 1412 1413 // Generate a new random bucket name. 1414 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 1415 location := "us-east-1" 1416 1417 // Make a new bucket. 1418 args["bucketName"] = bucketName 1419 args["location"] = location 1420 function = "MakeBucket()bucketName, location" 1421 err = c.MakeBucket(bucketName, location) 1422 if err != nil { 1423 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 1424 return 1425 } 1426 1427 // Upload 3 parts worth of data to use all 3 of multiparts 'workers' and have an extra part. 1428 // Use different data in part for multipart tests to check parts are uploaded in correct order. 1429 var fName = getMintDataDirFilePath("datafile-129-MB") 1430 if fName == "" { 1431 // Make a temp file with minPartSize bytes of data. 1432 file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") 1433 if err != nil { 1434 logError(testName, function, args, startTime, "", "TempFile creation failed", err) 1435 return 1436 } 1437 1438 // Upload 3 parts to utilize all 3 'workers' in multipart and still have a part to upload. 1439 if _, err = io.Copy(file, getDataReader("datafile-129-MB")); err != nil { 1440 logError(testName, function, args, startTime, "", "File copy failed", err) 1441 return 1442 } 1443 // Close the file pro-actively for windows. 1444 if err = file.Close(); err != nil { 1445 logError(testName, function, args, startTime, "", "File close failed", err) 1446 return 1447 } 1448 defer os.Remove(file.Name()) 1449 fName = file.Name() 1450 } 1451 totalSize := dataFileMap["datafile-129-MB"] 1452 1453 // Set base object name 1454 function = "FPutObject(bucketName, objectName, fileName, opts)" 1455 objectName := bucketName + "FPutObject" 1456 args["objectName"] = objectName + "-standard" 1457 args["fileName"] = fName 1458 args["opts"] = minio.PutObjectOptions{ContentType: "application/octet-stream"} 1459 1460 // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) 1461 n, err := c.FPutObject(bucketName, objectName+"-standard", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"}) 1462 1463 if err != nil { 1464 logError(testName, function, args, startTime, "", "FPutObject failed", err) 1465 return 1466 } 1467 if n != int64(totalSize) { 1468 logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(totalSize)+", got "+string(n), err) 1469 return 1470 } 1471 1472 // Perform FPutObject with no contentType provided (Expecting application/octet-stream) 1473 args["objectName"] = objectName + "-Octet" 1474 n, err = c.FPutObject(bucketName, objectName+"-Octet", fName, minio.PutObjectOptions{}) 1475 if err != nil { 1476 logError(testName, function, args, startTime, "", "File close failed", err) 1477 return 1478 } 1479 if n != int64(totalSize) { 1480 logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(totalSize)+", got "+string(n), err) 1481 return 1482 } 1483 srcFile, err := os.Open(fName) 1484 if err != nil { 1485 logError(testName, function, args, startTime, "", "File open failed", err) 1486 return 1487 } 1488 defer srcFile.Close() 1489 // Add extension to temp file name 1490 tmpFile, err := os.Create(fName + ".gtar") 1491 if err != nil { 1492 logError(testName, function, args, startTime, "", "File create failed", err) 1493 return 1494 } 1495 defer tmpFile.Close() 1496 _, err = io.Copy(tmpFile, srcFile) 1497 if err != nil { 1498 logError(testName, function, args, startTime, "", "File copy failed", err) 1499 return 1500 } 1501 1502 // Perform FPutObject with no contentType provided (Expecting application/x-gtar) 1503 args["objectName"] = objectName + "-GTar" 1504 args["opts"] = minio.PutObjectOptions{} 1505 n, err = c.FPutObject(bucketName, objectName+"-GTar", fName+".gtar", minio.PutObjectOptions{}) 1506 if err != nil { 1507 logError(testName, function, args, startTime, "", "FPutObject failed", err) 1508 return 1509 } 1510 if n != int64(totalSize) { 1511 logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(totalSize)+", got "+string(n), err) 1512 return 1513 } 1514 1515 // Check headers 1516 function = "StatObject(bucketName, objectName, opts)" 1517 args["objectName"] = objectName + "-standard" 1518 rStandard, err := c.StatObject(bucketName, objectName+"-standard", minio.StatObjectOptions{}) 1519 if err != nil { 1520 logError(testName, function, args, startTime, "", "StatObject failed", err) 1521 return 1522 } 1523 if rStandard.ContentType != "application/octet-stream" { 1524 logError(testName, function, args, startTime, "", "ContentType does not match, expected application/octet-stream, got "+rStandard.ContentType, err) 1525 return 1526 } 1527 1528 function = "StatObject(bucketName, objectName, opts)" 1529 args["objectName"] = objectName + "-Octet" 1530 rOctet, err := c.StatObject(bucketName, objectName+"-Octet", minio.StatObjectOptions{}) 1531 if err != nil { 1532 logError(testName, function, args, startTime, "", "StatObject failed", err) 1533 return 1534 } 1535 if rOctet.ContentType != "application/octet-stream" { 1536 logError(testName, function, args, startTime, "", "ContentType does not match, expected application/octet-stream, got "+rOctet.ContentType, err) 1537 return 1538 } 1539 1540 function = "StatObject(bucketName, objectName, opts)" 1541 args["objectName"] = objectName + "-GTar" 1542 rGTar, err := c.StatObject(bucketName, objectName+"-GTar", minio.StatObjectOptions{}) 1543 if err != nil { 1544 logError(testName, function, args, startTime, "", "StatObject failed", err) 1545 return 1546 } 1547 if rGTar.ContentType != "application/x-gtar" && rGTar.ContentType != "application/octet-stream" { 1548 logError(testName, function, args, startTime, "", "ContentType does not match, expected application/x-gtar or application/octet-stream, got "+rGTar.ContentType, err) 1549 return 1550 } 1551 1552 // Delete all objects and buckets 1553 if err = cleanupBucket(bucketName, c); err != nil { 1554 logError(testName, function, args, startTime, "", "Cleanup failed", err) 1555 return 1556 } 1557 1558 os.Remove(fName + ".gtar") 1559 successLogger(testName, function, args, startTime).Info() 1560 } 1561 1562 // Tests FPutObjectWithContext request context cancels after timeout 1563 func testFPutObjectWithContext() { 1564 // initialize logging params 1565 startTime := time.Now() 1566 testName := getFuncName() 1567 function := "FPutObject(bucketName, objectName, fileName, opts)" 1568 args := map[string]interface{}{ 1569 "bucketName": "", 1570 "objectName": "", 1571 "fileName": "", 1572 "opts": "", 1573 } 1574 // Seed random based on current time. 1575 rand.Seed(time.Now().Unix()) 1576 1577 // Instantiate new minio client object. 1578 c, err := minio.New( 1579 os.Getenv(serverEndpoint), 1580 os.Getenv(accessKey), 1581 os.Getenv(secretKey), 1582 mustParseBool(os.Getenv(enableHTTPS)), 1583 ) 1584 if err != nil { 1585 logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) 1586 return 1587 } 1588 1589 // Enable tracing, write to stderr. 1590 // c.TraceOn(os.Stderr) 1591 1592 // Set user agent. 1593 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 1594 1595 // Generate a new random bucket name. 1596 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 1597 args["bucketName"] = bucketName 1598 1599 // Make a new bucket. 1600 err = c.MakeBucket(bucketName, "us-east-1") 1601 if err != nil { 1602 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 1603 return 1604 } 1605 1606 // Upload 1 parts worth of data to use multipart upload. 1607 // Use different data in part for multipart tests to check parts are uploaded in correct order. 1608 var fName = getMintDataDirFilePath("datafile-1-MB") 1609 if fName == "" { 1610 // Make a temp file with 1 MiB bytes of data. 1611 file, err := ioutil.TempFile(os.TempDir(), "FPutObjectWithContextTest") 1612 if err != nil { 1613 logError(testName, function, args, startTime, "", "TempFile creation failed", err) 1614 return 1615 } 1616 1617 // Upload 1 parts to trigger multipart upload 1618 if _, err = io.Copy(file, getDataReader("datafile-1-MB")); err != nil { 1619 logError(testName, function, args, startTime, "", "File copy failed", err) 1620 return 1621 } 1622 // Close the file pro-actively for windows. 1623 if err = file.Close(); err != nil { 1624 logError(testName, function, args, startTime, "", "File close failed", err) 1625 return 1626 } 1627 defer os.Remove(file.Name()) 1628 fName = file.Name() 1629 } 1630 totalSize := dataFileMap["datafile-1-MB"] 1631 1632 // Set base object name 1633 objectName := bucketName + "FPutObjectWithContext" 1634 args["objectName"] = objectName 1635 ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) 1636 args["ctx"] = ctx 1637 defer cancel() 1638 1639 // Perform standard FPutObjectWithContext with contentType provided (Expecting application/octet-stream) 1640 _, err = c.FPutObjectWithContext(ctx, bucketName, objectName+"-Shorttimeout", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"}) 1641 if err == nil { 1642 logError(testName, function, args, startTime, "", "FPutObjectWithContext should fail on short timeout", err) 1643 return 1644 } 1645 ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) 1646 defer cancel() 1647 // Perform FPutObjectWithContext with a long timeout. Expect the put object to succeed 1648 n, err := c.FPutObjectWithContext(ctx, bucketName, objectName+"-Longtimeout", fName, minio.PutObjectOptions{}) 1649 if err != nil { 1650 logError(testName, function, args, startTime, "", "FPutObjectWithContext shouldn't fail on long timeout", err) 1651 return 1652 } 1653 if n != int64(totalSize) { 1654 logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(totalSize)+", got "+string(n), err) 1655 return 1656 } 1657 1658 _, err = c.StatObject(bucketName, objectName+"-Longtimeout", minio.StatObjectOptions{}) 1659 if err != nil { 1660 logError(testName, function, args, startTime, "", "StatObject failed", err) 1661 return 1662 } 1663 1664 // Delete all objects and buckets 1665 if err = cleanupBucket(bucketName, c); err != nil { 1666 logError(testName, function, args, startTime, "", "Cleanup failed", err) 1667 return 1668 } 1669 1670 successLogger(testName, function, args, startTime).Info() 1671 1672 } 1673 1674 // Tests FPutObjectWithContext request context cancels after timeout 1675 func testFPutObjectWithContextV2() { 1676 // initialize logging params 1677 startTime := time.Now() 1678 testName := getFuncName() 1679 function := "FPutObjectWithContext(ctx, bucketName, objectName, fileName, opts)" 1680 args := map[string]interface{}{ 1681 "bucketName": "", 1682 "objectName": "", 1683 "opts": "minio.PutObjectOptions{ContentType:objectContentType}", 1684 } 1685 // Seed random based on current time. 1686 rand.Seed(time.Now().Unix()) 1687 1688 // Instantiate new minio client object. 1689 c, err := minio.NewV2( 1690 os.Getenv(serverEndpoint), 1691 os.Getenv(accessKey), 1692 os.Getenv(secretKey), 1693 mustParseBool(os.Getenv(enableHTTPS)), 1694 ) 1695 if err != nil { 1696 logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) 1697 return 1698 } 1699 1700 // Enable tracing, write to stderr. 1701 // c.TraceOn(os.Stderr) 1702 1703 // Set user agent. 1704 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 1705 1706 // Generate a new random bucket name. 1707 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 1708 args["bucketName"] = bucketName 1709 1710 // Make a new bucket. 1711 err = c.MakeBucket(bucketName, "us-east-1") 1712 if err != nil { 1713 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 1714 return 1715 } 1716 1717 // Upload 1 parts worth of data to use multipart upload. 1718 // Use different data in part for multipart tests to check parts are uploaded in correct order. 1719 var fName = getMintDataDirFilePath("datafile-1-MB") 1720 if fName == "" { 1721 // Make a temp file with 1 MiB bytes of data. 1722 file, err := ioutil.TempFile(os.TempDir(), "FPutObjectWithContextTest") 1723 if err != nil { 1724 logError(testName, function, args, startTime, "", "Temp file creation failed", err) 1725 return 1726 } 1727 1728 // Upload 1 parts to trigger multipart upload 1729 if _, err = io.Copy(file, getDataReader("datafile-1-MB")); err != nil { 1730 logError(testName, function, args, startTime, "", "File copy failed", err) 1731 return 1732 } 1733 1734 // Close the file pro-actively for windows. 1735 if err = file.Close(); err != nil { 1736 logError(testName, function, args, startTime, "", "File close failed", err) 1737 return 1738 } 1739 defer os.Remove(file.Name()) 1740 fName = file.Name() 1741 } 1742 totalSize := dataFileMap["datafile-1-MB"] 1743 1744 // Set base object name 1745 objectName := bucketName + "FPutObjectWithContext" 1746 args["objectName"] = objectName 1747 1748 ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) 1749 args["ctx"] = ctx 1750 defer cancel() 1751 1752 // Perform standard FPutObjectWithContext with contentType provided (Expecting application/octet-stream) 1753 _, err = c.FPutObjectWithContext(ctx, bucketName, objectName+"-Shorttimeout", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"}) 1754 if err == nil { 1755 logError(testName, function, args, startTime, "", "FPutObjectWithContext should fail on short timeout", err) 1756 return 1757 } 1758 ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) 1759 defer cancel() 1760 // Perform FPutObjectWithContext with a long timeout. Expect the put object to succeed 1761 n, err := c.FPutObjectWithContext(ctx, bucketName, objectName+"-Longtimeout", fName, minio.PutObjectOptions{}) 1762 if err != nil { 1763 logError(testName, function, args, startTime, "", "FPutObjectWithContext shouldn't fail on longer timeout", err) 1764 return 1765 } 1766 if n != int64(totalSize) { 1767 logError(testName, function, args, startTime, "", "Number of bytes does not match:wanted"+string(totalSize)+" got "+string(n), err) 1768 return 1769 } 1770 1771 _, err = c.StatObject(bucketName, objectName+"-Longtimeout", minio.StatObjectOptions{}) 1772 if err != nil { 1773 logError(testName, function, args, startTime, "", "StatObject failed", err) 1774 return 1775 } 1776 1777 // Delete all objects and buckets 1778 if err = cleanupBucket(bucketName, c); err != nil { 1779 logError(testName, function, args, startTime, "", "Cleanup failed", err) 1780 return 1781 } 1782 1783 successLogger(testName, function, args, startTime).Info() 1784 1785 } 1786 1787 // Test validates putObject with context to see if request cancellation is honored. 1788 func testPutObjectWithContext() { 1789 // initialize logging params 1790 startTime := time.Now() 1791 testName := getFuncName() 1792 function := "PutObjectWithContext(ctx, bucketName, objectName, fileName, opts)" 1793 args := map[string]interface{}{ 1794 "ctx": "", 1795 "bucketName": "", 1796 "objectName": "", 1797 "opts": "", 1798 } 1799 // Instantiate new minio client object. 1800 c, err := minio.NewV4( 1801 os.Getenv(serverEndpoint), 1802 os.Getenv(accessKey), 1803 os.Getenv(secretKey), 1804 mustParseBool(os.Getenv(enableHTTPS)), 1805 ) 1806 if err != nil { 1807 logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) 1808 return 1809 } 1810 1811 // Enable tracing, write to stderr. 1812 // c.TraceOn(os.Stderr) 1813 1814 // Set user agent. 1815 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 1816 1817 // Make a new bucket. 1818 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 1819 args["bucketName"] = bucketName 1820 1821 err = c.MakeBucket(bucketName, "us-east-1") 1822 if err != nil { 1823 logError(testName, function, args, startTime, "", "MakeBucket call failed", err) 1824 return 1825 } 1826 bufSize := dataFileMap["datafile-33-kB"] 1827 var reader = getDataReader("datafile-33-kB") 1828 defer reader.Close() 1829 objectName := fmt.Sprintf("test-file-%v", rand.Uint32()) 1830 args["objectName"] = objectName 1831 1832 ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) 1833 args["ctx"] = ctx 1834 args["opts"] = minio.PutObjectOptions{ContentType: "binary/octet-stream"} 1835 defer cancel() 1836 1837 _, err = c.PutObjectWithContext(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) 1838 if err == nil { 1839 logError(testName, function, args, startTime, "", "PutObjectWithContext should fail on short timeout", err) 1840 return 1841 } 1842 1843 ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) 1844 args["ctx"] = ctx 1845 1846 defer cancel() 1847 reader = getDataReader("datafile-33-kB") 1848 defer reader.Close() 1849 _, err = c.PutObjectWithContext(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) 1850 if err != nil { 1851 logError(testName, function, args, startTime, "", "PutObjectWithContext with long timeout failed", err) 1852 return 1853 } 1854 1855 // Delete all objects and buckets 1856 if err = cleanupBucket(bucketName, c); err != nil { 1857 logError(testName, function, args, startTime, "", "Cleanup failed", err) 1858 return 1859 } 1860 1861 successLogger(testName, function, args, startTime).Info() 1862 1863 } 1864 1865 // Tests get object ReaderSeeker interface methods. 1866 func testGetObjectReadSeekFunctional() { 1867 // initialize logging params 1868 startTime := time.Now() 1869 testName := getFuncName() 1870 function := "GetObject(bucketName, objectName)" 1871 args := map[string]interface{}{} 1872 1873 // Seed random based on current time. 1874 rand.Seed(time.Now().Unix()) 1875 1876 // Instantiate new minio client object. 1877 c, err := minio.New( 1878 os.Getenv(serverEndpoint), 1879 os.Getenv(accessKey), 1880 os.Getenv(secretKey), 1881 mustParseBool(os.Getenv(enableHTTPS)), 1882 ) 1883 if err != nil { 1884 logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) 1885 return 1886 } 1887 1888 // Enable tracing, write to stderr. 1889 // c.TraceOn(os.Stderr) 1890 1891 // Set user agent. 1892 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 1893 1894 // Generate a new random bucket name. 1895 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 1896 args["bucketName"] = bucketName 1897 1898 // Make a new bucket. 1899 err = c.MakeBucket(bucketName, "us-east-1") 1900 if err != nil { 1901 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 1902 return 1903 } 1904 1905 defer func() { 1906 // Delete all objects and buckets 1907 if err = cleanupBucket(bucketName, c); err != nil { 1908 logError(testName, function, args, startTime, "", "Cleanup failed", err) 1909 return 1910 } 1911 }() 1912 1913 // Generate 33K of data. 1914 bufSize := dataFileMap["datafile-33-kB"] 1915 var reader = getDataReader("datafile-33-kB") 1916 defer reader.Close() 1917 1918 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 1919 args["objectName"] = objectName 1920 1921 buf, err := ioutil.ReadAll(reader) 1922 if err != nil { 1923 logError(testName, function, args, startTime, "", "ReadAll failed", err) 1924 return 1925 } 1926 1927 // Save the data 1928 n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) 1929 if err != nil { 1930 logError(testName, function, args, startTime, "", "PutObject failed", err) 1931 return 1932 } 1933 1934 if n != int64(bufSize) { 1935 logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) 1936 return 1937 } 1938 1939 // Read the data back 1940 r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) 1941 if err != nil { 1942 logError(testName, function, args, startTime, "", "GetObject failed", err) 1943 return 1944 } 1945 1946 st, err := r.Stat() 1947 if err != nil { 1948 logError(testName, function, args, startTime, "", "Stat object failed", err) 1949 return 1950 } 1951 1952 if st.Size != int64(bufSize) { 1953 logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) 1954 return 1955 } 1956 1957 // This following function helps us to compare data from the reader after seek 1958 // with the data from the original buffer 1959 cmpData := func(r io.Reader, start, end int) { 1960 if end-start == 0 { 1961 return 1962 } 1963 buffer := bytes.NewBuffer([]byte{}) 1964 if _, err := io.CopyN(buffer, r, int64(bufSize)); err != nil { 1965 if err != io.EOF { 1966 logError(testName, function, args, startTime, "", "CopyN failed", err) 1967 return 1968 } 1969 } 1970 if !bytes.Equal(buf[start:end], buffer.Bytes()) { 1971 logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) 1972 return 1973 } 1974 } 1975 1976 // Generic seek error for errors other than io.EOF 1977 seekErr := errors.New("seek error") 1978 1979 testCases := []struct { 1980 offset int64 1981 whence int 1982 pos int64 1983 err error 1984 shouldCmp bool 1985 start int 1986 end int 1987 }{ 1988 // Start from offset 0, fetch data and compare 1989 {0, 0, 0, nil, true, 0, 0}, 1990 // Start from offset 2048, fetch data and compare 1991 {2048, 0, 2048, nil, true, 2048, bufSize}, 1992 // Start from offset larger than possible 1993 {int64(bufSize) + 1024, 0, 0, seekErr, false, 0, 0}, 1994 // Move to offset 0 without comparing 1995 {0, 0, 0, nil, false, 0, 0}, 1996 // Move one step forward and compare 1997 {1, 1, 1, nil, true, 1, bufSize}, 1998 // Move larger than possible 1999 {int64(bufSize), 1, 0, seekErr, false, 0, 0}, 2000 // Provide negative offset with CUR_SEEK 2001 {int64(-1), 1, 0, seekErr, false, 0, 0}, 2002 // Test with whence SEEK_END and with positive offset 2003 {1024, 2, int64(bufSize) - 1024, io.EOF, true, 0, 0}, 2004 // Test with whence SEEK_END and with negative offset 2005 {-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize}, 2006 // Test with whence SEEK_END and with large negative offset 2007 {-int64(bufSize) * 2, 2, 0, seekErr, true, 0, 0}, 2008 } 2009 2010 for i, testCase := range testCases { 2011 // Perform seek operation 2012 n, err := r.Seek(testCase.offset, testCase.whence) 2013 // We expect an error 2014 if testCase.err == seekErr && err == nil { 2015 logError(testName, function, args, startTime, "", "Test "+string(i+1)+", unexpected err value: expected: "+testCase.err.Error()+", found: "+err.Error(), err) 2016 return 2017 } 2018 // We expect a specific error 2019 if testCase.err != seekErr && testCase.err != err { 2020 logError(testName, function, args, startTime, "", "Test "+string(i+1)+", unexpected err value: expected: "+testCase.err.Error()+", found: "+err.Error(), err) 2021 return 2022 } 2023 // If we expect an error go to the next loop 2024 if testCase.err != nil { 2025 continue 2026 } 2027 // Check the returned seek pos 2028 if n != testCase.pos { 2029 logError(testName, function, args, startTime, "", "Test "+string(i+1)+", number of bytes seeked does not match, expected "+string(testCase.pos)+", got "+string(n), err) 2030 return 2031 } 2032 // Compare only if shouldCmp is activated 2033 if testCase.shouldCmp { 2034 cmpData(r, testCase.start, testCase.end) 2035 } 2036 } 2037 successLogger(testName, function, args, startTime).Info() 2038 } 2039 2040 // Tests get object ReaderAt interface methods. 2041 func testGetObjectReadAtFunctional() { 2042 // initialize logging params 2043 startTime := time.Now() 2044 testName := getFuncName() 2045 function := "GetObject(bucketName, objectName)" 2046 args := map[string]interface{}{} 2047 2048 // Seed random based on current time. 2049 rand.Seed(time.Now().Unix()) 2050 2051 // Instantiate new minio client object. 2052 c, err := minio.New( 2053 os.Getenv(serverEndpoint), 2054 os.Getenv(accessKey), 2055 os.Getenv(secretKey), 2056 mustParseBool(os.Getenv(enableHTTPS)), 2057 ) 2058 if err != nil { 2059 logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) 2060 return 2061 } 2062 2063 // Enable tracing, write to stderr. 2064 // c.TraceOn(os.Stderr) 2065 2066 // Set user agent. 2067 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 2068 2069 // Generate a new random bucket name. 2070 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 2071 args["bucketName"] = bucketName 2072 2073 // Make a new bucket. 2074 err = c.MakeBucket(bucketName, "us-east-1") 2075 if err != nil { 2076 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 2077 return 2078 } 2079 2080 // Generate 33K of data. 2081 bufSize := dataFileMap["datafile-33-kB"] 2082 var reader = getDataReader("datafile-33-kB") 2083 defer reader.Close() 2084 2085 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 2086 args["objectName"] = objectName 2087 2088 buf, err := ioutil.ReadAll(reader) 2089 if err != nil { 2090 logError(testName, function, args, startTime, "", "ReadAll failed", err) 2091 return 2092 } 2093 2094 // Save the data 2095 n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) 2096 if err != nil { 2097 logError(testName, function, args, startTime, "", "PutObject failed", err) 2098 return 2099 } 2100 2101 if n != int64(bufSize) { 2102 logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) 2103 return 2104 } 2105 2106 // read the data back 2107 r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) 2108 if err != nil { 2109 logError(testName, function, args, startTime, "", "PutObject failed", err) 2110 return 2111 } 2112 offset := int64(2048) 2113 2114 // read directly 2115 buf1 := make([]byte, 512) 2116 buf2 := make([]byte, 512) 2117 buf3 := make([]byte, 512) 2118 buf4 := make([]byte, 512) 2119 2120 // Test readAt before stat is called such that objectInfo doesn't change. 2121 m, err := r.ReadAt(buf1, offset) 2122 if err != nil { 2123 logError(testName, function, args, startTime, "", "ReadAt failed", err) 2124 return 2125 } 2126 if m != len(buf1) { 2127 logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err) 2128 return 2129 } 2130 if !bytes.Equal(buf1, buf[offset:offset+512]) { 2131 logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) 2132 return 2133 } 2134 offset += 512 2135 2136 st, err := r.Stat() 2137 if err != nil { 2138 logError(testName, function, args, startTime, "", "Stat failed", err) 2139 return 2140 } 2141 2142 if st.Size != int64(bufSize) { 2143 logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) 2144 return 2145 } 2146 2147 m, err = r.ReadAt(buf2, offset) 2148 if err != nil { 2149 logError(testName, function, args, startTime, "", "ReadAt failed", err) 2150 return 2151 } 2152 if m != len(buf2) { 2153 logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err) 2154 return 2155 } 2156 if !bytes.Equal(buf2, buf[offset:offset+512]) { 2157 logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) 2158 return 2159 } 2160 2161 offset += 512 2162 m, err = r.ReadAt(buf3, offset) 2163 if err != nil { 2164 logError(testName, function, args, startTime, "", "ReadAt failed", err) 2165 return 2166 } 2167 if m != len(buf3) { 2168 logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+", got "+string(m), err) 2169 return 2170 } 2171 if !bytes.Equal(buf3, buf[offset:offset+512]) { 2172 logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) 2173 return 2174 } 2175 offset += 512 2176 m, err = r.ReadAt(buf4, offset) 2177 if err != nil { 2178 logError(testName, function, args, startTime, "", "ReadAt failed", err) 2179 return 2180 } 2181 if m != len(buf4) { 2182 logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+", got "+string(m), err) 2183 return 2184 } 2185 if !bytes.Equal(buf4, buf[offset:offset+512]) { 2186 logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) 2187 return 2188 } 2189 2190 buf5 := make([]byte, n) 2191 // Read the whole object. 2192 m, err = r.ReadAt(buf5, 0) 2193 if err != nil { 2194 if err != io.EOF { 2195 logError(testName, function, args, startTime, "", "ReadAt failed", err) 2196 return 2197 } 2198 } 2199 if m != len(buf5) { 2200 logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+", got "+string(m), err) 2201 return 2202 } 2203 if !bytes.Equal(buf, buf5) { 2204 logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err) 2205 return 2206 } 2207 2208 buf6 := make([]byte, n+1) 2209 // Read the whole object and beyond. 2210 _, err = r.ReadAt(buf6, 0) 2211 if err != nil { 2212 if err != io.EOF { 2213 logError(testName, function, args, startTime, "", "ReadAt failed", err) 2214 return 2215 } 2216 } 2217 // Delete all objects and buckets 2218 if err = cleanupBucket(bucketName, c); err != nil { 2219 logError(testName, function, args, startTime, "", "Cleanup failed", err) 2220 return 2221 } 2222 successLogger(testName, function, args, startTime).Info() 2223 } 2224 2225 // Reproduces issue https://github.com/minio/minio-go/issues/1137 2226 func testGetObjectReadAtWhenEOFWasReached() { 2227 // initialize logging params 2228 startTime := time.Now() 2229 testName := getFuncName() 2230 function := "GetObject(bucketName, objectName)" 2231 args := map[string]interface{}{} 2232 2233 // Seed random based on current time. 2234 rand.Seed(time.Now().Unix()) 2235 2236 // Instantiate new minio client object. 2237 c, err := minio.New( 2238 os.Getenv(serverEndpoint), 2239 os.Getenv(accessKey), 2240 os.Getenv(secretKey), 2241 mustParseBool(os.Getenv(enableHTTPS)), 2242 ) 2243 if err != nil { 2244 logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) 2245 return 2246 } 2247 2248 // Enable tracing, write to stderr. 2249 // c.TraceOn(os.Stderr) 2250 2251 // Set user agent. 2252 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 2253 2254 // Generate a new random bucket name. 2255 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 2256 args["bucketName"] = bucketName 2257 2258 // Make a new bucket. 2259 err = c.MakeBucket(bucketName, "us-east-1") 2260 if err != nil { 2261 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 2262 return 2263 } 2264 2265 // Generate 33K of data. 2266 bufSize := dataFileMap["datafile-33-kB"] 2267 var reader = getDataReader("datafile-33-kB") 2268 defer reader.Close() 2269 2270 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 2271 args["objectName"] = objectName 2272 2273 buf, err := ioutil.ReadAll(reader) 2274 if err != nil { 2275 logError(testName, function, args, startTime, "", "ReadAll failed", err) 2276 return 2277 } 2278 2279 // Save the data 2280 n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) 2281 if err != nil { 2282 logError(testName, function, args, startTime, "", "PutObject failed", err) 2283 return 2284 } 2285 2286 if n != int64(bufSize) { 2287 logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) 2288 return 2289 } 2290 2291 // read the data back 2292 r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) 2293 if err != nil { 2294 logError(testName, function, args, startTime, "", "PutObject failed", err) 2295 return 2296 } 2297 2298 // read directly 2299 buf1 := make([]byte, n) 2300 buf2 := make([]byte, 512) 2301 2302 m, err := r.Read(buf1) 2303 if err != nil { 2304 if err != io.EOF { 2305 logError(testName, function, args, startTime, "", "Read failed", err) 2306 return 2307 } 2308 } 2309 if m != len(buf1) { 2310 logError(testName, function, args, startTime, "", "Read read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err) 2311 return 2312 } 2313 if !bytes.Equal(buf1, buf) { 2314 logError(testName, function, args, startTime, "", "Incorrect count of Read data", err) 2315 return 2316 } 2317 2318 st, err := r.Stat() 2319 if err != nil { 2320 logError(testName, function, args, startTime, "", "Stat failed", err) 2321 return 2322 } 2323 2324 if st.Size != int64(bufSize) { 2325 logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) 2326 return 2327 } 2328 2329 m, err = r.ReadAt(buf2, 512) 2330 if err != nil { 2331 logError(testName, function, args, startTime, "", "ReadAt failed", err) 2332 return 2333 } 2334 if m != len(buf2) { 2335 logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err) 2336 return 2337 } 2338 if !bytes.Equal(buf2, buf[512:1024]) { 2339 logError(testName, function, args, startTime, "", "Incorrect count of ReadAt data", err) 2340 return 2341 } 2342 2343 // Delete all objects and buckets 2344 if err = cleanupBucket(bucketName, c); err != nil { 2345 logError(testName, function, args, startTime, "", "Cleanup failed", err) 2346 return 2347 } 2348 2349 successLogger(testName, function, args, startTime).Info() 2350 } 2351 2352 // Test Presigned Post Policy 2353 func testPresignedPostPolicy() { 2354 // initialize logging params 2355 startTime := time.Now() 2356 testName := getFuncName() 2357 function := "PresignedPostPolicy(policy)" 2358 args := map[string]interface{}{ 2359 "policy": "", 2360 } 2361 2362 // Seed random based on current time. 2363 rand.Seed(time.Now().Unix()) 2364 2365 // Instantiate new minio client object 2366 c, err := minio.NewV4( 2367 os.Getenv(serverEndpoint), 2368 os.Getenv(accessKey), 2369 os.Getenv(secretKey), 2370 mustParseBool(os.Getenv(enableHTTPS)), 2371 ) 2372 if err != nil { 2373 logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) 2374 return 2375 } 2376 2377 // Enable tracing, write to stderr. 2378 // c.TraceOn(os.Stderr) 2379 2380 // Set user agent. 2381 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 2382 2383 // Generate a new random bucket name. 2384 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 2385 2386 // Make a new bucket in 'us-east-1' (source bucket). 2387 err = c.MakeBucket(bucketName, "us-east-1") 2388 if err != nil { 2389 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 2390 return 2391 } 2392 2393 // Generate 33K of data. 2394 bufSize := dataFileMap["datafile-33-kB"] 2395 var reader = getDataReader("datafile-33-kB") 2396 defer reader.Close() 2397 2398 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 2399 // Azure requires the key to not start with a number 2400 metadataKey := randString(60, rand.NewSource(time.Now().UnixNano()), "user") 2401 metadataValue := randString(60, rand.NewSource(time.Now().UnixNano()), "") 2402 2403 buf, err := ioutil.ReadAll(reader) 2404 if err != nil { 2405 logError(testName, function, args, startTime, "", "ReadAll failed", err) 2406 return 2407 } 2408 2409 // Save the data 2410 n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) 2411 if err != nil { 2412 logError(testName, function, args, startTime, "", "PutObject failed", err) 2413 return 2414 } 2415 2416 if n != int64(bufSize) { 2417 logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+" got "+string(n), err) 2418 return 2419 } 2420 2421 policy := minio.NewPostPolicy() 2422 2423 if err := policy.SetBucket(""); err == nil { 2424 logError(testName, function, args, startTime, "", "SetBucket did not fail for invalid conditions", err) 2425 return 2426 } 2427 if err := policy.SetKey(""); err == nil { 2428 logError(testName, function, args, startTime, "", "SetKey did not fail for invalid conditions", err) 2429 return 2430 } 2431 if err := policy.SetKeyStartsWith(""); err == nil { 2432 logError(testName, function, args, startTime, "", "SetKeyStartsWith did not fail for invalid conditions", err) 2433 return 2434 } 2435 if err := policy.SetExpires(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)); err == nil { 2436 logError(testName, function, args, startTime, "", "SetExpires did not fail for invalid conditions", err) 2437 return 2438 } 2439 if err := policy.SetContentType(""); err == nil { 2440 logError(testName, function, args, startTime, "", "SetContentType did not fail for invalid conditions", err) 2441 return 2442 } 2443 if err := policy.SetContentLengthRange(1024*1024, 1024); err == nil { 2444 logError(testName, function, args, startTime, "", "SetContentLengthRange did not fail for invalid conditions", err) 2445 return 2446 } 2447 if err := policy.SetUserMetadata("", ""); err == nil { 2448 logError(testName, function, args, startTime, "", "SetUserMetadata did not fail for invalid conditions", err) 2449 return 2450 } 2451 2452 policy.SetBucket(bucketName) 2453 policy.SetKey(objectName) 2454 policy.SetExpires(time.Now().UTC().AddDate(0, 0, 10)) // expires in 10 days 2455 policy.SetContentType("binary/octet-stream") 2456 policy.SetContentLengthRange(10, 1024*1024) 2457 policy.SetUserMetadata(metadataKey, metadataValue) 2458 args["policy"] = policy.String() 2459 2460 presignedPostPolicyURL, formData, err := c.PresignedPostPolicy(policy) 2461 if err != nil { 2462 logError(testName, function, args, startTime, "", "PresignedPostPolicy failed", err) 2463 return 2464 } 2465 2466 var formBuf bytes.Buffer 2467 writer := multipart.NewWriter(&formBuf) 2468 for k, v := range formData { 2469 writer.WriteField(k, v) 2470 } 2471 2472 // Get a 33KB file to upload and test if set post policy works 2473 var filePath = getMintDataDirFilePath("datafile-33-kB") 2474 if filePath == "" { 2475 // Make a temp file with 33 KB data. 2476 file, err := ioutil.TempFile(os.TempDir(), "PresignedPostPolicyTest") 2477 if err != nil { 2478 logError(testName, function, args, startTime, "", "TempFile creation failed", err) 2479 return 2480 } 2481 if _, err = io.Copy(file, getDataReader("datafile-33-kB")); err != nil { 2482 logError(testName, function, args, startTime, "", "Copy failed", err) 2483 return 2484 } 2485 if err = file.Close(); err != nil { 2486 logError(testName, function, args, startTime, "", "File Close failed", err) 2487 return 2488 } 2489 filePath = file.Name() 2490 } 2491 2492 // add file to post request 2493 f, err := os.Open(filePath) 2494 defer f.Close() 2495 if err != nil { 2496 logError(testName, function, args, startTime, "", "File open failed", err) 2497 return 2498 } 2499 w, err := writer.CreateFormFile("file", filePath) 2500 if err != nil { 2501 logError(testName, function, args, startTime, "", "CreateFormFile failed", err) 2502 return 2503 } 2504 2505 _, err = io.Copy(w, f) 2506 if err != nil { 2507 logError(testName, function, args, startTime, "", "Copy failed", err) 2508 return 2509 } 2510 writer.Close() 2511 2512 transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS))) 2513 if err != nil { 2514 logError(testName, function, args, startTime, "", "DefaultTransport failed", err) 2515 return 2516 } 2517 2518 httpClient := &http.Client{ 2519 // Setting a sensible time out of 30secs to wait for response 2520 // headers. Request is pro-actively canceled after 30secs 2521 // with no response. 2522 Timeout: 30 * time.Second, 2523 Transport: transport, 2524 } 2525 2526 req, err := http.NewRequest(http.MethodPost, presignedPostPolicyURL.String(), bytes.NewReader(formBuf.Bytes())) 2527 if err != nil { 2528 logError(testName, function, args, startTime, "", "Http request failed", err) 2529 return 2530 } 2531 2532 req.Header.Set("Content-Type", writer.FormDataContentType()) 2533 2534 // make post request with correct form data 2535 res, err := httpClient.Do(req) 2536 if err != nil { 2537 logError(testName, function, args, startTime, "", "Http request failed", err) 2538 return 2539 } 2540 defer res.Body.Close() 2541 if res.StatusCode != http.StatusNoContent { 2542 logError(testName, function, args, startTime, "", "Http request failed", errors.New(res.Status)) 2543 return 2544 } 2545 2546 // expected path should be absolute path of the object 2547 var scheme string 2548 if mustParseBool(os.Getenv(enableHTTPS)) { 2549 scheme = "https://" 2550 } else { 2551 scheme = "http://" 2552 } 2553 2554 expectedLocation := scheme + os.Getenv(serverEndpoint) + "/" + bucketName + "/" + objectName 2555 expectedLocationBucketDNS := scheme + bucketName + "." + os.Getenv(serverEndpoint) + "/" + objectName 2556 2557 if val, ok := res.Header["Location"]; ok { 2558 if val[0] != expectedLocation && val[0] != expectedLocationBucketDNS { 2559 logError(testName, function, args, startTime, "", "Location in header response is incorrect", err) 2560 return 2561 } 2562 } else { 2563 logError(testName, function, args, startTime, "", "Location not found in header response", err) 2564 return 2565 } 2566 2567 // Delete all objects and buckets 2568 if err = cleanupBucket(bucketName, c); err != nil { 2569 logError(testName, function, args, startTime, "", "Cleanup failed", err) 2570 return 2571 } 2572 2573 successLogger(testName, function, args, startTime).Info() 2574 } 2575 2576 // Tests copy object 2577 func testCopyObject() { 2578 // initialize logging params 2579 startTime := time.Now() 2580 testName := getFuncName() 2581 function := "CopyObject(dst, src)" 2582 args := map[string]interface{}{} 2583 2584 // Seed random based on current time. 2585 rand.Seed(time.Now().Unix()) 2586 2587 // Instantiate new minio client object 2588 c, err := minio.NewV4( 2589 os.Getenv(serverEndpoint), 2590 os.Getenv(accessKey), 2591 os.Getenv(secretKey), 2592 mustParseBool(os.Getenv(enableHTTPS)), 2593 ) 2594 if err != nil { 2595 logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) 2596 return 2597 } 2598 2599 // Enable tracing, write to stderr. 2600 // c.TraceOn(os.Stderr) 2601 2602 // Set user agent. 2603 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 2604 2605 // Generate a new random bucket name. 2606 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 2607 2608 // Make a new bucket in 'us-east-1' (source bucket). 2609 err = c.MakeBucket(bucketName, "us-east-1") 2610 if err != nil { 2611 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 2612 return 2613 } 2614 2615 // Make a new bucket in 'us-east-1' (destination bucket). 2616 err = c.MakeBucket(bucketName+"-copy", "us-east-1") 2617 if err != nil { 2618 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 2619 return 2620 } 2621 2622 // Generate 33K of data. 2623 bufSize := dataFileMap["datafile-33-kB"] 2624 var reader = getDataReader("datafile-33-kB") 2625 2626 // Save the data 2627 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 2628 n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) 2629 if err != nil { 2630 logError(testName, function, args, startTime, "", "PutObject failed", err) 2631 return 2632 } 2633 2634 if n != int64(bufSize) { 2635 logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) 2636 return 2637 } 2638 2639 r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) 2640 if err != nil { 2641 logError(testName, function, args, startTime, "", "GetObject failed", err) 2642 return 2643 } 2644 // Check the various fields of source object against destination object. 2645 objInfo, err := r.Stat() 2646 if err != nil { 2647 logError(testName, function, args, startTime, "", "Stat failed", err) 2648 return 2649 } 2650 2651 // Copy Source 2652 src := minio.NewSourceInfo(bucketName, objectName, nil) 2653 args["src"] = src 2654 2655 // Set copy conditions. 2656 2657 // All invalid conditions first. 2658 err = src.SetModifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) 2659 if err == nil { 2660 logError(testName, function, args, startTime, "", "SetModifiedSinceCond did not fail for invalid conditions", err) 2661 return 2662 } 2663 err = src.SetUnmodifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) 2664 if err == nil { 2665 logError(testName, function, args, startTime, "", "SetUnmodifiedSinceCond did not fail for invalid conditions", err) 2666 return 2667 } 2668 err = src.SetMatchETagCond("") 2669 if err == nil { 2670 logError(testName, function, args, startTime, "", "SetMatchETagCond did not fail for invalid conditions", err) 2671 return 2672 } 2673 err = src.SetMatchETagExceptCond("") 2674 if err == nil { 2675 logError(testName, function, args, startTime, "", "SetMatchETagExceptCond did not fail for invalid conditions", err) 2676 return 2677 } 2678 2679 err = src.SetModifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) 2680 if err != nil { 2681 logError(testName, function, args, startTime, "", "SetModifiedSinceCond failed", err) 2682 return 2683 } 2684 err = src.SetMatchETagCond(objInfo.ETag) 2685 if err != nil { 2686 logError(testName, function, args, startTime, "", "SetMatchETagCond failed", err) 2687 return 2688 } 2689 2690 dst, err := minio.NewDestinationInfo(bucketName+"-copy", objectName+"-copy", nil, nil) 2691 args["dst"] = dst 2692 if err != nil { 2693 logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) 2694 return 2695 } 2696 2697 // Perform the Copy 2698 err = c.CopyObject(dst, src) 2699 if err != nil { 2700 logError(testName, function, args, startTime, "", "CopyObject failed", err) 2701 return 2702 } 2703 2704 // Source object 2705 r, err = c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) 2706 if err != nil { 2707 logError(testName, function, args, startTime, "", "GetObject failed", err) 2708 return 2709 } 2710 2711 // Destination object 2712 readerCopy, err := c.GetObject(bucketName+"-copy", objectName+"-copy", minio.GetObjectOptions{}) 2713 if err != nil { 2714 logError(testName, function, args, startTime, "", "GetObject failed", err) 2715 return 2716 } 2717 // Check the various fields of source object against destination object. 2718 objInfo, err = r.Stat() 2719 if err != nil { 2720 logError(testName, function, args, startTime, "", "Stat failed", err) 2721 return 2722 } 2723 objInfoCopy, err := readerCopy.Stat() 2724 if err != nil { 2725 logError(testName, function, args, startTime, "", "Stat failed", err) 2726 return 2727 } 2728 if objInfo.Size != objInfoCopy.Size { 2729 logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(objInfoCopy.Size)+", got "+string(objInfo.Size), err) 2730 return 2731 } 2732 2733 // Close all the get readers before proceeding with CopyObject operations. 2734 r.Close() 2735 readerCopy.Close() 2736 2737 // CopyObject again but with wrong conditions 2738 src = minio.NewSourceInfo(bucketName, objectName, nil) 2739 err = src.SetUnmodifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) 2740 if err != nil { 2741 logError(testName, function, args, startTime, "", "SetUnmodifiedSinceCond failed", err) 2742 return 2743 } 2744 err = src.SetMatchETagExceptCond(objInfo.ETag) 2745 if err != nil { 2746 logError(testName, function, args, startTime, "", "SetMatchETagExceptCond failed", err) 2747 return 2748 } 2749 2750 // Perform the Copy which should fail 2751 err = c.CopyObject(dst, src) 2752 if err == nil { 2753 logError(testName, function, args, startTime, "", "CopyObject did not fail for invalid conditions", err) 2754 return 2755 } 2756 2757 // Perform the Copy which should update only metadata. 2758 src = minio.NewSourceInfo(bucketName, objectName, nil) 2759 dst, err = minio.NewDestinationInfo(bucketName, objectName, nil, map[string]string{ 2760 "Copy": "should be same", 2761 }) 2762 args["dst"] = dst 2763 args["src"] = src 2764 if err != nil { 2765 logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) 2766 return 2767 } 2768 2769 err = c.CopyObject(dst, src) 2770 if err != nil { 2771 logError(testName, function, args, startTime, "", "CopyObject shouldn't fail", err) 2772 return 2773 } 2774 2775 oi, err := c.StatObject(bucketName, objectName, minio.StatObjectOptions{}) 2776 if err != nil { 2777 logError(testName, function, args, startTime, "", "StatObject failed", err) 2778 return 2779 } 2780 2781 stOpts := minio.StatObjectOptions{} 2782 stOpts.SetMatchETag(oi.ETag) 2783 objInfo, err = c.StatObject(bucketName, objectName, stOpts) 2784 if err != nil { 2785 logError(testName, function, args, startTime, "", "CopyObject ETag should match and not fail", err) 2786 return 2787 } 2788 2789 if objInfo.Metadata.Get("x-amz-meta-copy") != "should be same" { 2790 logError(testName, function, args, startTime, "", "CopyObject modified metadata should match", err) 2791 return 2792 } 2793 2794 // Delete all objects and buckets 2795 if err = cleanupBucket(bucketName, c); err != nil { 2796 logError(testName, function, args, startTime, "", "Cleanup failed", err) 2797 return 2798 } 2799 if err = cleanupBucket(bucketName+"-copy", c); err != nil { 2800 logError(testName, function, args, startTime, "", "Cleanup failed", err) 2801 return 2802 } 2803 successLogger(testName, function, args, startTime).Info() 2804 } 2805 2806 // Tests SSE-C get object ReaderSeeker interface methods. 2807 func testSSECEncryptedGetObjectReadSeekFunctional() { 2808 // initialize logging params 2809 startTime := time.Now() 2810 testName := getFuncName() 2811 function := "GetObject(bucketName, objectName)" 2812 args := map[string]interface{}{} 2813 2814 // Seed random based on current time. 2815 rand.Seed(time.Now().Unix()) 2816 2817 // Instantiate new minio client object. 2818 c, err := minio.New( 2819 os.Getenv(serverEndpoint), 2820 os.Getenv(accessKey), 2821 os.Getenv(secretKey), 2822 mustParseBool(os.Getenv(enableHTTPS)), 2823 ) 2824 if err != nil { 2825 logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) 2826 return 2827 } 2828 2829 // Enable tracing, write to stderr. 2830 // c.TraceOn(os.Stderr) 2831 2832 // Set user agent. 2833 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 2834 2835 // Generate a new random bucket name. 2836 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 2837 args["bucketName"] = bucketName 2838 2839 // Make a new bucket. 2840 err = c.MakeBucket(bucketName, "us-east-1") 2841 if err != nil { 2842 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 2843 return 2844 } 2845 2846 defer func() { 2847 // Delete all objects and buckets 2848 if err = cleanupBucket(bucketName, c); err != nil { 2849 logError(testName, function, args, startTime, "", "Cleanup failed", err) 2850 return 2851 } 2852 }() 2853 2854 // Generate 129MiB of data. 2855 bufSize := dataFileMap["datafile-129-MB"] 2856 var reader = getDataReader("datafile-129-MB") 2857 defer reader.Close() 2858 2859 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 2860 args["objectName"] = objectName 2861 2862 buf, err := ioutil.ReadAll(reader) 2863 if err != nil { 2864 logError(testName, function, args, startTime, "", "ReadAll failed", err) 2865 return 2866 } 2867 2868 // Save the data 2869 n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ 2870 ContentType: "binary/octet-stream", 2871 ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), 2872 }) 2873 if err != nil { 2874 logError(testName, function, args, startTime, "", "PutObject failed", err) 2875 return 2876 } 2877 2878 if n != int64(bufSize) { 2879 logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) 2880 return 2881 } 2882 2883 // Read the data back 2884 r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{ 2885 ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), 2886 }) 2887 if err != nil { 2888 logError(testName, function, args, startTime, "", "GetObject failed", err) 2889 return 2890 } 2891 defer r.Close() 2892 2893 st, err := r.Stat() 2894 if err != nil { 2895 logError(testName, function, args, startTime, "", "Stat object failed", err) 2896 return 2897 } 2898 2899 if st.Size != int64(bufSize) { 2900 logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) 2901 return 2902 } 2903 2904 // This following function helps us to compare data from the reader after seek 2905 // with the data from the original buffer 2906 cmpData := func(r io.Reader, start, end int) { 2907 if end-start == 0 { 2908 return 2909 } 2910 buffer := bytes.NewBuffer([]byte{}) 2911 if _, err := io.CopyN(buffer, r, int64(bufSize)); err != nil { 2912 if err != io.EOF { 2913 logError(testName, function, args, startTime, "", "CopyN failed", err) 2914 return 2915 } 2916 } 2917 if !bytes.Equal(buf[start:end], buffer.Bytes()) { 2918 logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) 2919 return 2920 } 2921 } 2922 2923 testCases := []struct { 2924 offset int64 2925 whence int 2926 pos int64 2927 err error 2928 shouldCmp bool 2929 start int 2930 end int 2931 }{ 2932 // Start from offset 0, fetch data and compare 2933 {0, 0, 0, nil, true, 0, 0}, 2934 // Start from offset 2048, fetch data and compare 2935 {2048, 0, 2048, nil, true, 2048, bufSize}, 2936 // Start from offset larger than possible 2937 {int64(bufSize) + 1024, 0, 0, io.EOF, false, 0, 0}, 2938 // Move to offset 0 without comparing 2939 {0, 0, 0, nil, false, 0, 0}, 2940 // Move one step forward and compare 2941 {1, 1, 1, nil, true, 1, bufSize}, 2942 // Move larger than possible 2943 {int64(bufSize), 1, 0, io.EOF, false, 0, 0}, 2944 // Provide negative offset with CUR_SEEK 2945 {int64(-1), 1, 0, fmt.Errorf("Negative position not allowed for 1"), false, 0, 0}, 2946 // Test with whence SEEK_END and with positive offset 2947 {1024, 2, 0, io.EOF, false, 0, 0}, 2948 // Test with whence SEEK_END and with negative offset 2949 {-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize}, 2950 // Test with whence SEEK_END and with large negative offset 2951 {-int64(bufSize) * 2, 2, 0, fmt.Errorf("Seeking at negative offset not allowed for 2"), false, 0, 0}, 2952 // Test with invalid whence 2953 {0, 3, 0, fmt.Errorf("Invalid whence 3"), false, 0, 0}, 2954 } 2955 2956 for i, testCase := range testCases { 2957 // Perform seek operation 2958 n, err := r.Seek(testCase.offset, testCase.whence) 2959 if err != nil && testCase.err == nil { 2960 // We expected success. 2961 logError(testName, function, args, startTime, "", 2962 fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) 2963 return 2964 } 2965 if err == nil && testCase.err != nil { 2966 // We expected failure, but got success. 2967 logError(testName, function, args, startTime, "", 2968 fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) 2969 return 2970 } 2971 if err != nil && testCase.err != nil { 2972 if err.Error() != testCase.err.Error() { 2973 // We expect a specific error 2974 logError(testName, function, args, startTime, "", 2975 fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) 2976 return 2977 } 2978 } 2979 // Check the returned seek pos 2980 if n != testCase.pos { 2981 logError(testName, function, args, startTime, "", 2982 fmt.Sprintf("Test %d, number of bytes seeked does not match, expected %d, got %d", i+1, testCase.pos, n), err) 2983 return 2984 } 2985 // Compare only if shouldCmp is activated 2986 if testCase.shouldCmp { 2987 cmpData(r, testCase.start, testCase.end) 2988 } 2989 } 2990 2991 successLogger(testName, function, args, startTime).Info() 2992 } 2993 2994 // Tests SSE-S3 get object ReaderSeeker interface methods. 2995 func testSSES3EncryptedGetObjectReadSeekFunctional() { 2996 // initialize logging params 2997 startTime := time.Now() 2998 testName := getFuncName() 2999 function := "GetObject(bucketName, objectName)" 3000 args := map[string]interface{}{} 3001 3002 // Seed random based on current time. 3003 rand.Seed(time.Now().Unix()) 3004 3005 // Instantiate new minio client object. 3006 c, err := minio.New( 3007 os.Getenv(serverEndpoint), 3008 os.Getenv(accessKey), 3009 os.Getenv(secretKey), 3010 mustParseBool(os.Getenv(enableHTTPS)), 3011 ) 3012 if err != nil { 3013 logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) 3014 return 3015 } 3016 3017 // Enable tracing, write to stderr. 3018 // c.TraceOn(os.Stderr) 3019 3020 // Set user agent. 3021 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 3022 3023 // Generate a new random bucket name. 3024 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 3025 args["bucketName"] = bucketName 3026 3027 // Make a new bucket. 3028 err = c.MakeBucket(bucketName, "us-east-1") 3029 if err != nil { 3030 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 3031 return 3032 } 3033 3034 defer func() { 3035 // Delete all objects and buckets 3036 if err = cleanupBucket(bucketName, c); err != nil { 3037 logError(testName, function, args, startTime, "", "Cleanup failed", err) 3038 return 3039 } 3040 }() 3041 3042 // Generate 129MiB of data. 3043 bufSize := dataFileMap["datafile-129-MB"] 3044 var reader = getDataReader("datafile-129-MB") 3045 defer reader.Close() 3046 3047 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 3048 args["objectName"] = objectName 3049 3050 buf, err := ioutil.ReadAll(reader) 3051 if err != nil { 3052 logError(testName, function, args, startTime, "", "ReadAll failed", err) 3053 return 3054 } 3055 3056 // Save the data 3057 n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ 3058 ContentType: "binary/octet-stream", 3059 ServerSideEncryption: encrypt.NewSSE(), 3060 }) 3061 if err != nil { 3062 logError(testName, function, args, startTime, "", "PutObject failed", err) 3063 return 3064 } 3065 3066 if n != int64(bufSize) { 3067 logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) 3068 return 3069 } 3070 3071 // Read the data back 3072 r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) 3073 if err != nil { 3074 logError(testName, function, args, startTime, "", "GetObject failed", err) 3075 return 3076 } 3077 defer r.Close() 3078 3079 st, err := r.Stat() 3080 if err != nil { 3081 logError(testName, function, args, startTime, "", "Stat object failed", err) 3082 return 3083 } 3084 3085 if st.Size != int64(bufSize) { 3086 logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) 3087 return 3088 } 3089 3090 // This following function helps us to compare data from the reader after seek 3091 // with the data from the original buffer 3092 cmpData := func(r io.Reader, start, end int) { 3093 if end-start == 0 { 3094 return 3095 } 3096 buffer := bytes.NewBuffer([]byte{}) 3097 if _, err := io.CopyN(buffer, r, int64(bufSize)); err != nil { 3098 if err != io.EOF { 3099 logError(testName, function, args, startTime, "", "CopyN failed", err) 3100 return 3101 } 3102 } 3103 if !bytes.Equal(buf[start:end], buffer.Bytes()) { 3104 logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) 3105 return 3106 } 3107 } 3108 3109 testCases := []struct { 3110 offset int64 3111 whence int 3112 pos int64 3113 err error 3114 shouldCmp bool 3115 start int 3116 end int 3117 }{ 3118 // Start from offset 0, fetch data and compare 3119 {0, 0, 0, nil, true, 0, 0}, 3120 // Start from offset 2048, fetch data and compare 3121 {2048, 0, 2048, nil, true, 2048, bufSize}, 3122 // Start from offset larger than possible 3123 {int64(bufSize) + 1024, 0, 0, io.EOF, false, 0, 0}, 3124 // Move to offset 0 without comparing 3125 {0, 0, 0, nil, false, 0, 0}, 3126 // Move one step forward and compare 3127 {1, 1, 1, nil, true, 1, bufSize}, 3128 // Move larger than possible 3129 {int64(bufSize), 1, 0, io.EOF, false, 0, 0}, 3130 // Provide negative offset with CUR_SEEK 3131 {int64(-1), 1, 0, fmt.Errorf("Negative position not allowed for 1"), false, 0, 0}, 3132 // Test with whence SEEK_END and with positive offset 3133 {1024, 2, 0, io.EOF, false, 0, 0}, 3134 // Test with whence SEEK_END and with negative offset 3135 {-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize}, 3136 // Test with whence SEEK_END and with large negative offset 3137 {-int64(bufSize) * 2, 2, 0, fmt.Errorf("Seeking at negative offset not allowed for 2"), false, 0, 0}, 3138 // Test with invalid whence 3139 {0, 3, 0, fmt.Errorf("Invalid whence 3"), false, 0, 0}, 3140 } 3141 3142 for i, testCase := range testCases { 3143 // Perform seek operation 3144 n, err := r.Seek(testCase.offset, testCase.whence) 3145 if err != nil && testCase.err == nil { 3146 // We expected success. 3147 logError(testName, function, args, startTime, "", 3148 fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) 3149 return 3150 } 3151 if err == nil && testCase.err != nil { 3152 // We expected failure, but got success. 3153 logError(testName, function, args, startTime, "", 3154 fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) 3155 return 3156 } 3157 if err != nil && testCase.err != nil { 3158 if err.Error() != testCase.err.Error() { 3159 // We expect a specific error 3160 logError(testName, function, args, startTime, "", 3161 fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) 3162 return 3163 } 3164 } 3165 // Check the returned seek pos 3166 if n != testCase.pos { 3167 logError(testName, function, args, startTime, "", 3168 fmt.Sprintf("Test %d, number of bytes seeked does not match, expected %d, got %d", i+1, testCase.pos, n), err) 3169 return 3170 } 3171 // Compare only if shouldCmp is activated 3172 if testCase.shouldCmp { 3173 cmpData(r, testCase.start, testCase.end) 3174 } 3175 } 3176 3177 successLogger(testName, function, args, startTime).Info() 3178 } 3179 3180 // Tests SSE-C get object ReaderAt interface methods. 3181 func testSSECEncryptedGetObjectReadAtFunctional() { 3182 // initialize logging params 3183 startTime := time.Now() 3184 testName := getFuncName() 3185 function := "GetObject(bucketName, objectName)" 3186 args := map[string]interface{}{} 3187 3188 // Seed random based on current time. 3189 rand.Seed(time.Now().Unix()) 3190 3191 // Instantiate new minio client object. 3192 c, err := minio.New( 3193 os.Getenv(serverEndpoint), 3194 os.Getenv(accessKey), 3195 os.Getenv(secretKey), 3196 mustParseBool(os.Getenv(enableHTTPS)), 3197 ) 3198 if err != nil { 3199 logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) 3200 return 3201 } 3202 3203 // Enable tracing, write to stderr. 3204 // c.TraceOn(os.Stderr) 3205 3206 // Set user agent. 3207 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 3208 3209 // Generate a new random bucket name. 3210 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 3211 args["bucketName"] = bucketName 3212 3213 // Make a new bucket. 3214 err = c.MakeBucket(bucketName, "us-east-1") 3215 if err != nil { 3216 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 3217 return 3218 } 3219 3220 // Generate 129MiB of data. 3221 bufSize := dataFileMap["datafile-129-MB"] 3222 var reader = getDataReader("datafile-129-MB") 3223 defer reader.Close() 3224 3225 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 3226 args["objectName"] = objectName 3227 3228 buf, err := ioutil.ReadAll(reader) 3229 if err != nil { 3230 logError(testName, function, args, startTime, "", "ReadAll failed", err) 3231 return 3232 } 3233 3234 // Save the data 3235 n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ 3236 ContentType: "binary/octet-stream", 3237 ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), 3238 }) 3239 if err != nil { 3240 logError(testName, function, args, startTime, "", "PutObject failed", err) 3241 return 3242 } 3243 3244 if n != int64(bufSize) { 3245 logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) 3246 return 3247 } 3248 3249 // read the data back 3250 r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{ 3251 ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), 3252 }) 3253 if err != nil { 3254 logError(testName, function, args, startTime, "", "PutObject failed", err) 3255 return 3256 } 3257 defer r.Close() 3258 3259 offset := int64(2048) 3260 3261 // read directly 3262 buf1 := make([]byte, 512) 3263 buf2 := make([]byte, 512) 3264 buf3 := make([]byte, 512) 3265 buf4 := make([]byte, 512) 3266 3267 // Test readAt before stat is called such that objectInfo doesn't change. 3268 m, err := r.ReadAt(buf1, offset) 3269 if err != nil { 3270 logError(testName, function, args, startTime, "", "ReadAt failed", err) 3271 return 3272 } 3273 if m != len(buf1) { 3274 logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err) 3275 return 3276 } 3277 if !bytes.Equal(buf1, buf[offset:offset+512]) { 3278 logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) 3279 return 3280 } 3281 offset += 512 3282 3283 st, err := r.Stat() 3284 if err != nil { 3285 logError(testName, function, args, startTime, "", "Stat failed", err) 3286 return 3287 } 3288 3289 if st.Size != int64(bufSize) { 3290 logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) 3291 return 3292 } 3293 3294 m, err = r.ReadAt(buf2, offset) 3295 if err != nil { 3296 logError(testName, function, args, startTime, "", "ReadAt failed", err) 3297 return 3298 } 3299 if m != len(buf2) { 3300 logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err) 3301 return 3302 } 3303 if !bytes.Equal(buf2, buf[offset:offset+512]) { 3304 logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) 3305 return 3306 } 3307 offset += 512 3308 m, err = r.ReadAt(buf3, offset) 3309 if err != nil { 3310 logError(testName, function, args, startTime, "", "ReadAt failed", err) 3311 return 3312 } 3313 if m != len(buf3) { 3314 logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+", got "+string(m), err) 3315 return 3316 } 3317 if !bytes.Equal(buf3, buf[offset:offset+512]) { 3318 logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) 3319 return 3320 } 3321 offset += 512 3322 m, err = r.ReadAt(buf4, offset) 3323 if err != nil { 3324 logError(testName, function, args, startTime, "", "ReadAt failed", err) 3325 return 3326 } 3327 if m != len(buf4) { 3328 logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+", got "+string(m), err) 3329 return 3330 } 3331 if !bytes.Equal(buf4, buf[offset:offset+512]) { 3332 logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) 3333 return 3334 } 3335 3336 buf5 := make([]byte, n) 3337 // Read the whole object. 3338 m, err = r.ReadAt(buf5, 0) 3339 if err != nil { 3340 if err != io.EOF { 3341 logError(testName, function, args, startTime, "", "ReadAt failed", err) 3342 return 3343 } 3344 } 3345 if m != len(buf5) { 3346 logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+", got "+string(m), err) 3347 return 3348 } 3349 if !bytes.Equal(buf, buf5) { 3350 logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err) 3351 return 3352 } 3353 3354 buf6 := make([]byte, n+1) 3355 // Read the whole object and beyond. 3356 _, err = r.ReadAt(buf6, 0) 3357 if err != nil { 3358 if err != io.EOF { 3359 logError(testName, function, args, startTime, "", "ReadAt failed", err) 3360 return 3361 } 3362 } 3363 // Delete all objects and buckets 3364 if err = cleanupBucket(bucketName, c); err != nil { 3365 logError(testName, function, args, startTime, "", "Cleanup failed", err) 3366 return 3367 } 3368 successLogger(testName, function, args, startTime).Info() 3369 } 3370 3371 // Tests SSE-S3 get object ReaderAt interface methods. 3372 func testSSES3EncryptedGetObjectReadAtFunctional() { 3373 // initialize logging params 3374 startTime := time.Now() 3375 testName := getFuncName() 3376 function := "GetObject(bucketName, objectName)" 3377 args := map[string]interface{}{} 3378 3379 // Seed random based on current time. 3380 rand.Seed(time.Now().Unix()) 3381 3382 // Instantiate new minio client object. 3383 c, err := minio.New( 3384 os.Getenv(serverEndpoint), 3385 os.Getenv(accessKey), 3386 os.Getenv(secretKey), 3387 mustParseBool(os.Getenv(enableHTTPS)), 3388 ) 3389 if err != nil { 3390 logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) 3391 return 3392 } 3393 3394 // Enable tracing, write to stderr. 3395 // c.TraceOn(os.Stderr) 3396 3397 // Set user agent. 3398 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 3399 3400 // Generate a new random bucket name. 3401 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 3402 args["bucketName"] = bucketName 3403 3404 // Make a new bucket. 3405 err = c.MakeBucket(bucketName, "us-east-1") 3406 if err != nil { 3407 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 3408 return 3409 } 3410 3411 // Generate 129MiB of data. 3412 bufSize := dataFileMap["datafile-129-MB"] 3413 var reader = getDataReader("datafile-129-MB") 3414 defer reader.Close() 3415 3416 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 3417 args["objectName"] = objectName 3418 3419 buf, err := ioutil.ReadAll(reader) 3420 if err != nil { 3421 logError(testName, function, args, startTime, "", "ReadAll failed", err) 3422 return 3423 } 3424 3425 // Save the data 3426 n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ 3427 ContentType: "binary/octet-stream", 3428 ServerSideEncryption: encrypt.NewSSE(), 3429 }) 3430 if err != nil { 3431 logError(testName, function, args, startTime, "", "PutObject failed", err) 3432 return 3433 } 3434 3435 if n != int64(bufSize) { 3436 logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) 3437 return 3438 } 3439 3440 // read the data back 3441 r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) 3442 if err != nil { 3443 logError(testName, function, args, startTime, "", "PutObject failed", err) 3444 return 3445 } 3446 defer r.Close() 3447 3448 offset := int64(2048) 3449 3450 // read directly 3451 buf1 := make([]byte, 512) 3452 buf2 := make([]byte, 512) 3453 buf3 := make([]byte, 512) 3454 buf4 := make([]byte, 512) 3455 3456 // Test readAt before stat is called such that objectInfo doesn't change. 3457 m, err := r.ReadAt(buf1, offset) 3458 if err != nil { 3459 logError(testName, function, args, startTime, "", "ReadAt failed", err) 3460 return 3461 } 3462 if m != len(buf1) { 3463 logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err) 3464 return 3465 } 3466 if !bytes.Equal(buf1, buf[offset:offset+512]) { 3467 logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) 3468 return 3469 } 3470 offset += 512 3471 3472 st, err := r.Stat() 3473 if err != nil { 3474 logError(testName, function, args, startTime, "", "Stat failed", err) 3475 return 3476 } 3477 3478 if st.Size != int64(bufSize) { 3479 logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) 3480 return 3481 } 3482 3483 m, err = r.ReadAt(buf2, offset) 3484 if err != nil { 3485 logError(testName, function, args, startTime, "", "ReadAt failed", err) 3486 return 3487 } 3488 if m != len(buf2) { 3489 logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err) 3490 return 3491 } 3492 if !bytes.Equal(buf2, buf[offset:offset+512]) { 3493 logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) 3494 return 3495 } 3496 offset += 512 3497 m, err = r.ReadAt(buf3, offset) 3498 if err != nil { 3499 logError(testName, function, args, startTime, "", "ReadAt failed", err) 3500 return 3501 } 3502 if m != len(buf3) { 3503 logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+", got "+string(m), err) 3504 return 3505 } 3506 if !bytes.Equal(buf3, buf[offset:offset+512]) { 3507 logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) 3508 return 3509 } 3510 offset += 512 3511 m, err = r.ReadAt(buf4, offset) 3512 if err != nil { 3513 logError(testName, function, args, startTime, "", "ReadAt failed", err) 3514 return 3515 } 3516 if m != len(buf4) { 3517 logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+", got "+string(m), err) 3518 return 3519 } 3520 if !bytes.Equal(buf4, buf[offset:offset+512]) { 3521 logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) 3522 return 3523 } 3524 3525 buf5 := make([]byte, n) 3526 // Read the whole object. 3527 m, err = r.ReadAt(buf5, 0) 3528 if err != nil { 3529 if err != io.EOF { 3530 logError(testName, function, args, startTime, "", "ReadAt failed", err) 3531 return 3532 } 3533 } 3534 if m != len(buf5) { 3535 logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+", got "+string(m), err) 3536 return 3537 } 3538 if !bytes.Equal(buf, buf5) { 3539 logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err) 3540 return 3541 } 3542 3543 buf6 := make([]byte, n+1) 3544 // Read the whole object and beyond. 3545 _, err = r.ReadAt(buf6, 0) 3546 if err != nil { 3547 if err != io.EOF { 3548 logError(testName, function, args, startTime, "", "ReadAt failed", err) 3549 return 3550 } 3551 } 3552 // Delete all objects and buckets 3553 if err = cleanupBucket(bucketName, c); err != nil { 3554 logError(testName, function, args, startTime, "", "Cleanup failed", err) 3555 return 3556 } 3557 successLogger(testName, function, args, startTime).Info() 3558 } 3559 3560 // testSSECEncryptionPutGet tests encryption with customer provided encryption keys 3561 func testSSECEncryptionPutGet() { 3562 // initialize logging params 3563 startTime := time.Now() 3564 testName := getFuncName() 3565 function := "PutEncryptedObject(bucketName, objectName, reader, sse)" 3566 args := map[string]interface{}{ 3567 "bucketName": "", 3568 "objectName": "", 3569 "sse": "", 3570 } 3571 // Seed random based on current time. 3572 rand.Seed(time.Now().Unix()) 3573 3574 // Instantiate new minio client object 3575 c, err := minio.NewV4( 3576 os.Getenv(serverEndpoint), 3577 os.Getenv(accessKey), 3578 os.Getenv(secretKey), 3579 mustParseBool(os.Getenv(enableHTTPS)), 3580 ) 3581 if err != nil { 3582 logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) 3583 return 3584 } 3585 3586 // Enable tracing, write to stderr. 3587 // c.TraceOn(os.Stderr) 3588 3589 // Set user agent. 3590 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 3591 3592 // Generate a new random bucket name. 3593 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 3594 args["bucketName"] = bucketName 3595 3596 // Make a new bucket. 3597 err = c.MakeBucket(bucketName, "us-east-1") 3598 if err != nil { 3599 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 3600 return 3601 } 3602 3603 testCases := []struct { 3604 buf []byte 3605 }{ 3606 {buf: bytes.Repeat([]byte("F"), 1)}, 3607 {buf: bytes.Repeat([]byte("F"), 15)}, 3608 {buf: bytes.Repeat([]byte("F"), 16)}, 3609 {buf: bytes.Repeat([]byte("F"), 17)}, 3610 {buf: bytes.Repeat([]byte("F"), 31)}, 3611 {buf: bytes.Repeat([]byte("F"), 32)}, 3612 {buf: bytes.Repeat([]byte("F"), 33)}, 3613 {buf: bytes.Repeat([]byte("F"), 1024)}, 3614 {buf: bytes.Repeat([]byte("F"), 1024*2)}, 3615 {buf: bytes.Repeat([]byte("F"), 1024*1024)}, 3616 } 3617 3618 const password = "correct horse battery staple" // https://xkcd.com/936/ 3619 3620 for i, testCase := range testCases { 3621 // Generate a random object name 3622 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 3623 args["objectName"] = objectName 3624 3625 // Secured object 3626 sse := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) 3627 args["sse"] = sse 3628 3629 // Put encrypted data 3630 _, err = c.PutObject(bucketName, objectName, bytes.NewReader(testCase.buf), int64(len(testCase.buf)), minio.PutObjectOptions{ServerSideEncryption: sse}) 3631 if err != nil { 3632 logError(testName, function, args, startTime, "", "PutEncryptedObject failed", err) 3633 return 3634 } 3635 3636 // Read the data back 3637 r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{ServerSideEncryption: sse}) 3638 if err != nil { 3639 logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err) 3640 return 3641 } 3642 defer r.Close() 3643 3644 // Compare the sent object with the received one 3645 recvBuffer := bytes.NewBuffer([]byte{}) 3646 if _, err = io.Copy(recvBuffer, r); err != nil { 3647 logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err) 3648 return 3649 } 3650 if recvBuffer.Len() != len(testCase.buf) { 3651 logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err) 3652 return 3653 } 3654 if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { 3655 logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err) 3656 return 3657 } 3658 3659 successLogger(testName, function, args, startTime).Info() 3660 3661 } 3662 3663 // Delete all objects and buckets 3664 if err = cleanupBucket(bucketName, c); err != nil { 3665 logError(testName, function, args, startTime, "", "Cleanup failed", err) 3666 return 3667 } 3668 3669 successLogger(testName, function, args, startTime).Info() 3670 } 3671 3672 // TestEncryptionFPut tests encryption with customer specified encryption keys 3673 func testSSECEncryptionFPut() { 3674 // initialize logging params 3675 startTime := time.Now() 3676 testName := getFuncName() 3677 function := "FPutEncryptedObject(bucketName, objectName, filePath, contentType, sse)" 3678 args := map[string]interface{}{ 3679 "bucketName": "", 3680 "objectName": "", 3681 "filePath": "", 3682 "contentType": "", 3683 "sse": "", 3684 } 3685 // Seed random based on current time. 3686 rand.Seed(time.Now().Unix()) 3687 3688 // Instantiate new minio client object 3689 c, err := minio.NewV4( 3690 os.Getenv(serverEndpoint), 3691 os.Getenv(accessKey), 3692 os.Getenv(secretKey), 3693 mustParseBool(os.Getenv(enableHTTPS)), 3694 ) 3695 if err != nil { 3696 logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) 3697 return 3698 } 3699 3700 // Enable tracing, write to stderr. 3701 // c.TraceOn(os.Stderr) 3702 3703 // Set user agent. 3704 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 3705 3706 // Generate a new random bucket name. 3707 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 3708 args["bucketName"] = bucketName 3709 3710 // Make a new bucket. 3711 err = c.MakeBucket(bucketName, "us-east-1") 3712 if err != nil { 3713 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 3714 return 3715 } 3716 3717 // Object custom metadata 3718 customContentType := "custom/contenttype" 3719 args["metadata"] = customContentType 3720 3721 testCases := []struct { 3722 buf []byte 3723 }{ 3724 {buf: bytes.Repeat([]byte("F"), 0)}, 3725 {buf: bytes.Repeat([]byte("F"), 1)}, 3726 {buf: bytes.Repeat([]byte("F"), 15)}, 3727 {buf: bytes.Repeat([]byte("F"), 16)}, 3728 {buf: bytes.Repeat([]byte("F"), 17)}, 3729 {buf: bytes.Repeat([]byte("F"), 31)}, 3730 {buf: bytes.Repeat([]byte("F"), 32)}, 3731 {buf: bytes.Repeat([]byte("F"), 33)}, 3732 {buf: bytes.Repeat([]byte("F"), 1024)}, 3733 {buf: bytes.Repeat([]byte("F"), 1024*2)}, 3734 {buf: bytes.Repeat([]byte("F"), 1024*1024)}, 3735 } 3736 3737 const password = "correct horse battery staple" // https://xkcd.com/936/ 3738 for i, testCase := range testCases { 3739 // Generate a random object name 3740 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 3741 args["objectName"] = objectName 3742 3743 // Secured object 3744 sse := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) 3745 args["sse"] = sse 3746 3747 // Generate a random file name. 3748 fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 3749 file, err := os.Create(fileName) 3750 if err != nil { 3751 logError(testName, function, args, startTime, "", "file create failed", err) 3752 return 3753 } 3754 _, err = file.Write(testCase.buf) 3755 if err != nil { 3756 logError(testName, function, args, startTime, "", "file write failed", err) 3757 return 3758 } 3759 file.Close() 3760 // Put encrypted data 3761 if _, err = c.FPutObject(bucketName, objectName, fileName, minio.PutObjectOptions{ServerSideEncryption: sse}); err != nil { 3762 logError(testName, function, args, startTime, "", "FPutEncryptedObject failed", err) 3763 return 3764 } 3765 3766 // Read the data back 3767 r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{ServerSideEncryption: sse}) 3768 if err != nil { 3769 logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err) 3770 return 3771 } 3772 defer r.Close() 3773 3774 // Compare the sent object with the received one 3775 recvBuffer := bytes.NewBuffer([]byte{}) 3776 if _, err = io.Copy(recvBuffer, r); err != nil { 3777 logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err) 3778 return 3779 } 3780 if recvBuffer.Len() != len(testCase.buf) { 3781 logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err) 3782 return 3783 } 3784 if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { 3785 logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err) 3786 return 3787 } 3788 3789 os.Remove(fileName) 3790 } 3791 3792 // Delete all objects and buckets 3793 if err = cleanupBucket(bucketName, c); err != nil { 3794 logError(testName, function, args, startTime, "", "Cleanup failed", err) 3795 return 3796 } 3797 3798 successLogger(testName, function, args, startTime).Info() 3799 } 3800 3801 // testSSES3EncryptionPutGet tests SSE-S3 encryption 3802 func testSSES3EncryptionPutGet() { 3803 // initialize logging params 3804 startTime := time.Now() 3805 testName := getFuncName() 3806 function := "PutEncryptedObject(bucketName, objectName, reader, sse)" 3807 args := map[string]interface{}{ 3808 "bucketName": "", 3809 "objectName": "", 3810 "sse": "", 3811 } 3812 // Seed random based on current time. 3813 rand.Seed(time.Now().Unix()) 3814 3815 // Instantiate new minio client object 3816 c, err := minio.NewV4( 3817 os.Getenv(serverEndpoint), 3818 os.Getenv(accessKey), 3819 os.Getenv(secretKey), 3820 mustParseBool(os.Getenv(enableHTTPS)), 3821 ) 3822 if err != nil { 3823 logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) 3824 return 3825 } 3826 3827 // Enable tracing, write to stderr. 3828 // c.TraceOn(os.Stderr) 3829 3830 // Set user agent. 3831 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 3832 3833 // Generate a new random bucket name. 3834 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 3835 args["bucketName"] = bucketName 3836 3837 // Make a new bucket. 3838 err = c.MakeBucket(bucketName, "us-east-1") 3839 if err != nil { 3840 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 3841 return 3842 } 3843 3844 testCases := []struct { 3845 buf []byte 3846 }{ 3847 {buf: bytes.Repeat([]byte("F"), 1)}, 3848 {buf: bytes.Repeat([]byte("F"), 15)}, 3849 {buf: bytes.Repeat([]byte("F"), 16)}, 3850 {buf: bytes.Repeat([]byte("F"), 17)}, 3851 {buf: bytes.Repeat([]byte("F"), 31)}, 3852 {buf: bytes.Repeat([]byte("F"), 32)}, 3853 {buf: bytes.Repeat([]byte("F"), 33)}, 3854 {buf: bytes.Repeat([]byte("F"), 1024)}, 3855 {buf: bytes.Repeat([]byte("F"), 1024*2)}, 3856 {buf: bytes.Repeat([]byte("F"), 1024*1024)}, 3857 } 3858 3859 for i, testCase := range testCases { 3860 // Generate a random object name 3861 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 3862 args["objectName"] = objectName 3863 3864 // Secured object 3865 sse := encrypt.NewSSE() 3866 args["sse"] = sse 3867 3868 // Put encrypted data 3869 _, err = c.PutObject(bucketName, objectName, bytes.NewReader(testCase.buf), int64(len(testCase.buf)), minio.PutObjectOptions{ServerSideEncryption: sse}) 3870 if err != nil { 3871 logError(testName, function, args, startTime, "", "PutEncryptedObject failed", err) 3872 return 3873 } 3874 3875 // Read the data back without any encryption headers 3876 r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) 3877 if err != nil { 3878 logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err) 3879 return 3880 } 3881 defer r.Close() 3882 3883 // Compare the sent object with the received one 3884 recvBuffer := bytes.NewBuffer([]byte{}) 3885 if _, err = io.Copy(recvBuffer, r); err != nil { 3886 logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err) 3887 return 3888 } 3889 if recvBuffer.Len() != len(testCase.buf) { 3890 logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err) 3891 return 3892 } 3893 if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { 3894 logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err) 3895 return 3896 } 3897 3898 successLogger(testName, function, args, startTime).Info() 3899 3900 } 3901 3902 // Delete all objects and buckets 3903 if err = cleanupBucket(bucketName, c); err != nil { 3904 logError(testName, function, args, startTime, "", "Cleanup failed", err) 3905 return 3906 } 3907 3908 successLogger(testName, function, args, startTime).Info() 3909 } 3910 3911 // TestSSES3EncryptionFPut tests server side encryption 3912 func testSSES3EncryptionFPut() { 3913 // initialize logging params 3914 startTime := time.Now() 3915 testName := getFuncName() 3916 function := "FPutEncryptedObject(bucketName, objectName, filePath, contentType, sse)" 3917 args := map[string]interface{}{ 3918 "bucketName": "", 3919 "objectName": "", 3920 "filePath": "", 3921 "contentType": "", 3922 "sse": "", 3923 } 3924 // Seed random based on current time. 3925 rand.Seed(time.Now().Unix()) 3926 3927 // Instantiate new minio client object 3928 c, err := minio.NewV4( 3929 os.Getenv(serverEndpoint), 3930 os.Getenv(accessKey), 3931 os.Getenv(secretKey), 3932 mustParseBool(os.Getenv(enableHTTPS)), 3933 ) 3934 if err != nil { 3935 logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) 3936 return 3937 } 3938 3939 // Enable tracing, write to stderr. 3940 // c.TraceOn(os.Stderr) 3941 3942 // Set user agent. 3943 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 3944 3945 // Generate a new random bucket name. 3946 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 3947 args["bucketName"] = bucketName 3948 3949 // Make a new bucket. 3950 err = c.MakeBucket(bucketName, "us-east-1") 3951 if err != nil { 3952 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 3953 return 3954 } 3955 3956 // Object custom metadata 3957 customContentType := "custom/contenttype" 3958 args["metadata"] = customContentType 3959 3960 testCases := []struct { 3961 buf []byte 3962 }{ 3963 {buf: bytes.Repeat([]byte("F"), 0)}, 3964 {buf: bytes.Repeat([]byte("F"), 1)}, 3965 {buf: bytes.Repeat([]byte("F"), 15)}, 3966 {buf: bytes.Repeat([]byte("F"), 16)}, 3967 {buf: bytes.Repeat([]byte("F"), 17)}, 3968 {buf: bytes.Repeat([]byte("F"), 31)}, 3969 {buf: bytes.Repeat([]byte("F"), 32)}, 3970 {buf: bytes.Repeat([]byte("F"), 33)}, 3971 {buf: bytes.Repeat([]byte("F"), 1024)}, 3972 {buf: bytes.Repeat([]byte("F"), 1024*2)}, 3973 {buf: bytes.Repeat([]byte("F"), 1024*1024)}, 3974 } 3975 3976 for i, testCase := range testCases { 3977 // Generate a random object name 3978 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 3979 args["objectName"] = objectName 3980 3981 // Secured object 3982 sse := encrypt.NewSSE() 3983 args["sse"] = sse 3984 3985 // Generate a random file name. 3986 fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 3987 file, err := os.Create(fileName) 3988 if err != nil { 3989 logError(testName, function, args, startTime, "", "file create failed", err) 3990 return 3991 } 3992 _, err = file.Write(testCase.buf) 3993 if err != nil { 3994 logError(testName, function, args, startTime, "", "file write failed", err) 3995 return 3996 } 3997 file.Close() 3998 // Put encrypted data 3999 if _, err = c.FPutObject(bucketName, objectName, fileName, minio.PutObjectOptions{ServerSideEncryption: sse}); err != nil { 4000 logError(testName, function, args, startTime, "", "FPutEncryptedObject failed", err) 4001 return 4002 } 4003 4004 // Read the data back 4005 r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) 4006 if err != nil { 4007 logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err) 4008 return 4009 } 4010 defer r.Close() 4011 4012 // Compare the sent object with the received one 4013 recvBuffer := bytes.NewBuffer([]byte{}) 4014 if _, err = io.Copy(recvBuffer, r); err != nil { 4015 logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err) 4016 return 4017 } 4018 if recvBuffer.Len() != len(testCase.buf) { 4019 logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err) 4020 return 4021 } 4022 if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { 4023 logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err) 4024 return 4025 } 4026 4027 os.Remove(fileName) 4028 } 4029 4030 // Delete all objects and buckets 4031 if err = cleanupBucket(bucketName, c); err != nil { 4032 logError(testName, function, args, startTime, "", "Cleanup failed", err) 4033 return 4034 } 4035 4036 successLogger(testName, function, args, startTime).Info() 4037 } 4038 4039 func testBucketNotification() { 4040 // initialize logging params 4041 startTime := time.Now() 4042 testName := getFuncName() 4043 function := "SetBucketNotification(bucketName)" 4044 args := map[string]interface{}{ 4045 "bucketName": "", 4046 } 4047 4048 if os.Getenv("NOTIFY_BUCKET") == "" || 4049 os.Getenv("NOTIFY_SERVICE") == "" || 4050 os.Getenv("NOTIFY_REGION") == "" || 4051 os.Getenv("NOTIFY_ACCOUNTID") == "" || 4052 os.Getenv("NOTIFY_RESOURCE") == "" { 4053 ignoredLog(testName, function, args, startTime, "Skipped notification test as it is not configured").Info() 4054 return 4055 } 4056 4057 // Seed random based on current time. 4058 rand.Seed(time.Now().Unix()) 4059 4060 c, err := minio.New( 4061 os.Getenv(serverEndpoint), 4062 os.Getenv(accessKey), 4063 os.Getenv(secretKey), 4064 mustParseBool(os.Getenv(enableHTTPS)), 4065 ) 4066 if err != nil { 4067 logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) 4068 return 4069 } 4070 4071 // Enable to debug 4072 // c.TraceOn(os.Stderr) 4073 4074 // Set user agent. 4075 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 4076 4077 bucketName := os.Getenv("NOTIFY_BUCKET") 4078 args["bucketName"] = bucketName 4079 4080 topicArn := minio.NewArn("aws", os.Getenv("NOTIFY_SERVICE"), os.Getenv("NOTIFY_REGION"), os.Getenv("NOTIFY_ACCOUNTID"), os.Getenv("NOTIFY_RESOURCE")) 4081 queueArn := minio.NewArn("aws", "dummy-service", "dummy-region", "dummy-accountid", "dummy-resource") 4082 4083 topicConfig := minio.NewNotificationConfig(topicArn) 4084 4085 topicConfig.AddEvents(minio.ObjectCreatedAll, minio.ObjectRemovedAll) 4086 topicConfig.AddFilterSuffix("jpg") 4087 4088 queueConfig := minio.NewNotificationConfig(queueArn) 4089 queueConfig.AddEvents(minio.ObjectCreatedAll) 4090 queueConfig.AddFilterPrefix("photos/") 4091 4092 bNotification := minio.BucketNotification{} 4093 bNotification.AddTopic(topicConfig) 4094 4095 // Add the same topicConfig again, should have no effect 4096 // because it is duplicated 4097 bNotification.AddTopic(topicConfig) 4098 if len(bNotification.TopicConfigs) != 1 { 4099 logError(testName, function, args, startTime, "", "Duplicate entry added", err) 4100 return 4101 } 4102 4103 // Add and remove a queue config 4104 bNotification.AddQueue(queueConfig) 4105 bNotification.RemoveQueueByArn(queueArn) 4106 4107 err = c.SetBucketNotification(bucketName, bNotification) 4108 if err != nil { 4109 logError(testName, function, args, startTime, "", "SetBucketNotification failed", err) 4110 return 4111 } 4112 4113 bNotification, err = c.GetBucketNotification(bucketName) 4114 if err != nil { 4115 logError(testName, function, args, startTime, "", "GetBucketNotification failed", err) 4116 return 4117 } 4118 4119 if len(bNotification.TopicConfigs) != 1 { 4120 logError(testName, function, args, startTime, "", "Topic config is empty", err) 4121 return 4122 } 4123 4124 if bNotification.TopicConfigs[0].Filter.S3Key.FilterRules[0].Value != "jpg" { 4125 logError(testName, function, args, startTime, "", "Couldn't get the suffix", err) 4126 return 4127 } 4128 4129 err = c.RemoveAllBucketNotification(bucketName) 4130 if err != nil { 4131 logError(testName, function, args, startTime, "", "RemoveAllBucketNotification failed", err) 4132 return 4133 } 4134 4135 // Delete all objects and buckets 4136 if err = cleanupBucket(bucketName, c); err != nil { 4137 logError(testName, function, args, startTime, "", "Cleanup failed", err) 4138 return 4139 } 4140 4141 successLogger(testName, function, args, startTime).Info() 4142 } 4143 4144 // Tests comprehensive list of all methods. 4145 func testFunctional() { 4146 // initialize logging params 4147 startTime := time.Now() 4148 testName := getFuncName() 4149 function := "testFunctional()" 4150 functionAll := "" 4151 args := map[string]interface{}{} 4152 4153 // Seed random based on current time. 4154 rand.Seed(time.Now().Unix()) 4155 4156 c, err := minio.New( 4157 os.Getenv(serverEndpoint), 4158 os.Getenv(accessKey), 4159 os.Getenv(secretKey), 4160 mustParseBool(os.Getenv(enableHTTPS)), 4161 ) 4162 if err != nil { 4163 logError(testName, function, nil, startTime, "", "MinIO client object creation failed", err) 4164 return 4165 } 4166 4167 // Enable to debug 4168 // c.TraceOn(os.Stderr) 4169 4170 // Set user agent. 4171 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 4172 4173 // Generate a new random bucket name. 4174 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 4175 4176 // Make a new bucket. 4177 function = "MakeBucket(bucketName, region)" 4178 functionAll = "MakeBucket(bucketName, region)" 4179 args["bucketName"] = bucketName 4180 err = c.MakeBucket(bucketName, "us-east-1") 4181 4182 if err != nil { 4183 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 4184 return 4185 } 4186 4187 // Generate a random file name. 4188 fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 4189 file, err := os.Create(fileName) 4190 if err != nil { 4191 logError(testName, function, args, startTime, "", "File creation failed", err) 4192 return 4193 } 4194 for i := 0; i < 3; i++ { 4195 buf := make([]byte, rand.Intn(1<<19)) 4196 _, err = file.Write(buf) 4197 if err != nil { 4198 logError(testName, function, args, startTime, "", "File write failed", err) 4199 return 4200 } 4201 } 4202 file.Close() 4203 4204 // Verify if bucket exits and you have access. 4205 var exists bool 4206 function = "BucketExists(bucketName)" 4207 functionAll += ", " + function 4208 args = map[string]interface{}{ 4209 "bucketName": bucketName, 4210 } 4211 exists, err = c.BucketExists(bucketName) 4212 4213 if err != nil { 4214 logError(testName, function, args, startTime, "", "BucketExists failed", err) 4215 return 4216 } 4217 if !exists { 4218 logError(testName, function, args, startTime, "", "Could not find the bucket", err) 4219 return 4220 } 4221 4222 // Verify if bucket exits and you have access with context. 4223 function = "BucketExistsWithContext(ctx, bucketName)" 4224 functionAll += ", " + function 4225 args = map[string]interface{}{ 4226 "bucketName": bucketName, 4227 } 4228 exists, err = c.BucketExistsWithContext(context.Background(), bucketName) 4229 4230 if err != nil { 4231 logError(testName, function, args, startTime, "", "BucketExistsWithContext failed", err) 4232 return 4233 } 4234 if !exists { 4235 logError(testName, function, args, startTime, "", "Could not find the bucket", err) 4236 return 4237 } 4238 4239 // Asserting the default bucket policy. 4240 function = "GetBucketPolicy(bucketName)" 4241 functionAll += ", " + function 4242 args = map[string]interface{}{ 4243 "bucketName": bucketName, 4244 } 4245 nilPolicy, err := c.GetBucketPolicy(bucketName) 4246 if err != nil { 4247 logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err) 4248 return 4249 } 4250 if nilPolicy != "" { 4251 logError(testName, function, args, startTime, "", "policy should be set to nil", err) 4252 return 4253 } 4254 4255 // Set the bucket policy to 'public readonly'. 4256 function = "SetBucketPolicy(bucketName, readOnlyPolicy)" 4257 functionAll += ", " + function 4258 4259 readOnlyPolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::` + bucketName + `"]}]}` 4260 args = map[string]interface{}{ 4261 "bucketName": bucketName, 4262 "bucketPolicy": readOnlyPolicy, 4263 } 4264 4265 err = c.SetBucketPolicy(bucketName, readOnlyPolicy) 4266 if err != nil { 4267 logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) 4268 return 4269 } 4270 // should return policy `readonly`. 4271 function = "GetBucketPolicy(bucketName)" 4272 functionAll += ", " + function 4273 args = map[string]interface{}{ 4274 "bucketName": bucketName, 4275 } 4276 _, err = c.GetBucketPolicy(bucketName) 4277 if err != nil { 4278 logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err) 4279 return 4280 } 4281 4282 // Make the bucket 'public writeonly'. 4283 function = "SetBucketPolicy(bucketName, writeOnlyPolicy)" 4284 functionAll += ", " + function 4285 4286 writeOnlyPolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:ListBucketMultipartUploads"],"Resource":["arn:aws:s3:::` + bucketName + `"]}]}` 4287 args = map[string]interface{}{ 4288 "bucketName": bucketName, 4289 "bucketPolicy": writeOnlyPolicy, 4290 } 4291 err = c.SetBucketPolicy(bucketName, writeOnlyPolicy) 4292 4293 if err != nil { 4294 logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) 4295 return 4296 } 4297 // should return policy `writeonly`. 4298 function = "GetBucketPolicy(bucketName)" 4299 functionAll += ", " + function 4300 args = map[string]interface{}{ 4301 "bucketName": bucketName, 4302 } 4303 4304 _, err = c.GetBucketPolicy(bucketName) 4305 if err != nil { 4306 logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err) 4307 return 4308 } 4309 4310 // Make the bucket 'public read/write'. 4311 function = "SetBucketPolicy(bucketName, readWritePolicy)" 4312 functionAll += ", " + function 4313 4314 readWritePolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Resource":["arn:aws:s3:::` + bucketName + `"]}]}` 4315 4316 args = map[string]interface{}{ 4317 "bucketName": bucketName, 4318 "bucketPolicy": readWritePolicy, 4319 } 4320 err = c.SetBucketPolicy(bucketName, readWritePolicy) 4321 4322 if err != nil { 4323 logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) 4324 return 4325 } 4326 // should return policy `readwrite`. 4327 function = "GetBucketPolicy(bucketName)" 4328 functionAll += ", " + function 4329 args = map[string]interface{}{ 4330 "bucketName": bucketName, 4331 } 4332 _, err = c.GetBucketPolicy(bucketName) 4333 if err != nil { 4334 logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err) 4335 return 4336 } 4337 4338 // List all buckets. 4339 function = "ListBuckets()" 4340 functionAll += ", " + function 4341 args = nil 4342 buckets, err := c.ListBuckets() 4343 4344 if len(buckets) == 0 { 4345 logError(testName, function, args, startTime, "", "Found bucket list to be empty", err) 4346 return 4347 } 4348 if err != nil { 4349 logError(testName, function, args, startTime, "", "ListBuckets failed", err) 4350 return 4351 } 4352 4353 // List all buckets with context. 4354 function = "ListBucketsWithContext()" 4355 functionAll += ", " + function 4356 args = nil 4357 buckets, err = c.ListBucketsWithContext(context.Background()) 4358 4359 if len(buckets) == 0 { 4360 logError(testName, function, args, startTime, "", "Found bucket list to be empty", err) 4361 return 4362 } 4363 if err != nil { 4364 logError(testName, function, args, startTime, "", "ListBucketsWithContext failed", err) 4365 return 4366 } 4367 4368 // Verify if previously created bucket is listed in list buckets. 4369 bucketFound := false 4370 for _, bucket := range buckets { 4371 if bucket.Name == bucketName { 4372 bucketFound = true 4373 } 4374 } 4375 4376 // If bucket not found error out. 4377 if !bucketFound { 4378 logError(testName, function, args, startTime, "", "Bucket: "+bucketName+" not found", err) 4379 return 4380 } 4381 4382 objectName := bucketName + "unique" 4383 4384 // Generate data 4385 buf := bytes.Repeat([]byte("f"), 1<<19) 4386 4387 function = "PutObject(bucketName, objectName, reader, contentType)" 4388 functionAll += ", " + function 4389 args = map[string]interface{}{ 4390 "bucketName": bucketName, 4391 "objectName": objectName, 4392 "contentType": "", 4393 } 4394 4395 n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) 4396 if err != nil { 4397 logError(testName, function, args, startTime, "", "PutObject failed", err) 4398 return 4399 } 4400 4401 if n != int64(len(buf)) { 4402 logError(testName, function, args, startTime, "", "Length doesn't match, expected "+string(int64(len(buf)))+" got "+string(n), err) 4403 return 4404 } 4405 4406 args = map[string]interface{}{ 4407 "bucketName": bucketName, 4408 "objectName": objectName + "-nolength", 4409 "contentType": "binary/octet-stream", 4410 } 4411 4412 n, err = c.PutObject(bucketName, objectName+"-nolength", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) 4413 if err != nil { 4414 logError(testName, function, args, startTime, "", "PutObject failed", err) 4415 return 4416 } 4417 4418 if n != int64(len(buf)) { 4419 logError(testName, function, args, startTime, "", "Length doesn't match, expected "+string(int64(len(buf)))+" got "+string(n), err) 4420 return 4421 } 4422 4423 // Instantiate a done channel to close all listing. 4424 doneCh := make(chan struct{}) 4425 defer close(doneCh) 4426 4427 objFound := false 4428 isRecursive := true // Recursive is true. 4429 4430 function = "ListObjects(bucketName, objectName, isRecursive, doneCh)" 4431 functionAll += ", " + function 4432 args = map[string]interface{}{ 4433 "bucketName": bucketName, 4434 "objectName": objectName, 4435 "isRecursive": isRecursive, 4436 } 4437 4438 for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) { 4439 if obj.Key == objectName { 4440 objFound = true 4441 break 4442 } 4443 } 4444 if !objFound { 4445 logError(testName, function, args, startTime, "", "Object "+objectName+" not found", err) 4446 return 4447 } 4448 4449 objFound = false 4450 isRecursive = true // Recursive is true. 4451 function = "ListObjectsV2(bucketName, objectName, isRecursive, doneCh)" 4452 functionAll += ", " + function 4453 args = map[string]interface{}{ 4454 "bucketName": bucketName, 4455 "objectName": objectName, 4456 "isRecursive": isRecursive, 4457 } 4458 4459 for obj := range c.ListObjectsV2(bucketName, objectName, isRecursive, doneCh) { 4460 if obj.Key == objectName { 4461 objFound = true 4462 break 4463 } 4464 } 4465 if !objFound { 4466 logError(testName, function, args, startTime, "", "Object "+objectName+" not found", err) 4467 return 4468 } 4469 4470 incompObjNotFound := true 4471 4472 function = "ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh)" 4473 functionAll += ", " + function 4474 args = map[string]interface{}{ 4475 "bucketName": bucketName, 4476 "objectName": objectName, 4477 "isRecursive": isRecursive, 4478 } 4479 4480 for objIncompl := range c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) { 4481 if objIncompl.Key != "" { 4482 incompObjNotFound = false 4483 break 4484 } 4485 } 4486 if !incompObjNotFound { 4487 logError(testName, function, args, startTime, "", "Unexpected dangling incomplete upload found", err) 4488 return 4489 } 4490 4491 function = "GetObject(bucketName, objectName)" 4492 functionAll += ", " + function 4493 args = map[string]interface{}{ 4494 "bucketName": bucketName, 4495 "objectName": objectName, 4496 } 4497 newReader, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) 4498 4499 if err != nil { 4500 logError(testName, function, args, startTime, "", "GetObject failed", err) 4501 return 4502 } 4503 4504 newReadBytes, err := ioutil.ReadAll(newReader) 4505 if err != nil { 4506 logError(testName, function, args, startTime, "", "ReadAll failed", err) 4507 return 4508 } 4509 4510 if !bytes.Equal(newReadBytes, buf) { 4511 logError(testName, function, args, startTime, "", "GetObject bytes mismatch", err) 4512 return 4513 } 4514 newReader.Close() 4515 4516 function = "FGetObject(bucketName, objectName, fileName)" 4517 functionAll += ", " + function 4518 args = map[string]interface{}{ 4519 "bucketName": bucketName, 4520 "objectName": objectName, 4521 "fileName": fileName + "-f", 4522 } 4523 err = c.FGetObject(bucketName, objectName, fileName+"-f", minio.GetObjectOptions{}) 4524 4525 if err != nil { 4526 logError(testName, function, args, startTime, "", "FGetObject failed", err) 4527 return 4528 } 4529 4530 function = "PresignedHeadObject(bucketName, objectName, expires, reqParams)" 4531 functionAll += ", " + function 4532 args = map[string]interface{}{ 4533 "bucketName": bucketName, 4534 "objectName": "", 4535 "expires": 3600 * time.Second, 4536 } 4537 if _, err = c.PresignedHeadObject(bucketName, "", 3600*time.Second, nil); err == nil { 4538 logError(testName, function, args, startTime, "", "PresignedHeadObject success", err) 4539 return 4540 } 4541 4542 // Generate presigned HEAD object url. 4543 function = "PresignedHeadObject(bucketName, objectName, expires, reqParams)" 4544 functionAll += ", " + function 4545 args = map[string]interface{}{ 4546 "bucketName": bucketName, 4547 "objectName": objectName, 4548 "expires": 3600 * time.Second, 4549 } 4550 presignedHeadURL, err := c.PresignedHeadObject(bucketName, objectName, 3600*time.Second, nil) 4551 4552 if err != nil { 4553 logError(testName, function, args, startTime, "", "PresignedHeadObject failed", err) 4554 return 4555 } 4556 4557 transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS))) 4558 if err != nil { 4559 logError(testName, function, args, startTime, "", "DefaultTransport failed", err) 4560 return 4561 } 4562 4563 httpClient := &http.Client{ 4564 // Setting a sensible time out of 30secs to wait for response 4565 // headers. Request is pro-actively canceled after 30secs 4566 // with no response. 4567 Timeout: 30 * time.Second, 4568 Transport: transport, 4569 } 4570 4571 req, err := http.NewRequest(http.MethodHead, presignedHeadURL.String(), nil) 4572 if err != nil { 4573 logError(testName, function, args, startTime, "", "PresignedHeadObject request was incorrect", err) 4574 return 4575 } 4576 4577 // Verify if presigned url works. 4578 resp, err := httpClient.Do(req) 4579 if err != nil { 4580 logError(testName, function, args, startTime, "", "PresignedHeadObject response incorrect", err) 4581 return 4582 } 4583 if resp.StatusCode != http.StatusOK { 4584 logError(testName, function, args, startTime, "", "PresignedHeadObject response incorrect, status "+string(resp.StatusCode), err) 4585 return 4586 } 4587 if resp.Header.Get("ETag") == "" { 4588 logError(testName, function, args, startTime, "", "PresignedHeadObject response incorrect", err) 4589 return 4590 } 4591 resp.Body.Close() 4592 4593 function = "PresignedGetObject(bucketName, objectName, expires, reqParams)" 4594 functionAll += ", " + function 4595 args = map[string]interface{}{ 4596 "bucketName": bucketName, 4597 "objectName": "", 4598 "expires": 3600 * time.Second, 4599 } 4600 _, err = c.PresignedGetObject(bucketName, "", 3600*time.Second, nil) 4601 if err == nil { 4602 logError(testName, function, args, startTime, "", "PresignedGetObject success", err) 4603 return 4604 } 4605 4606 // Generate presigned GET object url. 4607 function = "PresignedGetObject(bucketName, objectName, expires, reqParams)" 4608 functionAll += ", " + function 4609 args = map[string]interface{}{ 4610 "bucketName": bucketName, 4611 "objectName": objectName, 4612 "expires": 3600 * time.Second, 4613 } 4614 presignedGetURL, err := c.PresignedGetObject(bucketName, objectName, 3600*time.Second, nil) 4615 4616 if err != nil { 4617 logError(testName, function, args, startTime, "", "PresignedGetObject failed", err) 4618 return 4619 } 4620 4621 // Verify if presigned url works. 4622 req, err = http.NewRequest(http.MethodGet, presignedGetURL.String(), nil) 4623 if err != nil { 4624 logError(testName, function, args, startTime, "", "PresignedGetObject request incorrect", err) 4625 return 4626 } 4627 4628 resp, err = httpClient.Do(req) 4629 if err != nil { 4630 logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) 4631 return 4632 } 4633 if resp.StatusCode != http.StatusOK { 4634 logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect, status "+string(resp.StatusCode), err) 4635 return 4636 } 4637 newPresignedBytes, err := ioutil.ReadAll(resp.Body) 4638 if err != nil { 4639 logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) 4640 return 4641 } 4642 resp.Body.Close() 4643 if !bytes.Equal(newPresignedBytes, buf) { 4644 logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) 4645 return 4646 } 4647 4648 // Set request parameters. 4649 reqParams := make(url.Values) 4650 reqParams.Set("response-content-disposition", "attachment; filename=\"test.txt\"") 4651 args = map[string]interface{}{ 4652 "bucketName": bucketName, 4653 "objectName": objectName, 4654 "expires": 3600 * time.Second, 4655 "reqParams": reqParams, 4656 } 4657 presignedGetURL, err = c.PresignedGetObject(bucketName, objectName, 3600*time.Second, reqParams) 4658 4659 if err != nil { 4660 logError(testName, function, args, startTime, "", "PresignedGetObject failed", err) 4661 return 4662 } 4663 4664 // Verify if presigned url works. 4665 req, err = http.NewRequest(http.MethodGet, presignedGetURL.String(), nil) 4666 if err != nil { 4667 logError(testName, function, args, startTime, "", "PresignedGetObject request incorrect", err) 4668 return 4669 } 4670 4671 resp, err = httpClient.Do(req) 4672 if err != nil { 4673 logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) 4674 return 4675 } 4676 if resp.StatusCode != http.StatusOK { 4677 logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect, status "+string(resp.StatusCode), err) 4678 return 4679 } 4680 newPresignedBytes, err = ioutil.ReadAll(resp.Body) 4681 if err != nil { 4682 logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) 4683 return 4684 } 4685 if !bytes.Equal(newPresignedBytes, buf) { 4686 logError(testName, function, args, startTime, "", "Bytes mismatch for presigned GET URL", err) 4687 return 4688 } 4689 if resp.Header.Get("Content-Disposition") != "attachment; filename=\"test.txt\"" { 4690 logError(testName, function, args, startTime, "", "wrong Content-Disposition received "+string(resp.Header.Get("Content-Disposition")), err) 4691 return 4692 } 4693 4694 function = "PresignedPutObject(bucketName, objectName, expires)" 4695 functionAll += ", " + function 4696 args = map[string]interface{}{ 4697 "bucketName": bucketName, 4698 "objectName": "", 4699 "expires": 3600 * time.Second, 4700 } 4701 _, err = c.PresignedPutObject(bucketName, "", 3600*time.Second) 4702 if err == nil { 4703 logError(testName, function, args, startTime, "", "PresignedPutObject success", err) 4704 return 4705 } 4706 4707 function = "PresignedPutObject(bucketName, objectName, expires)" 4708 functionAll += ", " + function 4709 args = map[string]interface{}{ 4710 "bucketName": bucketName, 4711 "objectName": objectName + "-presigned", 4712 "expires": 3600 * time.Second, 4713 } 4714 presignedPutURL, err := c.PresignedPutObject(bucketName, objectName+"-presigned", 3600*time.Second) 4715 4716 if err != nil { 4717 logError(testName, function, args, startTime, "", "PresignedPutObject failed", err) 4718 return 4719 } 4720 4721 buf = bytes.Repeat([]byte("g"), 1<<19) 4722 4723 req, err = http.NewRequest(http.MethodPut, presignedPutURL.String(), bytes.NewReader(buf)) 4724 if err != nil { 4725 logError(testName, function, args, startTime, "", "Couldn't make HTTP request with PresignedPutObject URL", err) 4726 return 4727 } 4728 4729 resp, err = httpClient.Do(req) 4730 if err != nil { 4731 logError(testName, function, args, startTime, "", "PresignedPutObject failed", err) 4732 return 4733 } 4734 4735 newReader, err = c.GetObject(bucketName, objectName+"-presigned", minio.GetObjectOptions{}) 4736 if err != nil { 4737 logError(testName, function, args, startTime, "", "GetObject after PresignedPutObject failed", err) 4738 return 4739 } 4740 4741 newReadBytes, err = ioutil.ReadAll(newReader) 4742 if err != nil { 4743 logError(testName, function, args, startTime, "", "ReadAll after GetObject failed", err) 4744 return 4745 } 4746 4747 if !bytes.Equal(newReadBytes, buf) { 4748 logError(testName, function, args, startTime, "", "Bytes mismatch", err) 4749 return 4750 } 4751 4752 function = "RemoveObject(bucketName, objectName)" 4753 functionAll += ", " + function 4754 args = map[string]interface{}{ 4755 "bucketName": bucketName, 4756 "objectName": objectName, 4757 } 4758 err = c.RemoveObject(bucketName, objectName) 4759 4760 if err != nil { 4761 logError(testName, function, args, startTime, "", "RemoveObject failed", err) 4762 return 4763 } 4764 args["objectName"] = objectName + "-f" 4765 err = c.RemoveObject(bucketName, objectName+"-f") 4766 4767 if err != nil { 4768 logError(testName, function, args, startTime, "", "RemoveObject failed", err) 4769 return 4770 } 4771 4772 args["objectName"] = objectName + "-nolength" 4773 err = c.RemoveObject(bucketName, objectName+"-nolength") 4774 4775 if err != nil { 4776 logError(testName, function, args, startTime, "", "RemoveObject failed", err) 4777 return 4778 } 4779 4780 args["objectName"] = objectName + "-presigned" 4781 err = c.RemoveObject(bucketName, objectName+"-presigned") 4782 4783 if err != nil { 4784 logError(testName, function, args, startTime, "", "RemoveObject failed", err) 4785 return 4786 } 4787 4788 function = "RemoveBucket(bucketName)" 4789 functionAll += ", " + function 4790 args = map[string]interface{}{ 4791 "bucketName": bucketName, 4792 } 4793 err = c.RemoveBucket(bucketName) 4794 4795 if err != nil { 4796 logError(testName, function, args, startTime, "", "RemoveBucket failed", err) 4797 return 4798 } 4799 err = c.RemoveBucket(bucketName) 4800 if err == nil { 4801 logError(testName, function, args, startTime, "", "RemoveBucket did not fail for invalid bucket name", err) 4802 return 4803 } 4804 if err.Error() != "The specified bucket does not exist" { 4805 logError(testName, function, args, startTime, "", "RemoveBucket failed", err) 4806 return 4807 } 4808 4809 os.Remove(fileName) 4810 os.Remove(fileName + "-f") 4811 successLogger(testName, functionAll, args, startTime).Info() 4812 } 4813 4814 // Test for validating GetObject Reader* methods functioning when the 4815 // object is modified in the object store. 4816 func testGetObjectModified() { 4817 // initialize logging params 4818 startTime := time.Now() 4819 testName := getFuncName() 4820 function := "GetObject(bucketName, objectName)" 4821 args := map[string]interface{}{} 4822 4823 // Instantiate new minio client object. 4824 c, err := minio.NewV4( 4825 os.Getenv(serverEndpoint), 4826 os.Getenv(accessKey), 4827 os.Getenv(secretKey), 4828 mustParseBool(os.Getenv(enableHTTPS)), 4829 ) 4830 4831 if err != nil { 4832 logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) 4833 return 4834 } 4835 4836 // Enable tracing, write to stderr. 4837 // c.TraceOn(os.Stderr) 4838 4839 // Set user agent. 4840 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 4841 4842 // Make a new bucket. 4843 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 4844 args["bucketName"] = bucketName 4845 4846 err = c.MakeBucket(bucketName, "us-east-1") 4847 if err != nil { 4848 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 4849 return 4850 } 4851 defer c.RemoveBucket(bucketName) 4852 4853 // Upload an object. 4854 objectName := "myobject" 4855 args["objectName"] = objectName 4856 content := "helloworld" 4857 _, err = c.PutObject(bucketName, objectName, strings.NewReader(content), int64(len(content)), minio.PutObjectOptions{ContentType: "application/text"}) 4858 if err != nil { 4859 logError(testName, function, args, startTime, "", "Failed to upload "+objectName+", to bucket "+bucketName, err) 4860 return 4861 } 4862 4863 defer c.RemoveObject(bucketName, objectName) 4864 4865 reader, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) 4866 if err != nil { 4867 logError(testName, function, args, startTime, "", "Failed to GetObject "+objectName+", from bucket "+bucketName, err) 4868 return 4869 } 4870 defer reader.Close() 4871 4872 // Read a few bytes of the object. 4873 b := make([]byte, 5) 4874 n, err := reader.ReadAt(b, 0) 4875 if err != nil { 4876 logError(testName, function, args, startTime, "", "Failed to read object "+objectName+", from bucket "+bucketName+" at an offset", err) 4877 return 4878 } 4879 4880 // Upload different contents to the same object while object is being read. 4881 newContent := "goodbyeworld" 4882 _, err = c.PutObject(bucketName, objectName, strings.NewReader(newContent), int64(len(newContent)), minio.PutObjectOptions{ContentType: "application/text"}) 4883 if err != nil { 4884 logError(testName, function, args, startTime, "", "Failed to upload "+objectName+", to bucket "+bucketName, err) 4885 return 4886 } 4887 4888 // Confirm that a Stat() call in between doesn't change the Object's cached etag. 4889 _, err = reader.Stat() 4890 expectedError := "At least one of the pre-conditions you specified did not hold" 4891 if err.Error() != expectedError { 4892 logError(testName, function, args, startTime, "", "Expected Stat to fail with error "+expectedError+", but received "+err.Error(), err) 4893 return 4894 } 4895 4896 // Read again only to find object contents have been modified since last read. 4897 _, err = reader.ReadAt(b, int64(n)) 4898 if err.Error() != expectedError { 4899 logError(testName, function, args, startTime, "", "Expected ReadAt to fail with error "+expectedError+", but received "+err.Error(), err) 4900 return 4901 } 4902 // Delete all objects and buckets 4903 if err = cleanupBucket(bucketName, c); err != nil { 4904 logError(testName, function, args, startTime, "", "Cleanup failed", err) 4905 return 4906 } 4907 4908 successLogger(testName, function, args, startTime).Info() 4909 } 4910 4911 // Test validates putObject to upload a file seeked at a given offset. 4912 func testPutObjectUploadSeekedObject() { 4913 // initialize logging params 4914 startTime := time.Now() 4915 testName := getFuncName() 4916 function := "PutObject(bucketName, objectName, fileToUpload, contentType)" 4917 args := map[string]interface{}{ 4918 "bucketName": "", 4919 "objectName": "", 4920 "fileToUpload": "", 4921 "contentType": "binary/octet-stream", 4922 } 4923 4924 // Instantiate new minio client object. 4925 c, err := minio.NewV4( 4926 os.Getenv(serverEndpoint), 4927 os.Getenv(accessKey), 4928 os.Getenv(secretKey), 4929 mustParseBool(os.Getenv(enableHTTPS)), 4930 ) 4931 if err != nil { 4932 logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) 4933 return 4934 } 4935 4936 // Enable tracing, write to stderr. 4937 // c.TraceOn(os.Stderr) 4938 4939 // Set user agent. 4940 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 4941 4942 // Make a new bucket. 4943 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 4944 args["bucketName"] = bucketName 4945 4946 err = c.MakeBucket(bucketName, "us-east-1") 4947 if err != nil { 4948 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 4949 return 4950 } 4951 defer c.RemoveBucket(bucketName) 4952 4953 var tempfile *os.File 4954 4955 if fileName := getMintDataDirFilePath("datafile-100-kB"); fileName != "" { 4956 tempfile, err = os.Open(fileName) 4957 if err != nil { 4958 logError(testName, function, args, startTime, "", "File open failed", err) 4959 return 4960 } 4961 args["fileToUpload"] = fileName 4962 } else { 4963 tempfile, err = ioutil.TempFile("", "minio-go-upload-test-") 4964 if err != nil { 4965 logError(testName, function, args, startTime, "", "TempFile create failed", err) 4966 return 4967 } 4968 args["fileToUpload"] = tempfile.Name() 4969 4970 // Generate 100kB data 4971 if _, err = io.Copy(tempfile, getDataReader("datafile-100-kB")); err != nil { 4972 logError(testName, function, args, startTime, "", "File copy failed", err) 4973 return 4974 } 4975 4976 defer os.Remove(tempfile.Name()) 4977 4978 // Seek back to the beginning of the file. 4979 tempfile.Seek(0, 0) 4980 } 4981 var length = 100 * humanize.KiByte 4982 objectName := fmt.Sprintf("test-file-%v", rand.Uint32()) 4983 args["objectName"] = objectName 4984 4985 offset := length / 2 4986 if _, err = tempfile.Seek(int64(offset), 0); err != nil { 4987 logError(testName, function, args, startTime, "", "TempFile seek failed", err) 4988 return 4989 } 4990 4991 n, err := c.PutObject(bucketName, objectName, tempfile, int64(length-offset), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) 4992 if err != nil { 4993 logError(testName, function, args, startTime, "", "PutObject failed", err) 4994 return 4995 } 4996 if n != int64(length-offset) { 4997 logError(testName, function, args, startTime, "", fmt.Sprintf("Invalid length returned, expected %d got %d", int64(length-offset), n), err) 4998 return 4999 } 5000 tempfile.Close() 5001 5002 obj, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) 5003 if err != nil { 5004 logError(testName, function, args, startTime, "", "GetObject failed", err) 5005 return 5006 } 5007 defer obj.Close() 5008 5009 n, err = obj.Seek(int64(offset), 0) 5010 if err != nil { 5011 logError(testName, function, args, startTime, "", "Seek failed", err) 5012 return 5013 } 5014 if n != int64(offset) { 5015 logError(testName, function, args, startTime, "", fmt.Sprintf("Invalid offset returned, expected %d got %d", int64(offset), n), err) 5016 return 5017 } 5018 5019 n, err = c.PutObject(bucketName, objectName+"getobject", obj, int64(length-offset), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) 5020 if err != nil { 5021 logError(testName, function, args, startTime, "", "PutObject failed", err) 5022 return 5023 } 5024 if n != int64(length-offset) { 5025 logError(testName, function, args, startTime, "", fmt.Sprintf("Invalid offset returned, expected %d got %d", int64(length-offset), n), err) 5026 return 5027 } 5028 5029 // Delete all objects and buckets 5030 if err = cleanupBucket(bucketName, c); err != nil { 5031 logError(testName, function, args, startTime, "", "Cleanup failed", err) 5032 return 5033 } 5034 5035 successLogger(testName, function, args, startTime).Info() 5036 } 5037 5038 // Tests bucket re-create errors. 5039 func testMakeBucketErrorV2() { 5040 // initialize logging params 5041 startTime := time.Now() 5042 testName := getFuncName() 5043 function := "MakeBucket(bucketName, region)" 5044 args := map[string]interface{}{ 5045 "bucketName": "", 5046 "region": "eu-west-1", 5047 } 5048 5049 if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { 5050 ignoredLog(testName, function, args, startTime, "Skipped region functional tests for non s3 runs").Info() 5051 return 5052 } 5053 5054 // Seed random based on current time. 5055 rand.Seed(time.Now().Unix()) 5056 5057 // Instantiate new minio client object. 5058 c, err := minio.NewV2( 5059 os.Getenv(serverEndpoint), 5060 os.Getenv(accessKey), 5061 os.Getenv(secretKey), 5062 mustParseBool(os.Getenv(enableHTTPS)), 5063 ) 5064 if err != nil { 5065 logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) 5066 return 5067 } 5068 5069 // Enable tracing, write to stderr. 5070 // c.TraceOn(os.Stderr) 5071 5072 // Set user agent. 5073 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 5074 5075 // Generate a new random bucket name. 5076 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 5077 region := "eu-west-1" 5078 args["bucketName"] = bucketName 5079 args["region"] = region 5080 5081 // Make a new bucket in 'eu-west-1'. 5082 if err = c.MakeBucket(bucketName, region); err != nil { 5083 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 5084 return 5085 } 5086 if err = c.MakeBucket(bucketName, region); err == nil { 5087 logError(testName, function, args, startTime, "", "MakeBucket did not fail for existing bucket name", err) 5088 return 5089 } 5090 // Verify valid error response from server. 5091 if minio.ToErrorResponse(err).Code != "BucketAlreadyExists" && 5092 minio.ToErrorResponse(err).Code != "BucketAlreadyOwnedByYou" { 5093 logError(testName, function, args, startTime, "", "Invalid error returned by server", err) 5094 } 5095 // Delete all objects and buckets 5096 if err = cleanupBucket(bucketName, c); err != nil { 5097 logError(testName, function, args, startTime, "", "Cleanup failed", err) 5098 return 5099 } 5100 5101 successLogger(testName, function, args, startTime).Info() 5102 } 5103 5104 // Test get object reader to not throw error on being closed twice. 5105 func testGetObjectClosedTwiceV2() { 5106 // initialize logging params 5107 startTime := time.Now() 5108 testName := getFuncName() 5109 function := "MakeBucket(bucketName, region)" 5110 args := map[string]interface{}{ 5111 "bucketName": "", 5112 "region": "eu-west-1", 5113 } 5114 5115 // Seed random based on current time. 5116 rand.Seed(time.Now().Unix()) 5117 5118 // Instantiate new minio client object. 5119 c, err := minio.NewV2( 5120 os.Getenv(serverEndpoint), 5121 os.Getenv(accessKey), 5122 os.Getenv(secretKey), 5123 mustParseBool(os.Getenv(enableHTTPS)), 5124 ) 5125 if err != nil { 5126 logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) 5127 return 5128 } 5129 5130 // Enable tracing, write to stderr. 5131 // c.TraceOn(os.Stderr) 5132 5133 // Set user agent. 5134 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 5135 5136 // Generate a new random bucket name. 5137 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 5138 args["bucketName"] = bucketName 5139 5140 // Make a new bucket. 5141 err = c.MakeBucket(bucketName, "us-east-1") 5142 if err != nil { 5143 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 5144 return 5145 } 5146 5147 // Generate 33K of data. 5148 bufSize := dataFileMap["datafile-33-kB"] 5149 var reader = getDataReader("datafile-33-kB") 5150 defer reader.Close() 5151 5152 // Save the data 5153 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 5154 args["objectName"] = objectName 5155 5156 n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) 5157 if err != nil { 5158 logError(testName, function, args, startTime, "", "PutObject failed", err) 5159 return 5160 } 5161 5162 if n != int64(bufSize) { 5163 logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(n), err) 5164 return 5165 } 5166 5167 // Read the data back 5168 r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) 5169 if err != nil { 5170 logError(testName, function, args, startTime, "", "GetObject failed", err) 5171 return 5172 } 5173 5174 st, err := r.Stat() 5175 if err != nil { 5176 logError(testName, function, args, startTime, "", "Stat failed", err) 5177 return 5178 } 5179 5180 if st.Size != int64(bufSize) { 5181 logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(st.Size), err) 5182 return 5183 } 5184 if err := r.Close(); err != nil { 5185 logError(testName, function, args, startTime, "", "Stat failed", err) 5186 return 5187 } 5188 if err := r.Close(); err == nil { 5189 logError(testName, function, args, startTime, "", "Object is already closed, should return error", err) 5190 return 5191 } 5192 5193 // Delete all objects and buckets 5194 if err = cleanupBucket(bucketName, c); err != nil { 5195 logError(testName, function, args, startTime, "", "Cleanup failed", err) 5196 return 5197 } 5198 5199 successLogger(testName, function, args, startTime).Info() 5200 } 5201 5202 // Tests FPutObject hidden contentType setting 5203 func testFPutObjectV2() { 5204 // initialize logging params 5205 startTime := time.Now() 5206 testName := getFuncName() 5207 function := "FPutObject(bucketName, objectName, fileName, opts)" 5208 args := map[string]interface{}{ 5209 "bucketName": "", 5210 "objectName": "", 5211 "fileName": "", 5212 "opts": "", 5213 } 5214 5215 // Seed random based on current time. 5216 rand.Seed(time.Now().Unix()) 5217 5218 // Instantiate new minio client object. 5219 c, err := minio.NewV2( 5220 os.Getenv(serverEndpoint), 5221 os.Getenv(accessKey), 5222 os.Getenv(secretKey), 5223 mustParseBool(os.Getenv(enableHTTPS)), 5224 ) 5225 if err != nil { 5226 logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) 5227 return 5228 } 5229 5230 // Enable tracing, write to stderr. 5231 // c.TraceOn(os.Stderr) 5232 5233 // Set user agent. 5234 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 5235 5236 // Generate a new random bucket name. 5237 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 5238 args["bucketName"] = bucketName 5239 5240 // Make a new bucket. 5241 err = c.MakeBucket(bucketName, "us-east-1") 5242 if err != nil { 5243 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 5244 return 5245 } 5246 5247 // Make a temp file with 11*1024*1024 bytes of data. 5248 file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") 5249 if err != nil { 5250 logError(testName, function, args, startTime, "", "TempFile creation failed", err) 5251 return 5252 } 5253 5254 r := bytes.NewReader(bytes.Repeat([]byte("b"), 11*1024*1024)) 5255 n, err := io.CopyN(file, r, 11*1024*1024) 5256 if err != nil { 5257 logError(testName, function, args, startTime, "", "Copy failed", err) 5258 return 5259 } 5260 if n != int64(11*1024*1024) { 5261 logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err) 5262 return 5263 } 5264 5265 // Close the file pro-actively for windows. 5266 err = file.Close() 5267 if err != nil { 5268 logError(testName, function, args, startTime, "", "File close failed", err) 5269 return 5270 } 5271 5272 // Set base object name 5273 objectName := bucketName + "FPutObject" 5274 args["objectName"] = objectName 5275 args["fileName"] = file.Name() 5276 5277 // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) 5278 n, err = c.FPutObject(bucketName, objectName+"-standard", file.Name(), minio.PutObjectOptions{ContentType: "application/octet-stream"}) 5279 if err != nil { 5280 logError(testName, function, args, startTime, "", "FPutObject failed", err) 5281 return 5282 } 5283 if n != int64(11*1024*1024) { 5284 logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err) 5285 return 5286 } 5287 5288 // Perform FPutObject with no contentType provided (Expecting application/octet-stream) 5289 args["objectName"] = objectName + "-Octet" 5290 args["contentType"] = "" 5291 5292 n, err = c.FPutObject(bucketName, objectName+"-Octet", file.Name(), minio.PutObjectOptions{}) 5293 if err != nil { 5294 logError(testName, function, args, startTime, "", "FPutObject failed", err) 5295 return 5296 } 5297 if n != int64(11*1024*1024) { 5298 logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err) 5299 return 5300 } 5301 5302 // Add extension to temp file name 5303 fileName := file.Name() 5304 err = os.Rename(fileName, fileName+".gtar") 5305 if err != nil { 5306 logError(testName, function, args, startTime, "", "Rename failed", err) 5307 return 5308 } 5309 5310 // Perform FPutObject with no contentType provided (Expecting application/x-gtar) 5311 args["objectName"] = objectName + "-Octet" 5312 args["contentType"] = "" 5313 args["fileName"] = fileName + ".gtar" 5314 5315 n, err = c.FPutObject(bucketName, objectName+"-GTar", fileName+".gtar", minio.PutObjectOptions{}) 5316 if err != nil { 5317 logError(testName, function, args, startTime, "", "FPutObject failed", err) 5318 return 5319 } 5320 if n != int64(11*1024*1024) { 5321 logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err) 5322 return 5323 } 5324 5325 // Check headers 5326 rStandard, err := c.StatObject(bucketName, objectName+"-standard", minio.StatObjectOptions{}) 5327 if err != nil { 5328 logError(testName, function, args, startTime, "", "StatObject failed", err) 5329 return 5330 } 5331 if rStandard.ContentType != "application/octet-stream" { 5332 logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/octet-stream , got "+rStandard.ContentType, err) 5333 return 5334 } 5335 5336 rOctet, err := c.StatObject(bucketName, objectName+"-Octet", minio.StatObjectOptions{}) 5337 if err != nil { 5338 logError(testName, function, args, startTime, "", "StatObject failed", err) 5339 return 5340 } 5341 if rOctet.ContentType != "application/octet-stream" { 5342 logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/octet-stream , got "+rOctet.ContentType, err) 5343 return 5344 } 5345 5346 rGTar, err := c.StatObject(bucketName, objectName+"-GTar", minio.StatObjectOptions{}) 5347 if err != nil { 5348 logError(testName, function, args, startTime, "", "StatObject failed", err) 5349 return 5350 } 5351 if rGTar.ContentType != "application/x-gtar" && rGTar.ContentType != "application/octet-stream" { 5352 logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/x-gtar , got "+rGTar.ContentType, err) 5353 return 5354 } 5355 5356 rGTar, err = c.StatObjectWithContext(context.Background(), bucketName, objectName+"-GTar", minio.StatObjectOptions{}) 5357 if err != nil { 5358 logError(testName, function, args, startTime, "", "StatObject failed", err) 5359 return 5360 } 5361 if rGTar.ContentType != "application/x-gtar" && rGTar.ContentType != "application/octet-stream" { 5362 logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/x-gtar , got "+rGTar.ContentType, err) 5363 return 5364 } 5365 5366 // Delete all objects and buckets 5367 if err = cleanupBucket(bucketName, c); err != nil { 5368 logError(testName, function, args, startTime, "", "Cleanup failed", err) 5369 return 5370 } 5371 5372 os.Remove(fileName + ".gtar") 5373 successLogger(testName, function, args, startTime).Info() 5374 } 5375 5376 // Tests various bucket supported formats. 5377 func testMakeBucketRegionsV2() { 5378 // initialize logging params 5379 startTime := time.Now() 5380 testName := getFuncName() 5381 function := "MakeBucket(bucketName, region)" 5382 args := map[string]interface{}{ 5383 "bucketName": "", 5384 "region": "eu-west-1", 5385 } 5386 5387 if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { 5388 ignoredLog(testName, function, args, startTime, "Skipped region functional tests for non s3 runs").Info() 5389 return 5390 } 5391 5392 // Seed random based on current time. 5393 rand.Seed(time.Now().Unix()) 5394 5395 // Instantiate new minio client object. 5396 c, err := minio.NewV2( 5397 os.Getenv(serverEndpoint), 5398 os.Getenv(accessKey), 5399 os.Getenv(secretKey), 5400 mustParseBool(os.Getenv(enableHTTPS)), 5401 ) 5402 if err != nil { 5403 logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) 5404 return 5405 } 5406 5407 // Enable tracing, write to stderr. 5408 // c.TraceOn(os.Stderr) 5409 5410 // Set user agent. 5411 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 5412 5413 // Generate a new random bucket name. 5414 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 5415 args["bucketName"] = bucketName 5416 5417 // Make a new bucket in 'eu-central-1'. 5418 if err = c.MakeBucket(bucketName, "eu-west-1"); err != nil { 5419 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 5420 return 5421 } 5422 5423 if err = cleanupBucket(bucketName, c); err != nil { 5424 logError(testName, function, args, startTime, "", "Cleanup failed", err) 5425 return 5426 } 5427 5428 // Make a new bucket with '.' in its name, in 'us-west-2'. This 5429 // request is internally staged into a path style instead of 5430 // virtual host style. 5431 if err = c.MakeBucket(bucketName+".withperiod", "us-west-2"); err != nil { 5432 args["bucketName"] = bucketName + ".withperiod" 5433 args["region"] = "us-west-2" 5434 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 5435 return 5436 } 5437 5438 // Delete all objects and buckets 5439 if err = cleanupBucket(bucketName+".withperiod", c); err != nil { 5440 logError(testName, function, args, startTime, "", "Cleanup failed", err) 5441 return 5442 } 5443 5444 successLogger(testName, function, args, startTime).Info() 5445 } 5446 5447 // Tests get object ReaderSeeker interface methods. 5448 func testGetObjectReadSeekFunctionalV2() { 5449 // initialize logging params 5450 startTime := time.Now() 5451 testName := getFuncName() 5452 function := "GetObject(bucketName, objectName)" 5453 args := map[string]interface{}{} 5454 5455 // Seed random based on current time. 5456 rand.Seed(time.Now().Unix()) 5457 5458 // Instantiate new minio client object. 5459 c, err := minio.NewV2( 5460 os.Getenv(serverEndpoint), 5461 os.Getenv(accessKey), 5462 os.Getenv(secretKey), 5463 mustParseBool(os.Getenv(enableHTTPS)), 5464 ) 5465 if err != nil { 5466 logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) 5467 return 5468 } 5469 5470 // Enable tracing, write to stderr. 5471 // c.TraceOn(os.Stderr) 5472 5473 // Set user agent. 5474 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 5475 5476 // Generate a new random bucket name. 5477 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 5478 args["bucketName"] = bucketName 5479 5480 // Make a new bucket. 5481 err = c.MakeBucket(bucketName, "us-east-1") 5482 if err != nil { 5483 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 5484 return 5485 } 5486 5487 // Generate 33K of data. 5488 bufSize := dataFileMap["datafile-33-kB"] 5489 var reader = getDataReader("datafile-33-kB") 5490 defer reader.Close() 5491 5492 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 5493 args["objectName"] = objectName 5494 5495 buf, err := ioutil.ReadAll(reader) 5496 if err != nil { 5497 logError(testName, function, args, startTime, "", "ReadAll failed", err) 5498 return 5499 } 5500 5501 // Save the data. 5502 n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) 5503 if err != nil { 5504 logError(testName, function, args, startTime, "", "PutObject failed", err) 5505 return 5506 } 5507 5508 if n != int64(bufSize) { 5509 logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+" got "+string(n), err) 5510 return 5511 } 5512 5513 // Read the data back 5514 r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) 5515 if err != nil { 5516 logError(testName, function, args, startTime, "", "GetObject failed", err) 5517 return 5518 } 5519 defer r.Close() 5520 5521 st, err := r.Stat() 5522 if err != nil { 5523 logError(testName, function, args, startTime, "", "Stat failed", err) 5524 return 5525 } 5526 5527 if st.Size != int64(bufSize) { 5528 logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+" got "+string(st.Size), err) 5529 return 5530 } 5531 5532 offset := int64(2048) 5533 n, err = r.Seek(offset, 0) 5534 if err != nil { 5535 logError(testName, function, args, startTime, "", "Seek failed", err) 5536 return 5537 } 5538 if n != offset { 5539 logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset)+" got "+string(n), err) 5540 return 5541 } 5542 n, err = r.Seek(0, 1) 5543 if err != nil { 5544 logError(testName, function, args, startTime, "", "Seek failed", err) 5545 return 5546 } 5547 if n != offset { 5548 logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset)+" got "+string(n), err) 5549 return 5550 } 5551 _, err = r.Seek(offset, 2) 5552 if err == nil { 5553 logError(testName, function, args, startTime, "", "Seek on positive offset for whence '2' should error out", err) 5554 return 5555 } 5556 n, err = r.Seek(-offset, 2) 5557 if err != nil { 5558 logError(testName, function, args, startTime, "", "Seek failed", err) 5559 return 5560 } 5561 if n != st.Size-offset { 5562 logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(st.Size-offset)+" got "+string(n), err) 5563 return 5564 } 5565 5566 var buffer1 bytes.Buffer 5567 if _, err = io.CopyN(&buffer1, r, st.Size); err != nil { 5568 if err != io.EOF { 5569 logError(testName, function, args, startTime, "", "Copy failed", err) 5570 return 5571 } 5572 } 5573 if !bytes.Equal(buf[len(buf)-int(offset):], buffer1.Bytes()) { 5574 logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) 5575 return 5576 } 5577 5578 // Seek again and read again. 5579 n, err = r.Seek(offset-1, 0) 5580 if err != nil { 5581 logError(testName, function, args, startTime, "", "Seek failed", err) 5582 return 5583 } 5584 if n != (offset - 1) { 5585 logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset-1)+" got "+string(n), err) 5586 return 5587 } 5588 5589 var buffer2 bytes.Buffer 5590 if _, err = io.CopyN(&buffer2, r, st.Size); err != nil { 5591 if err != io.EOF { 5592 logError(testName, function, args, startTime, "", "Copy failed", err) 5593 return 5594 } 5595 } 5596 // Verify now lesser bytes. 5597 if !bytes.Equal(buf[2047:], buffer2.Bytes()) { 5598 logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) 5599 return 5600 } 5601 5602 // Delete all objects and buckets 5603 if err = cleanupBucket(bucketName, c); err != nil { 5604 logError(testName, function, args, startTime, "", "Cleanup failed", err) 5605 return 5606 } 5607 5608 successLogger(testName, function, args, startTime).Info() 5609 } 5610 5611 // Tests get object ReaderAt interface methods. 5612 func testGetObjectReadAtFunctionalV2() { 5613 // initialize logging params 5614 startTime := time.Now() 5615 testName := getFuncName() 5616 function := "GetObject(bucketName, objectName)" 5617 args := map[string]interface{}{} 5618 5619 // Seed random based on current time. 5620 rand.Seed(time.Now().Unix()) 5621 5622 // Instantiate new minio client object. 5623 c, err := minio.NewV2( 5624 os.Getenv(serverEndpoint), 5625 os.Getenv(accessKey), 5626 os.Getenv(secretKey), 5627 mustParseBool(os.Getenv(enableHTTPS)), 5628 ) 5629 if err != nil { 5630 logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) 5631 return 5632 } 5633 5634 // Enable tracing, write to stderr. 5635 // c.TraceOn(os.Stderr) 5636 5637 // Set user agent. 5638 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 5639 5640 // Generate a new random bucket name. 5641 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 5642 args["bucketName"] = bucketName 5643 5644 // Make a new bucket. 5645 err = c.MakeBucket(bucketName, "us-east-1") 5646 if err != nil { 5647 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 5648 return 5649 } 5650 5651 // Generate 33K of data. 5652 bufSize := dataFileMap["datafile-33-kB"] 5653 var reader = getDataReader("datafile-33-kB") 5654 defer reader.Close() 5655 5656 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 5657 args["objectName"] = objectName 5658 5659 buf, err := ioutil.ReadAll(reader) 5660 if err != nil { 5661 logError(testName, function, args, startTime, "", "ReadAll failed", err) 5662 return 5663 } 5664 5665 // Save the data 5666 n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) 5667 if err != nil { 5668 logError(testName, function, args, startTime, "", "PutObject failed", err) 5669 return 5670 } 5671 5672 if n != int64(bufSize) { 5673 logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(n), err) 5674 return 5675 } 5676 5677 // Read the data back 5678 r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) 5679 if err != nil { 5680 logError(testName, function, args, startTime, "", "GetObject failed", err) 5681 return 5682 } 5683 defer r.Close() 5684 5685 st, err := r.Stat() 5686 if err != nil { 5687 logError(testName, function, args, startTime, "", "Stat failed", err) 5688 return 5689 } 5690 5691 if st.Size != int64(bufSize) { 5692 logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(st.Size), err) 5693 return 5694 } 5695 5696 offset := int64(2048) 5697 5698 // Read directly 5699 buf2 := make([]byte, 512) 5700 buf3 := make([]byte, 512) 5701 buf4 := make([]byte, 512) 5702 5703 m, err := r.ReadAt(buf2, offset) 5704 if err != nil { 5705 logError(testName, function, args, startTime, "", "ReadAt failed", err) 5706 return 5707 } 5708 if m != len(buf2) { 5709 logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+" got "+string(m), err) 5710 return 5711 } 5712 if !bytes.Equal(buf2, buf[offset:offset+512]) { 5713 logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) 5714 return 5715 } 5716 offset += 512 5717 m, err = r.ReadAt(buf3, offset) 5718 if err != nil { 5719 logError(testName, function, args, startTime, "", "ReadAt failed", err) 5720 return 5721 } 5722 if m != len(buf3) { 5723 logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+" got "+string(m), err) 5724 return 5725 } 5726 if !bytes.Equal(buf3, buf[offset:offset+512]) { 5727 logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) 5728 return 5729 } 5730 offset += 512 5731 m, err = r.ReadAt(buf4, offset) 5732 if err != nil { 5733 logError(testName, function, args, startTime, "", "ReadAt failed", err) 5734 return 5735 } 5736 if m != len(buf4) { 5737 logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+" got "+string(m), err) 5738 return 5739 } 5740 if !bytes.Equal(buf4, buf[offset:offset+512]) { 5741 logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) 5742 return 5743 } 5744 5745 buf5 := make([]byte, n) 5746 // Read the whole object. 5747 m, err = r.ReadAt(buf5, 0) 5748 if err != nil { 5749 if err != io.EOF { 5750 logError(testName, function, args, startTime, "", "ReadAt failed", err) 5751 return 5752 } 5753 } 5754 if m != len(buf5) { 5755 logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+" got "+string(m), err) 5756 return 5757 } 5758 if !bytes.Equal(buf, buf5) { 5759 logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err) 5760 return 5761 } 5762 5763 buf6 := make([]byte, n+1) 5764 // Read the whole object and beyond. 5765 _, err = r.ReadAt(buf6, 0) 5766 if err != nil { 5767 if err != io.EOF { 5768 logError(testName, function, args, startTime, "", "ReadAt failed", err) 5769 return 5770 } 5771 } 5772 // Delete all objects and buckets 5773 if err = cleanupBucket(bucketName, c); err != nil { 5774 logError(testName, function, args, startTime, "", "Cleanup failed", err) 5775 return 5776 } 5777 5778 successLogger(testName, function, args, startTime).Info() 5779 } 5780 5781 // Tests copy object 5782 func testCopyObjectV2() { 5783 // initialize logging params 5784 startTime := time.Now() 5785 testName := getFuncName() 5786 function := "CopyObject(destination, source)" 5787 args := map[string]interface{}{} 5788 5789 // Seed random based on current time. 5790 rand.Seed(time.Now().Unix()) 5791 5792 // Instantiate new minio client object 5793 c, err := minio.NewV2( 5794 os.Getenv(serverEndpoint), 5795 os.Getenv(accessKey), 5796 os.Getenv(secretKey), 5797 mustParseBool(os.Getenv(enableHTTPS)), 5798 ) 5799 if err != nil { 5800 logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) 5801 return 5802 } 5803 5804 // Enable tracing, write to stderr. 5805 // c.TraceOn(os.Stderr) 5806 5807 // Set user agent. 5808 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 5809 5810 // Generate a new random bucket name. 5811 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 5812 5813 // Make a new bucket in 'us-east-1' (source bucket). 5814 err = c.MakeBucket(bucketName, "us-east-1") 5815 if err != nil { 5816 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 5817 return 5818 } 5819 5820 // Make a new bucket in 'us-east-1' (destination bucket). 5821 err = c.MakeBucket(bucketName+"-copy", "us-east-1") 5822 if err != nil { 5823 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 5824 return 5825 } 5826 5827 // Generate 33K of data. 5828 bufSize := dataFileMap["datafile-33-kB"] 5829 var reader = getDataReader("datafile-33-kB") 5830 defer reader.Close() 5831 5832 // Save the data 5833 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 5834 n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) 5835 if err != nil { 5836 logError(testName, function, args, startTime, "", "PutObject failed", err) 5837 return 5838 } 5839 5840 if n != int64(bufSize) { 5841 logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+" got "+string(n), err) 5842 return 5843 } 5844 5845 r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) 5846 if err != nil { 5847 logError(testName, function, args, startTime, "", "GetObject failed", err) 5848 return 5849 } 5850 // Check the various fields of source object against destination object. 5851 objInfo, err := r.Stat() 5852 if err != nil { 5853 logError(testName, function, args, startTime, "", "Stat failed", err) 5854 return 5855 } 5856 r.Close() 5857 5858 // Copy Source 5859 src := minio.NewSourceInfo(bucketName, objectName, nil) 5860 args["source"] = src 5861 5862 // Set copy conditions. 5863 5864 // All invalid conditions first. 5865 err = src.SetModifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) 5866 if err == nil { 5867 logError(testName, function, args, startTime, "", "SetModifiedSinceCond did not fail for invalid conditions", err) 5868 return 5869 } 5870 err = src.SetUnmodifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) 5871 if err == nil { 5872 logError(testName, function, args, startTime, "", "SetUnmodifiedSinceCond did not fail for invalid conditions", err) 5873 return 5874 } 5875 err = src.SetMatchETagCond("") 5876 if err == nil { 5877 logError(testName, function, args, startTime, "", "SetMatchETagCond did not fail for invalid conditions", err) 5878 return 5879 } 5880 err = src.SetMatchETagExceptCond("") 5881 if err == nil { 5882 logError(testName, function, args, startTime, "", "SetMatchETagExceptCond did not fail for invalid conditions", err) 5883 return 5884 } 5885 5886 err = src.SetModifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) 5887 if err != nil { 5888 logError(testName, function, args, startTime, "", "SetModifiedSinceCond failed", err) 5889 return 5890 } 5891 err = src.SetMatchETagCond(objInfo.ETag) 5892 if err != nil { 5893 logError(testName, function, args, startTime, "", "SetMatchETagCond failed", err) 5894 return 5895 } 5896 5897 dst, err := minio.NewDestinationInfo(bucketName+"-copy", objectName+"-copy", nil, nil) 5898 args["destination"] = dst 5899 if err != nil { 5900 logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) 5901 return 5902 } 5903 5904 // Perform the Copy 5905 err = c.CopyObject(dst, src) 5906 if err != nil { 5907 logError(testName, function, args, startTime, "", "CopyObject failed", err) 5908 return 5909 } 5910 5911 // Source object 5912 r, err = c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) 5913 if err != nil { 5914 logError(testName, function, args, startTime, "", "GetObject failed", err) 5915 return 5916 } 5917 // Destination object 5918 readerCopy, err := c.GetObject(bucketName+"-copy", objectName+"-copy", minio.GetObjectOptions{}) 5919 if err != nil { 5920 logError(testName, function, args, startTime, "", "GetObject failed", err) 5921 return 5922 } 5923 // Check the various fields of source object against destination object. 5924 objInfo, err = r.Stat() 5925 if err != nil { 5926 logError(testName, function, args, startTime, "", "Stat failed", err) 5927 return 5928 } 5929 objInfoCopy, err := readerCopy.Stat() 5930 if err != nil { 5931 logError(testName, function, args, startTime, "", "Stat failed", err) 5932 return 5933 } 5934 if objInfo.Size != objInfoCopy.Size { 5935 logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(objInfoCopy.Size)+" got "+string(objInfo.Size), err) 5936 return 5937 } 5938 5939 // Close all the readers. 5940 r.Close() 5941 readerCopy.Close() 5942 5943 // CopyObject again but with wrong conditions 5944 src = minio.NewSourceInfo(bucketName, objectName, nil) 5945 err = src.SetUnmodifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) 5946 if err != nil { 5947 logError(testName, function, args, startTime, "", "SetUnmodifiedSinceCond failed", err) 5948 return 5949 } 5950 err = src.SetMatchETagExceptCond(objInfo.ETag) 5951 if err != nil { 5952 logError(testName, function, args, startTime, "", "SetMatchETagExceptCond failed", err) 5953 return 5954 } 5955 5956 // Perform the Copy which should fail 5957 err = c.CopyObject(dst, src) 5958 if err == nil { 5959 logError(testName, function, args, startTime, "", "CopyObject did not fail for invalid conditions", err) 5960 return 5961 } 5962 5963 // Delete all objects and buckets 5964 if err = cleanupBucket(bucketName, c); err != nil { 5965 logError(testName, function, args, startTime, "", "Cleanup failed", err) 5966 return 5967 } 5968 if err = cleanupBucket(bucketName+"-copy", c); err != nil { 5969 logError(testName, function, args, startTime, "", "Cleanup failed", err) 5970 return 5971 } 5972 successLogger(testName, function, args, startTime).Info() 5973 } 5974 5975 func testComposeObjectErrorCasesWrapper(c *minio.Client) { 5976 // initialize logging params 5977 startTime := time.Now() 5978 testName := getFuncName() 5979 function := "ComposeObject(destination, sourceList)" 5980 args := map[string]interface{}{} 5981 5982 // Generate a new random bucket name. 5983 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 5984 5985 // Make a new bucket in 'us-east-1' (source bucket). 5986 err := c.MakeBucket(bucketName, "us-east-1") 5987 5988 if err != nil { 5989 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 5990 return 5991 } 5992 5993 // Test that more than 10K source objects cannot be 5994 // concatenated. 5995 srcArr := [10001]minio.SourceInfo{} 5996 srcSlice := srcArr[:] 5997 dst, err := minio.NewDestinationInfo(bucketName, "object", nil, nil) 5998 if err != nil { 5999 logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) 6000 return 6001 } 6002 6003 args["destination"] = dst 6004 // Just explain about srcArr in args["sourceList"] 6005 // to stop having 10,001 null headers logged 6006 args["sourceList"] = "source array of 10,001 elements" 6007 if err := c.ComposeObject(dst, srcSlice); err == nil { 6008 logError(testName, function, args, startTime, "", "Expected error in ComposeObject", err) 6009 return 6010 } else if err.Error() != "There must be as least one and up to 10000 source objects." { 6011 logError(testName, function, args, startTime, "", "Got unexpected error", err) 6012 return 6013 } 6014 6015 // Create a source with invalid offset spec and check that 6016 // error is returned: 6017 // 1. Create the source object. 6018 const badSrcSize = 5 * 1024 * 1024 6019 buf := bytes.Repeat([]byte("1"), badSrcSize) 6020 _, err = c.PutObject(bucketName, "badObject", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) 6021 if err != nil { 6022 logError(testName, function, args, startTime, "", "PutObject failed", err) 6023 return 6024 } 6025 // 2. Set invalid range spec on the object (going beyond 6026 // object size) 6027 badSrc := minio.NewSourceInfo(bucketName, "badObject", nil) 6028 err = badSrc.SetRange(1, badSrcSize) 6029 if err != nil { 6030 logError(testName, function, args, startTime, "", "Setting NewSourceInfo failed", err) 6031 return 6032 } 6033 // 3. ComposeObject call should fail. 6034 if err := c.ComposeObject(dst, []minio.SourceInfo{badSrc}); err == nil { 6035 logError(testName, function, args, startTime, "", "ComposeObject expected to fail", err) 6036 return 6037 } else if !strings.Contains(err.Error(), "has invalid segment-to-copy") { 6038 logError(testName, function, args, startTime, "", "Got invalid error", err) 6039 return 6040 } 6041 6042 // Delete all objects and buckets 6043 if err = cleanupBucket(bucketName, c); err != nil { 6044 logError(testName, function, args, startTime, "", "Cleanup failed", err) 6045 return 6046 } 6047 6048 successLogger(testName, function, args, startTime).Info() 6049 } 6050 6051 // Test expected error cases 6052 func testComposeObjectErrorCasesV2() { 6053 // initialize logging params 6054 startTime := time.Now() 6055 testName := getFuncName() 6056 function := "ComposeObject(destination, sourceList)" 6057 args := map[string]interface{}{} 6058 6059 // Instantiate new minio client object 6060 c, err := minio.NewV2( 6061 os.Getenv(serverEndpoint), 6062 os.Getenv(accessKey), 6063 os.Getenv(secretKey), 6064 mustParseBool(os.Getenv(enableHTTPS)), 6065 ) 6066 if err != nil { 6067 logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) 6068 return 6069 } 6070 6071 testComposeObjectErrorCasesWrapper(c) 6072 } 6073 6074 func testComposeMultipleSources(c *minio.Client) { 6075 // initialize logging params 6076 startTime := time.Now() 6077 testName := getFuncName() 6078 function := "ComposeObject(destination, sourceList)" 6079 args := map[string]interface{}{ 6080 "destination": "", 6081 "sourceList": "", 6082 } 6083 6084 // Generate a new random bucket name. 6085 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 6086 // Make a new bucket in 'us-east-1' (source bucket). 6087 err := c.MakeBucket(bucketName, "us-east-1") 6088 if err != nil { 6089 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 6090 return 6091 } 6092 6093 // Upload a small source object 6094 const srcSize = 1024 * 1024 * 5 6095 buf := bytes.Repeat([]byte("1"), srcSize) 6096 _, err = c.PutObject(bucketName, "srcObject", bytes.NewReader(buf), int64(srcSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) 6097 if err != nil { 6098 logError(testName, function, args, startTime, "", "PutObject failed", err) 6099 return 6100 } 6101 6102 // We will append 10 copies of the object. 6103 srcs := []minio.SourceInfo{} 6104 for i := 0; i < 10; i++ { 6105 srcs = append(srcs, minio.NewSourceInfo(bucketName, "srcObject", nil)) 6106 } 6107 // make the last part very small 6108 err = srcs[9].SetRange(0, 0) 6109 if err != nil { 6110 logError(testName, function, args, startTime, "", "SetRange failed", err) 6111 return 6112 } 6113 args["sourceList"] = srcs 6114 6115 dst, err := minio.NewDestinationInfo(bucketName, "dstObject", nil, nil) 6116 args["destination"] = dst 6117 6118 if err != nil { 6119 logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) 6120 return 6121 } 6122 err = c.ComposeObject(dst, srcs) 6123 if err != nil { 6124 logError(testName, function, args, startTime, "", "ComposeObject failed", err) 6125 return 6126 } 6127 6128 objProps, err := c.StatObject(bucketName, "dstObject", minio.StatObjectOptions{}) 6129 if err != nil { 6130 logError(testName, function, args, startTime, "", "StatObject failed", err) 6131 return 6132 } 6133 6134 if objProps.Size != 9*srcSize+1 { 6135 logError(testName, function, args, startTime, "", "Size mismatched! Expected "+string(10000*srcSize)+" got "+string(objProps.Size), err) 6136 return 6137 } 6138 // Delete all objects and buckets 6139 if err = cleanupBucket(bucketName, c); err != nil { 6140 logError(testName, function, args, startTime, "", "Cleanup failed", err) 6141 return 6142 } 6143 successLogger(testName, function, args, startTime).Info() 6144 } 6145 6146 // Test concatenating multiple objects objects 6147 func testCompose10KSourcesV2() { 6148 // initialize logging params 6149 startTime := time.Now() 6150 testName := getFuncName() 6151 function := "ComposeObject(destination, sourceList)" 6152 args := map[string]interface{}{} 6153 6154 // Instantiate new minio client object 6155 c, err := minio.NewV2( 6156 os.Getenv(serverEndpoint), 6157 os.Getenv(accessKey), 6158 os.Getenv(secretKey), 6159 mustParseBool(os.Getenv(enableHTTPS)), 6160 ) 6161 if err != nil { 6162 logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) 6163 return 6164 } 6165 6166 testComposeMultipleSources(c) 6167 } 6168 6169 func testEncryptedEmptyObject() { 6170 // initialize logging params 6171 startTime := time.Now() 6172 testName := getFuncName() 6173 function := "PutObject(bucketName, objectName, reader, objectSize, opts)" 6174 args := map[string]interface{}{} 6175 6176 // Instantiate new minio client object 6177 c, err := minio.NewV4( 6178 os.Getenv(serverEndpoint), 6179 os.Getenv(accessKey), 6180 os.Getenv(secretKey), 6181 mustParseBool(os.Getenv(enableHTTPS)), 6182 ) 6183 if err != nil { 6184 logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) 6185 return 6186 } 6187 6188 // Generate a new random bucket name. 6189 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 6190 args["bucketName"] = bucketName 6191 // Make a new bucket in 'us-east-1' (source bucket). 6192 err = c.MakeBucket(bucketName, "us-east-1") 6193 if err != nil { 6194 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 6195 return 6196 } 6197 6198 sse := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"object")) 6199 6200 // 1. create an sse-c encrypted object to copy by uploading 6201 const srcSize = 0 6202 var buf []byte // Empty buffer 6203 args["objectName"] = "object" 6204 _, err = c.PutObject(bucketName, "object", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ServerSideEncryption: sse}) 6205 if err != nil { 6206 logError(testName, function, args, startTime, "", "PutObject call failed", err) 6207 return 6208 } 6209 6210 // 2. Test CopyObject for an empty object 6211 dstInfo, err := minio.NewDestinationInfo(bucketName, "new-object", sse, nil) 6212 if err != nil { 6213 args["objectName"] = "new-object" 6214 function = "NewDestinationInfo(bucketName, objectName, sse, userMetadata)" 6215 logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) 6216 return 6217 } 6218 srcInfo := minio.NewSourceInfo(bucketName, "object", sse) 6219 if err = c.CopyObject(dstInfo, srcInfo); err != nil { 6220 function = "CopyObject(dstInfo, srcInfo)" 6221 logError(testName, function, map[string]interface{}{}, startTime, "", "CopyObject failed", err) 6222 return 6223 } 6224 6225 // 3. Test Key rotation 6226 newSSE := encrypt.DefaultPBKDF([]byte("Don't Panic"), []byte(bucketName+"new-object")) 6227 dstInfo, err = minio.NewDestinationInfo(bucketName, "new-object", newSSE, nil) 6228 if err != nil { 6229 args["objectName"] = "new-object" 6230 function = "NewDestinationInfo(bucketName, objectName, encryptSSEC, userMetadata)" 6231 logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) 6232 return 6233 } 6234 6235 srcInfo = minio.NewSourceInfo(bucketName, "new-object", sse) 6236 if err = c.CopyObject(dstInfo, srcInfo); err != nil { 6237 function = "CopyObject(dstInfo, srcInfo)" 6238 logError(testName, function, map[string]interface{}{}, startTime, "", "CopyObject with key rotation failed", err) 6239 return 6240 } 6241 6242 // 4. Download the object. 6243 reader, err := c.GetObject(bucketName, "new-object", minio.GetObjectOptions{ServerSideEncryption: newSSE}) 6244 if err != nil { 6245 logError(testName, function, args, startTime, "", "GetObject failed", err) 6246 return 6247 } 6248 defer reader.Close() 6249 6250 decBytes, err := ioutil.ReadAll(reader) 6251 if err != nil { 6252 logError(testName, function, map[string]interface{}{}, startTime, "", "ReadAll failed", err) 6253 return 6254 } 6255 if !bytes.Equal(decBytes, buf) { 6256 logError(testName, function, map[string]interface{}{}, startTime, "", "Downloaded object doesn't match the empty encrypted object", err) 6257 return 6258 } 6259 // Delete all objects and buckets 6260 delete(args, "objectName") 6261 if err = cleanupBucket(bucketName, c); err != nil { 6262 logError(testName, function, args, startTime, "", "Cleanup failed", err) 6263 return 6264 } 6265 6266 successLogger(testName, function, args, startTime).Info() 6267 } 6268 6269 func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc, sseDst encrypt.ServerSide) { 6270 // initialize logging params 6271 startTime := time.Now() 6272 testName := getFuncName() 6273 function := "CopyObject(destination, source)" 6274 args := map[string]interface{}{} 6275 var srcEncryption, dstEncryption encrypt.ServerSide 6276 6277 // Make a new bucket in 'us-east-1' (source bucket). 6278 err := c.MakeBucket(bucketName, "us-east-1") 6279 if err != nil { 6280 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 6281 return 6282 } 6283 6284 // 1. create an sse-c encrypted object to copy by uploading 6285 const srcSize = 1024 * 1024 6286 buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 5MiB 6287 _, err = c.PutObject(bucketName, "srcObject", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ 6288 ServerSideEncryption: sseSrc, 6289 }) 6290 if err != nil { 6291 logError(testName, function, args, startTime, "", "PutObject call failed", err) 6292 return 6293 } 6294 6295 if sseSrc != nil && sseSrc.Type() != encrypt.S3 { 6296 srcEncryption = sseSrc 6297 } 6298 6299 // 2. copy object and change encryption key 6300 src := minio.NewSourceInfo(bucketName, "srcObject", srcEncryption) 6301 args["source"] = src 6302 dst, err := minio.NewDestinationInfo(bucketName, "dstObject", sseDst, nil) 6303 if err != nil { 6304 logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) 6305 return 6306 } 6307 args["destination"] = dst 6308 6309 err = c.CopyObject(dst, src) 6310 if err != nil { 6311 logError(testName, function, args, startTime, "", "CopyObject failed", err) 6312 return 6313 } 6314 6315 if sseDst != nil && sseDst.Type() != encrypt.S3 { 6316 dstEncryption = sseDst 6317 } 6318 // 3. get copied object and check if content is equal 6319 coreClient := minio.Core{c} 6320 reader, _, _, err := coreClient.GetObject(bucketName, "dstObject", minio.GetObjectOptions{ServerSideEncryption: dstEncryption}) 6321 if err != nil { 6322 logError(testName, function, args, startTime, "", "GetObject failed", err) 6323 return 6324 } 6325 6326 decBytes, err := ioutil.ReadAll(reader) 6327 if err != nil { 6328 logError(testName, function, args, startTime, "", "ReadAll failed", err) 6329 return 6330 } 6331 if !bytes.Equal(decBytes, buf) { 6332 logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err) 6333 return 6334 } 6335 reader.Close() 6336 6337 // Test key rotation for source object in-place. 6338 var newSSE encrypt.ServerSide 6339 if sseSrc != nil && sseSrc.Type() == encrypt.SSEC { 6340 newSSE = encrypt.DefaultPBKDF([]byte("Don't Panic"), []byte(bucketName+"srcObject")) // replace key 6341 } 6342 if sseSrc != nil && sseSrc.Type() == encrypt.S3 { 6343 newSSE = encrypt.NewSSE() 6344 } 6345 if newSSE != nil { 6346 dst, err = minio.NewDestinationInfo(bucketName, "srcObject", newSSE, nil) 6347 if err != nil { 6348 logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) 6349 return 6350 } 6351 args["destination"] = dst 6352 6353 err = c.CopyObject(dst, src) 6354 if err != nil { 6355 logError(testName, function, args, startTime, "", "CopyObject failed", err) 6356 return 6357 } 6358 6359 // Get copied object and check if content is equal 6360 reader, _, _, err = coreClient.GetObject(bucketName, "srcObject", minio.GetObjectOptions{ServerSideEncryption: newSSE}) 6361 if err != nil { 6362 logError(testName, function, args, startTime, "", "GetObject failed", err) 6363 return 6364 } 6365 6366 decBytes, err = ioutil.ReadAll(reader) 6367 if err != nil { 6368 logError(testName, function, args, startTime, "", "ReadAll failed", err) 6369 return 6370 } 6371 if !bytes.Equal(decBytes, buf) { 6372 logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err) 6373 return 6374 } 6375 reader.Close() 6376 // Test in-place decryption. 6377 dst, err = minio.NewDestinationInfo(bucketName, "srcObject", nil, nil) 6378 if err != nil { 6379 logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) 6380 return 6381 } 6382 args["destination"] = dst 6383 6384 src = minio.NewSourceInfo(bucketName, "srcObject", newSSE) 6385 args["source"] = src 6386 err = c.CopyObject(dst, src) 6387 if err != nil { 6388 logError(testName, function, args, startTime, "", "CopyObject Key rotation failed", err) 6389 return 6390 } 6391 } 6392 6393 // Get copied decrypted object and check if content is equal 6394 reader, _, _, err = coreClient.GetObject(bucketName, "srcObject", minio.GetObjectOptions{}) 6395 if err != nil { 6396 logError(testName, function, args, startTime, "", "GetObject failed", err) 6397 return 6398 } 6399 defer reader.Close() 6400 6401 decBytes, err = ioutil.ReadAll(reader) 6402 if err != nil { 6403 logError(testName, function, args, startTime, "", "ReadAll failed", err) 6404 return 6405 } 6406 if !bytes.Equal(decBytes, buf) { 6407 logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err) 6408 return 6409 } 6410 6411 // Delete all objects and buckets 6412 if err = cleanupBucket(bucketName, c); err != nil { 6413 logError(testName, function, args, startTime, "", "Cleanup failed", err) 6414 return 6415 } 6416 6417 successLogger(testName, function, args, startTime).Info() 6418 } 6419 6420 // Test encrypted copy object 6421 func testUnencryptedToSSECCopyObject() { 6422 // initialize logging params 6423 startTime := time.Now() 6424 testName := getFuncName() 6425 function := "CopyObject(destination, source)" 6426 args := map[string]interface{}{} 6427 6428 // Instantiate new minio client object 6429 c, err := minio.NewV4( 6430 os.Getenv(serverEndpoint), 6431 os.Getenv(accessKey), 6432 os.Getenv(secretKey), 6433 mustParseBool(os.Getenv(enableHTTPS)), 6434 ) 6435 if err != nil { 6436 logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) 6437 return 6438 } 6439 // Generate a new random bucket name. 6440 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 6441 6442 var sseSrc encrypt.ServerSide 6443 sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject")) 6444 // c.TraceOn(os.Stderr) 6445 testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) 6446 } 6447 6448 // Test encrypted copy object 6449 func testUnencryptedToSSES3CopyObject() { 6450 // initialize logging params 6451 startTime := time.Now() 6452 testName := getFuncName() 6453 function := "CopyObject(destination, source)" 6454 args := map[string]interface{}{} 6455 6456 // Instantiate new minio client object 6457 c, err := minio.NewV4( 6458 os.Getenv(serverEndpoint), 6459 os.Getenv(accessKey), 6460 os.Getenv(secretKey), 6461 mustParseBool(os.Getenv(enableHTTPS)), 6462 ) 6463 if err != nil { 6464 logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) 6465 return 6466 } 6467 // Generate a new random bucket name. 6468 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 6469 6470 var sseSrc encrypt.ServerSide 6471 sseDst := encrypt.NewSSE() 6472 // c.TraceOn(os.Stderr) 6473 testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) 6474 } 6475 6476 // Test encrypted copy object 6477 func testUnencryptedToUnencryptedCopyObject() { 6478 // initialize logging params 6479 startTime := time.Now() 6480 testName := getFuncName() 6481 function := "CopyObject(destination, source)" 6482 args := map[string]interface{}{} 6483 6484 // Instantiate new minio client object 6485 c, err := minio.NewV4( 6486 os.Getenv(serverEndpoint), 6487 os.Getenv(accessKey), 6488 os.Getenv(secretKey), 6489 mustParseBool(os.Getenv(enableHTTPS)), 6490 ) 6491 if err != nil { 6492 logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) 6493 return 6494 } 6495 // Generate a new random bucket name. 6496 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 6497 6498 var sseSrc, sseDst encrypt.ServerSide 6499 // c.TraceOn(os.Stderr) 6500 testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) 6501 } 6502 6503 // Test encrypted copy object 6504 func testEncryptedSSECToSSECCopyObject() { 6505 // initialize logging params 6506 startTime := time.Now() 6507 testName := getFuncName() 6508 function := "CopyObject(destination, source)" 6509 args := map[string]interface{}{} 6510 6511 // Instantiate new minio client object 6512 c, err := minio.NewV4( 6513 os.Getenv(serverEndpoint), 6514 os.Getenv(accessKey), 6515 os.Getenv(secretKey), 6516 mustParseBool(os.Getenv(enableHTTPS)), 6517 ) 6518 if err != nil { 6519 logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) 6520 return 6521 } 6522 // Generate a new random bucket name. 6523 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 6524 6525 sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject")) 6526 sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject")) 6527 // c.TraceOn(os.Stderr) 6528 testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) 6529 } 6530 6531 // Test encrypted copy object 6532 func testEncryptedSSECToSSES3CopyObject() { 6533 // initialize logging params 6534 startTime := time.Now() 6535 testName := getFuncName() 6536 function := "CopyObject(destination, source)" 6537 args := map[string]interface{}{} 6538 6539 // Instantiate new minio client object 6540 c, err := minio.NewV4( 6541 os.Getenv(serverEndpoint), 6542 os.Getenv(accessKey), 6543 os.Getenv(secretKey), 6544 mustParseBool(os.Getenv(enableHTTPS)), 6545 ) 6546 if err != nil { 6547 logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) 6548 return 6549 } 6550 // Generate a new random bucket name. 6551 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 6552 6553 sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject")) 6554 sseDst := encrypt.NewSSE() 6555 // c.TraceOn(os.Stderr) 6556 testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) 6557 } 6558 6559 // Test encrypted copy object 6560 func testEncryptedSSECToUnencryptedCopyObject() { 6561 // initialize logging params 6562 startTime := time.Now() 6563 testName := getFuncName() 6564 function := "CopyObject(destination, source)" 6565 args := map[string]interface{}{} 6566 6567 // Instantiate new minio client object 6568 c, err := minio.NewV4( 6569 os.Getenv(serverEndpoint), 6570 os.Getenv(accessKey), 6571 os.Getenv(secretKey), 6572 mustParseBool(os.Getenv(enableHTTPS)), 6573 ) 6574 if err != nil { 6575 logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) 6576 return 6577 } 6578 // Generate a new random bucket name. 6579 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 6580 6581 sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject")) 6582 var sseDst encrypt.ServerSide 6583 // c.TraceOn(os.Stderr) 6584 testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) 6585 } 6586 6587 // Test encrypted copy object 6588 func testEncryptedSSES3ToSSECCopyObject() { 6589 // initialize logging params 6590 startTime := time.Now() 6591 testName := getFuncName() 6592 function := "CopyObject(destination, source)" 6593 args := map[string]interface{}{} 6594 6595 // Instantiate new minio client object 6596 c, err := minio.NewV4( 6597 os.Getenv(serverEndpoint), 6598 os.Getenv(accessKey), 6599 os.Getenv(secretKey), 6600 mustParseBool(os.Getenv(enableHTTPS)), 6601 ) 6602 if err != nil { 6603 logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) 6604 return 6605 } 6606 // Generate a new random bucket name. 6607 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 6608 6609 sseSrc := encrypt.NewSSE() 6610 sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject")) 6611 // c.TraceOn(os.Stderr) 6612 testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) 6613 } 6614 6615 // Test encrypted copy object 6616 func testEncryptedSSES3ToSSES3CopyObject() { 6617 // initialize logging params 6618 startTime := time.Now() 6619 testName := getFuncName() 6620 function := "CopyObject(destination, source)" 6621 args := map[string]interface{}{} 6622 6623 // Instantiate new minio client object 6624 c, err := minio.NewV4( 6625 os.Getenv(serverEndpoint), 6626 os.Getenv(accessKey), 6627 os.Getenv(secretKey), 6628 mustParseBool(os.Getenv(enableHTTPS)), 6629 ) 6630 if err != nil { 6631 logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) 6632 return 6633 } 6634 // Generate a new random bucket name. 6635 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 6636 6637 sseSrc := encrypt.NewSSE() 6638 sseDst := encrypt.NewSSE() 6639 // c.TraceOn(os.Stderr) 6640 testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) 6641 } 6642 6643 // Test encrypted copy object 6644 func testEncryptedSSES3ToUnencryptedCopyObject() { 6645 // initialize logging params 6646 startTime := time.Now() 6647 testName := getFuncName() 6648 function := "CopyObject(destination, source)" 6649 args := map[string]interface{}{} 6650 6651 // Instantiate new minio client object 6652 c, err := minio.NewV4( 6653 os.Getenv(serverEndpoint), 6654 os.Getenv(accessKey), 6655 os.Getenv(secretKey), 6656 mustParseBool(os.Getenv(enableHTTPS)), 6657 ) 6658 if err != nil { 6659 logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) 6660 return 6661 } 6662 // Generate a new random bucket name. 6663 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 6664 6665 sseSrc := encrypt.NewSSE() 6666 var sseDst encrypt.ServerSide 6667 // c.TraceOn(os.Stderr) 6668 testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) 6669 } 6670 6671 // Test encrypted copy object 6672 func testEncryptedCopyObjectV2() { 6673 // initialize logging params 6674 startTime := time.Now() 6675 testName := getFuncName() 6676 function := "CopyObject(destination, source)" 6677 args := map[string]interface{}{} 6678 6679 // Instantiate new minio client object 6680 c, err := minio.NewV2( 6681 os.Getenv(serverEndpoint), 6682 os.Getenv(accessKey), 6683 os.Getenv(secretKey), 6684 mustParseBool(os.Getenv(enableHTTPS)), 6685 ) 6686 if err != nil { 6687 logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) 6688 return 6689 } 6690 // Generate a new random bucket name. 6691 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 6692 6693 sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject")) 6694 sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject")) 6695 // c.TraceOn(os.Stderr) 6696 testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) 6697 } 6698 6699 func testDecryptedCopyObject() { 6700 // initialize logging params 6701 startTime := time.Now() 6702 testName := getFuncName() 6703 function := "CopyObject(destination, source)" 6704 args := map[string]interface{}{} 6705 6706 // Instantiate new minio client object 6707 c, err := minio.New( 6708 os.Getenv(serverEndpoint), 6709 os.Getenv(accessKey), 6710 os.Getenv(secretKey), 6711 mustParseBool(os.Getenv(enableHTTPS)), 6712 ) 6713 if err != nil { 6714 logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) 6715 return 6716 } 6717 6718 bucketName, objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-"), "object" 6719 if err = c.MakeBucket(bucketName, "us-east-1"); err != nil { 6720 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 6721 return 6722 } 6723 6724 encryption := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)) 6725 _, err = c.PutObject(bucketName, objectName, bytes.NewReader(bytes.Repeat([]byte("a"), 1024*1024)), 1024*1024, minio.PutObjectOptions{ 6726 ServerSideEncryption: encryption, 6727 }) 6728 if err != nil { 6729 logError(testName, function, args, startTime, "", "PutObject call failed", err) 6730 return 6731 } 6732 6733 src := minio.NewSourceInfo(bucketName, objectName, encrypt.SSECopy(encryption)) 6734 args["source"] = src 6735 dst, err := minio.NewDestinationInfo(bucketName, "decrypted-"+objectName, nil, nil) 6736 if err != nil { 6737 logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) 6738 return 6739 } 6740 args["destination"] = dst 6741 6742 if err = c.CopyObject(dst, src); err != nil { 6743 logError(testName, function, args, startTime, "", "CopyObject failed", err) 6744 return 6745 } 6746 if _, err = c.GetObject(bucketName, "decrypted-"+objectName, minio.GetObjectOptions{}); err != nil { 6747 logError(testName, function, args, startTime, "", "GetObject failed", err) 6748 return 6749 } 6750 successLogger(testName, function, args, startTime).Info() 6751 } 6752 6753 func testSSECMultipartEncryptedToSSECCopyObjectPart() { 6754 // initialize logging params 6755 startTime := time.Now() 6756 testName := getFuncName() 6757 function := "CopyObjectPart(destination, source)" 6758 args := map[string]interface{}{} 6759 6760 // Instantiate new minio client object 6761 client, err := minio.NewV4( 6762 os.Getenv(serverEndpoint), 6763 os.Getenv(accessKey), 6764 os.Getenv(secretKey), 6765 mustParseBool(os.Getenv(enableHTTPS)), 6766 ) 6767 if err != nil { 6768 logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) 6769 return 6770 } 6771 6772 // Instantiate new core client object. 6773 c := minio.Core{client} 6774 6775 // Enable tracing, write to stderr. 6776 // c.TraceOn(os.Stderr) 6777 6778 // Set user agent. 6779 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 6780 6781 // Generate a new random bucket name. 6782 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") 6783 6784 // Make a new bucket. 6785 err = c.MakeBucket(bucketName, "us-east-1") 6786 if err != nil { 6787 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 6788 } 6789 defer cleanupBucket(bucketName, client) 6790 // Make a buffer with 6MB of data 6791 buf := bytes.Repeat([]byte("abcdef"), 1024*1024) 6792 6793 // Save the data 6794 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 6795 password := "correct horse battery staple" 6796 srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) 6797 6798 // Upload a 6MB object using multipart mechanism 6799 uploadID, err := c.NewMultipartUpload(bucketName, objectName, minio.PutObjectOptions{ServerSideEncryption: srcencryption}) 6800 if err != nil { 6801 logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) 6802 } 6803 6804 var completeParts []minio.CompletePart 6805 6806 part, err := c.PutObjectPart(bucketName, objectName, uploadID, 1, bytes.NewReader(buf[:5*1024*1024]), 5*1024*1024, "", "", srcencryption) 6807 if err != nil { 6808 logError(testName, function, args, startTime, "", "PutObjectPart call failed", err) 6809 } 6810 completeParts = append(completeParts, minio.CompletePart{PartNumber: part.PartNumber, ETag: part.ETag}) 6811 6812 part, err = c.PutObjectPart(bucketName, objectName, uploadID, 2, bytes.NewReader(buf[5*1024*1024:]), 1024*1024, "", "", srcencryption) 6813 if err != nil { 6814 logError(testName, function, args, startTime, "", "PutObjectPart call failed", err) 6815 } 6816 completeParts = append(completeParts, minio.CompletePart{PartNumber: part.PartNumber, ETag: part.ETag}) 6817 6818 // Complete the multipart upload 6819 _, err = c.CompleteMultipartUpload(bucketName, objectName, uploadID, completeParts) 6820 if err != nil { 6821 logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) 6822 } 6823 6824 // Stat the object and check its length matches 6825 objInfo, err := c.StatObject(bucketName, objectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: srcencryption}}) 6826 if err != nil { 6827 logError(testName, function, args, startTime, "", "StatObject call failed", err) 6828 } 6829 6830 destBucketName := bucketName 6831 destObjectName := objectName + "-dest" 6832 dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName)) 6833 6834 uploadID, err = c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) 6835 if err != nil { 6836 logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) 6837 } 6838 6839 // Content of the destination object will be two copies of 6840 // `objectName` concatenated, followed by first byte of 6841 // `objectName`. 6842 metadata := make(map[string]string) 6843 header := make(http.Header) 6844 encrypt.SSECopy(srcencryption).Marshal(header) 6845 dstencryption.Marshal(header) 6846 for k, v := range header { 6847 metadata[k] = v[0] 6848 } 6849 6850 metadata["x-amz-copy-source-if-match"] = objInfo.ETag 6851 6852 // First of three parts 6853 fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) 6854 if err != nil { 6855 logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) 6856 } 6857 6858 // Second of three parts 6859 sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) 6860 if err != nil { 6861 logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) 6862 } 6863 6864 // Last of three parts 6865 lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) 6866 if err != nil { 6867 logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) 6868 } 6869 6870 // Complete the multipart upload 6871 _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) 6872 if err != nil { 6873 logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) 6874 } 6875 6876 // Stat the object and check its length matches 6877 objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: dstencryption}}) 6878 if err != nil { 6879 logError(testName, function, args, startTime, "", "StatObject call failed", err) 6880 } 6881 6882 if objInfo.Size != (6*1024*1024)*2+1 { 6883 logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) 6884 } 6885 6886 // Now we read the data back 6887 getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption} 6888 getOpts.SetRange(0, 6*1024*1024-1) 6889 r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) 6890 if err != nil { 6891 logError(testName, function, args, startTime, "", "GetObject call failed", err) 6892 } 6893 getBuf := make([]byte, 6*1024*1024) 6894 _, err = io.ReadFull(r, getBuf) 6895 if err != nil { 6896 logError(testName, function, args, startTime, "", "Read buffer failed", err) 6897 } 6898 if !bytes.Equal(getBuf, buf) { 6899 logError(testName, function, args, startTime, "", "Got unexpected data in first 6MB", err) 6900 } 6901 6902 getOpts.SetRange(6*1024*1024, 0) 6903 r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) 6904 if err != nil { 6905 logError(testName, function, args, startTime, "", "GetObject call failed", err) 6906 } 6907 getBuf = make([]byte, 6*1024*1024+1) 6908 _, err = io.ReadFull(r, getBuf) 6909 if err != nil { 6910 logError(testName, function, args, startTime, "", "Read buffer failed", err) 6911 } 6912 if !bytes.Equal(getBuf[:6*1024*1024], buf) { 6913 logError(testName, function, args, startTime, "", "Got unexpected data in second 6MB", err) 6914 } 6915 if getBuf[6*1024*1024] != buf[0] { 6916 logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) 6917 } 6918 6919 successLogger(testName, function, args, startTime).Info() 6920 6921 // Do not need to remove destBucketName its same as bucketName. 6922 } 6923 6924 // Test Core CopyObjectPart implementation 6925 func testSSECEncryptedToSSECCopyObjectPart() { 6926 // initialize logging params 6927 startTime := time.Now() 6928 testName := getFuncName() 6929 function := "CopyObjectPart(destination, source)" 6930 args := map[string]interface{}{} 6931 6932 // Instantiate new minio client object 6933 client, err := minio.NewV4( 6934 os.Getenv(serverEndpoint), 6935 os.Getenv(accessKey), 6936 os.Getenv(secretKey), 6937 mustParseBool(os.Getenv(enableHTTPS)), 6938 ) 6939 if err != nil { 6940 logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) 6941 return 6942 } 6943 6944 // Instantiate new core client object. 6945 c := minio.Core{client} 6946 6947 // Enable tracing, write to stderr. 6948 // c.TraceOn(os.Stderr) 6949 6950 // Set user agent. 6951 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 6952 6953 // Generate a new random bucket name. 6954 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") 6955 6956 // Make a new bucket. 6957 err = c.MakeBucket(bucketName, "us-east-1") 6958 if err != nil { 6959 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 6960 } 6961 defer cleanupBucket(bucketName, client) 6962 // Make a buffer with 5MB of data 6963 buf := bytes.Repeat([]byte("abcde"), 1024*1024) 6964 6965 // Save the data 6966 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 6967 password := "correct horse battery staple" 6968 srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) 6969 putmetadata := map[string]string{ 6970 "Content-Type": "binary/octet-stream", 6971 } 6972 opts := minio.PutObjectOptions{ 6973 UserMetadata: putmetadata, 6974 ServerSideEncryption: srcencryption, 6975 } 6976 objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) 6977 if err != nil { 6978 logError(testName, function, args, startTime, "", "PutObject call failed", err) 6979 } 6980 6981 if objInfo.Size != int64(len(buf)) { 6982 logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) 6983 } 6984 6985 destBucketName := bucketName 6986 destObjectName := objectName + "-dest" 6987 dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName)) 6988 6989 uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) 6990 if err != nil { 6991 logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) 6992 } 6993 6994 // Content of the destination object will be two copies of 6995 // `objectName` concatenated, followed by first byte of 6996 // `objectName`. 6997 metadata := make(map[string]string) 6998 header := make(http.Header) 6999 encrypt.SSECopy(srcencryption).Marshal(header) 7000 dstencryption.Marshal(header) 7001 for k, v := range header { 7002 metadata[k] = v[0] 7003 } 7004 7005 metadata["x-amz-copy-source-if-match"] = objInfo.ETag 7006 7007 // First of three parts 7008 fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) 7009 if err != nil { 7010 logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) 7011 } 7012 7013 // Second of three parts 7014 sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) 7015 if err != nil { 7016 logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) 7017 } 7018 7019 // Last of three parts 7020 lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) 7021 if err != nil { 7022 logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) 7023 } 7024 7025 // Complete the multipart upload 7026 _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) 7027 if err != nil { 7028 logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) 7029 } 7030 7031 // Stat the object and check its length matches 7032 objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: dstencryption}}) 7033 if err != nil { 7034 logError(testName, function, args, startTime, "", "StatObject call failed", err) 7035 } 7036 7037 if objInfo.Size != (5*1024*1024)*2+1 { 7038 logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) 7039 } 7040 7041 // Now we read the data back 7042 getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption} 7043 getOpts.SetRange(0, 5*1024*1024-1) 7044 r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) 7045 if err != nil { 7046 logError(testName, function, args, startTime, "", "GetObject call failed", err) 7047 } 7048 getBuf := make([]byte, 5*1024*1024) 7049 _, err = io.ReadFull(r, getBuf) 7050 if err != nil { 7051 logError(testName, function, args, startTime, "", "Read buffer failed", err) 7052 } 7053 if !bytes.Equal(getBuf, buf) { 7054 logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) 7055 } 7056 7057 getOpts.SetRange(5*1024*1024, 0) 7058 r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) 7059 if err != nil { 7060 logError(testName, function, args, startTime, "", "GetObject call failed", err) 7061 } 7062 getBuf = make([]byte, 5*1024*1024+1) 7063 _, err = io.ReadFull(r, getBuf) 7064 if err != nil { 7065 logError(testName, function, args, startTime, "", "Read buffer failed", err) 7066 } 7067 if !bytes.Equal(getBuf[:5*1024*1024], buf) { 7068 logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) 7069 } 7070 if getBuf[5*1024*1024] != buf[0] { 7071 logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) 7072 } 7073 7074 successLogger(testName, function, args, startTime).Info() 7075 7076 // Do not need to remove destBucketName its same as bucketName. 7077 } 7078 7079 // Test Core CopyObjectPart implementation for SSEC encrypted to unencrypted copy 7080 func testSSECEncryptedToUnencryptedCopyPart() { 7081 // initialize logging params 7082 startTime := time.Now() 7083 testName := getFuncName() 7084 function := "CopyObjectPart(destination, source)" 7085 args := map[string]interface{}{} 7086 7087 // Instantiate new minio client object 7088 client, err := minio.NewV4( 7089 os.Getenv(serverEndpoint), 7090 os.Getenv(accessKey), 7091 os.Getenv(secretKey), 7092 mustParseBool(os.Getenv(enableHTTPS)), 7093 ) 7094 if err != nil { 7095 logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) 7096 return 7097 } 7098 7099 // Instantiate new core client object. 7100 c := minio.Core{client} 7101 7102 // Enable tracing, write to stderr. 7103 // c.TraceOn(os.Stderr) 7104 7105 // Set user agent. 7106 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 7107 7108 // Generate a new random bucket name. 7109 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") 7110 7111 // Make a new bucket. 7112 err = c.MakeBucket(bucketName, "us-east-1") 7113 if err != nil { 7114 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 7115 } 7116 defer cleanupBucket(bucketName, client) 7117 // Make a buffer with 5MB of data 7118 buf := bytes.Repeat([]byte("abcde"), 1024*1024) 7119 7120 // Save the data 7121 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 7122 password := "correct horse battery staple" 7123 srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) 7124 7125 opts := minio.PutObjectOptions{ 7126 UserMetadata: map[string]string{ 7127 "Content-Type": "binary/octet-stream", 7128 }, 7129 ServerSideEncryption: srcencryption, 7130 } 7131 objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) 7132 if err != nil { 7133 logError(testName, function, args, startTime, "", "PutObject call failed", err) 7134 } 7135 7136 if objInfo.Size != int64(len(buf)) { 7137 logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) 7138 } 7139 7140 destBucketName := bucketName 7141 destObjectName := objectName + "-dest" 7142 var dstencryption encrypt.ServerSide 7143 7144 uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) 7145 if err != nil { 7146 logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) 7147 } 7148 7149 // Content of the destination object will be two copies of 7150 // `objectName` concatenated, followed by first byte of 7151 // `objectName`. 7152 metadata := make(map[string]string) 7153 header := make(http.Header) 7154 encrypt.SSECopy(srcencryption).Marshal(header) 7155 for k, v := range header { 7156 metadata[k] = v[0] 7157 } 7158 7159 metadata["x-amz-copy-source-if-match"] = objInfo.ETag 7160 7161 // First of three parts 7162 fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) 7163 if err != nil { 7164 logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) 7165 } 7166 7167 // Second of three parts 7168 sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) 7169 if err != nil { 7170 logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) 7171 } 7172 7173 // Last of three parts 7174 lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) 7175 if err != nil { 7176 logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) 7177 } 7178 7179 // Complete the multipart upload 7180 _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) 7181 if err != nil { 7182 logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) 7183 } 7184 7185 // Stat the object and check its length matches 7186 objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) 7187 if err != nil { 7188 logError(testName, function, args, startTime, "", "StatObject call failed", err) 7189 } 7190 7191 if objInfo.Size != (5*1024*1024)*2+1 { 7192 logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) 7193 } 7194 7195 // Now we read the data back 7196 getOpts := minio.GetObjectOptions{} 7197 getOpts.SetRange(0, 5*1024*1024-1) 7198 r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) 7199 if err != nil { 7200 logError(testName, function, args, startTime, "", "GetObject call failed", err) 7201 } 7202 getBuf := make([]byte, 5*1024*1024) 7203 _, err = io.ReadFull(r, getBuf) 7204 if err != nil { 7205 logError(testName, function, args, startTime, "", "Read buffer failed", err) 7206 } 7207 if !bytes.Equal(getBuf, buf) { 7208 logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) 7209 } 7210 7211 getOpts.SetRange(5*1024*1024, 0) 7212 r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) 7213 if err != nil { 7214 logError(testName, function, args, startTime, "", "GetObject call failed", err) 7215 } 7216 getBuf = make([]byte, 5*1024*1024+1) 7217 _, err = io.ReadFull(r, getBuf) 7218 if err != nil { 7219 logError(testName, function, args, startTime, "", "Read buffer failed", err) 7220 } 7221 if !bytes.Equal(getBuf[:5*1024*1024], buf) { 7222 logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) 7223 } 7224 if getBuf[5*1024*1024] != buf[0] { 7225 logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) 7226 } 7227 7228 successLogger(testName, function, args, startTime).Info() 7229 7230 // Do not need to remove destBucketName its same as bucketName. 7231 } 7232 7233 // Test Core CopyObjectPart implementation for SSEC encrypted to SSE-S3 encrypted copy 7234 func testSSECEncryptedToSSES3CopyObjectPart() { 7235 // initialize logging params 7236 startTime := time.Now() 7237 testName := getFuncName() 7238 function := "CopyObjectPart(destination, source)" 7239 args := map[string]interface{}{} 7240 7241 // Instantiate new minio client object 7242 client, err := minio.NewV4( 7243 os.Getenv(serverEndpoint), 7244 os.Getenv(accessKey), 7245 os.Getenv(secretKey), 7246 mustParseBool(os.Getenv(enableHTTPS)), 7247 ) 7248 if err != nil { 7249 logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) 7250 return 7251 } 7252 7253 // Instantiate new core client object. 7254 c := minio.Core{client} 7255 7256 // Enable tracing, write to stderr. 7257 // c.TraceOn(os.Stderr) 7258 7259 // Set user agent. 7260 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 7261 7262 // Generate a new random bucket name. 7263 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") 7264 7265 // Make a new bucket. 7266 err = c.MakeBucket(bucketName, "us-east-1") 7267 if err != nil { 7268 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 7269 } 7270 defer cleanupBucket(bucketName, client) 7271 // Make a buffer with 5MB of data 7272 buf := bytes.Repeat([]byte("abcde"), 1024*1024) 7273 7274 // Save the data 7275 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 7276 password := "correct horse battery staple" 7277 srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) 7278 putmetadata := map[string]string{ 7279 "Content-Type": "binary/octet-stream", 7280 } 7281 opts := minio.PutObjectOptions{ 7282 UserMetadata: putmetadata, 7283 ServerSideEncryption: srcencryption, 7284 } 7285 7286 objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) 7287 if err != nil { 7288 logError(testName, function, args, startTime, "", "PutObject call failed", err) 7289 } 7290 7291 if objInfo.Size != int64(len(buf)) { 7292 logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) 7293 } 7294 7295 destBucketName := bucketName 7296 destObjectName := objectName + "-dest" 7297 dstencryption := encrypt.NewSSE() 7298 7299 uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) 7300 if err != nil { 7301 logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) 7302 } 7303 7304 // Content of the destination object will be two copies of 7305 // `objectName` concatenated, followed by first byte of 7306 // `objectName`. 7307 metadata := make(map[string]string) 7308 header := make(http.Header) 7309 encrypt.SSECopy(srcencryption).Marshal(header) 7310 dstencryption.Marshal(header) 7311 7312 for k, v := range header { 7313 metadata[k] = v[0] 7314 } 7315 7316 metadata["x-amz-copy-source-if-match"] = objInfo.ETag 7317 7318 // First of three parts 7319 fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) 7320 if err != nil { 7321 logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) 7322 } 7323 7324 // Second of three parts 7325 sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) 7326 if err != nil { 7327 logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) 7328 } 7329 7330 // Last of three parts 7331 lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) 7332 if err != nil { 7333 logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) 7334 } 7335 7336 // Complete the multipart upload 7337 _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) 7338 if err != nil { 7339 logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) 7340 } 7341 7342 // Stat the object and check its length matches 7343 objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) 7344 if err != nil { 7345 logError(testName, function, args, startTime, "", "StatObject call failed", err) 7346 } 7347 7348 if objInfo.Size != (5*1024*1024)*2+1 { 7349 logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) 7350 } 7351 7352 // Now we read the data back 7353 getOpts := minio.GetObjectOptions{} 7354 getOpts.SetRange(0, 5*1024*1024-1) 7355 r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) 7356 if err != nil { 7357 logError(testName, function, args, startTime, "", "GetObject call failed", err) 7358 } 7359 getBuf := make([]byte, 5*1024*1024) 7360 _, err = io.ReadFull(r, getBuf) 7361 if err != nil { 7362 logError(testName, function, args, startTime, "", "Read buffer failed", err) 7363 } 7364 if !bytes.Equal(getBuf, buf) { 7365 logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) 7366 } 7367 7368 getOpts.SetRange(5*1024*1024, 0) 7369 r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) 7370 if err != nil { 7371 logError(testName, function, args, startTime, "", "GetObject call failed", err) 7372 } 7373 getBuf = make([]byte, 5*1024*1024+1) 7374 _, err = io.ReadFull(r, getBuf) 7375 if err != nil { 7376 logError(testName, function, args, startTime, "", "Read buffer failed", err) 7377 } 7378 if !bytes.Equal(getBuf[:5*1024*1024], buf) { 7379 logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) 7380 } 7381 if getBuf[5*1024*1024] != buf[0] { 7382 logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) 7383 } 7384 7385 successLogger(testName, function, args, startTime).Info() 7386 7387 // Do not need to remove destBucketName its same as bucketName. 7388 } 7389 7390 // Test Core CopyObjectPart implementation for unencrypted to SSEC encryption copy part 7391 func testUnencryptedToSSECCopyObjectPart() { 7392 // initialize logging params 7393 startTime := time.Now() 7394 testName := getFuncName() 7395 function := "CopyObjectPart(destination, source)" 7396 args := map[string]interface{}{} 7397 7398 // Instantiate new minio client object 7399 client, err := minio.NewV4( 7400 os.Getenv(serverEndpoint), 7401 os.Getenv(accessKey), 7402 os.Getenv(secretKey), 7403 mustParseBool(os.Getenv(enableHTTPS)), 7404 ) 7405 if err != nil { 7406 logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) 7407 return 7408 } 7409 7410 // Instantiate new core client object. 7411 c := minio.Core{client} 7412 7413 // Enable tracing, write to stderr. 7414 // c.TraceOn(os.Stderr) 7415 7416 // Set user agent. 7417 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 7418 7419 // Generate a new random bucket name. 7420 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") 7421 7422 // Make a new bucket. 7423 err = c.MakeBucket(bucketName, "us-east-1") 7424 if err != nil { 7425 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 7426 } 7427 defer cleanupBucket(bucketName, client) 7428 // Make a buffer with 5MB of data 7429 buf := bytes.Repeat([]byte("abcde"), 1024*1024) 7430 7431 // Save the data 7432 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 7433 password := "correct horse battery staple" 7434 putmetadata := map[string]string{ 7435 "Content-Type": "binary/octet-stream", 7436 } 7437 opts := minio.PutObjectOptions{ 7438 UserMetadata: putmetadata, 7439 } 7440 objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) 7441 if err != nil { 7442 logError(testName, function, args, startTime, "", "PutObject call failed", err) 7443 } 7444 7445 if objInfo.Size != int64(len(buf)) { 7446 logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) 7447 } 7448 7449 destBucketName := bucketName 7450 destObjectName := objectName + "-dest" 7451 dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName)) 7452 7453 uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) 7454 if err != nil { 7455 logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) 7456 } 7457 7458 // Content of the destination object will be two copies of 7459 // `objectName` concatenated, followed by first byte of 7460 // `objectName`. 7461 metadata := make(map[string]string) 7462 header := make(http.Header) 7463 dstencryption.Marshal(header) 7464 for k, v := range header { 7465 metadata[k] = v[0] 7466 } 7467 7468 metadata["x-amz-copy-source-if-match"] = objInfo.ETag 7469 7470 // First of three parts 7471 fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) 7472 if err != nil { 7473 logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) 7474 } 7475 7476 // Second of three parts 7477 sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) 7478 if err != nil { 7479 logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) 7480 } 7481 7482 // Last of three parts 7483 lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) 7484 if err != nil { 7485 logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) 7486 } 7487 7488 // Complete the multipart upload 7489 _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) 7490 if err != nil { 7491 logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) 7492 } 7493 7494 // Stat the object and check its length matches 7495 objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: dstencryption}}) 7496 if err != nil { 7497 logError(testName, function, args, startTime, "", "StatObject call failed", err) 7498 } 7499 7500 if objInfo.Size != (5*1024*1024)*2+1 { 7501 logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) 7502 } 7503 7504 // Now we read the data back 7505 getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption} 7506 getOpts.SetRange(0, 5*1024*1024-1) 7507 r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) 7508 if err != nil { 7509 logError(testName, function, args, startTime, "", "GetObject call failed", err) 7510 } 7511 getBuf := make([]byte, 5*1024*1024) 7512 _, err = io.ReadFull(r, getBuf) 7513 if err != nil { 7514 logError(testName, function, args, startTime, "", "Read buffer failed", err) 7515 } 7516 if !bytes.Equal(getBuf, buf) { 7517 logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) 7518 } 7519 7520 getOpts.SetRange(5*1024*1024, 0) 7521 r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) 7522 if err != nil { 7523 logError(testName, function, args, startTime, "", "GetObject call failed", err) 7524 } 7525 getBuf = make([]byte, 5*1024*1024+1) 7526 _, err = io.ReadFull(r, getBuf) 7527 if err != nil { 7528 logError(testName, function, args, startTime, "", "Read buffer failed", err) 7529 } 7530 if !bytes.Equal(getBuf[:5*1024*1024], buf) { 7531 logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) 7532 } 7533 if getBuf[5*1024*1024] != buf[0] { 7534 logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) 7535 } 7536 7537 successLogger(testName, function, args, startTime).Info() 7538 7539 // Do not need to remove destBucketName its same as bucketName. 7540 } 7541 7542 // Test Core CopyObjectPart implementation for unencrypted to unencrypted copy 7543 func testUnencryptedToUnencryptedCopyPart() { 7544 // initialize logging params 7545 startTime := time.Now() 7546 testName := getFuncName() 7547 function := "CopyObjectPart(destination, source)" 7548 args := map[string]interface{}{} 7549 7550 // Instantiate new minio client object 7551 client, err := minio.NewV4( 7552 os.Getenv(serverEndpoint), 7553 os.Getenv(accessKey), 7554 os.Getenv(secretKey), 7555 mustParseBool(os.Getenv(enableHTTPS)), 7556 ) 7557 if err != nil { 7558 logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) 7559 return 7560 } 7561 7562 // Instantiate new core client object. 7563 c := minio.Core{client} 7564 7565 // Enable tracing, write to stderr. 7566 // c.TraceOn(os.Stderr) 7567 7568 // Set user agent. 7569 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 7570 7571 // Generate a new random bucket name. 7572 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") 7573 7574 // Make a new bucket. 7575 err = c.MakeBucket(bucketName, "us-east-1") 7576 if err != nil { 7577 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 7578 } 7579 defer cleanupBucket(bucketName, client) 7580 // Make a buffer with 5MB of data 7581 buf := bytes.Repeat([]byte("abcde"), 1024*1024) 7582 7583 // Save the data 7584 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 7585 putmetadata := map[string]string{ 7586 "Content-Type": "binary/octet-stream", 7587 } 7588 opts := minio.PutObjectOptions{ 7589 UserMetadata: putmetadata, 7590 } 7591 objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) 7592 if err != nil { 7593 logError(testName, function, args, startTime, "", "PutObject call failed", err) 7594 } 7595 7596 if objInfo.Size != int64(len(buf)) { 7597 logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) 7598 } 7599 7600 destBucketName := bucketName 7601 destObjectName := objectName + "-dest" 7602 7603 uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{}) 7604 if err != nil { 7605 logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) 7606 } 7607 7608 // Content of the destination object will be two copies of 7609 // `objectName` concatenated, followed by first byte of 7610 // `objectName`. 7611 metadata := make(map[string]string) 7612 header := make(http.Header) 7613 for k, v := range header { 7614 metadata[k] = v[0] 7615 } 7616 7617 metadata["x-amz-copy-source-if-match"] = objInfo.ETag 7618 7619 // First of three parts 7620 fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) 7621 if err != nil { 7622 logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) 7623 } 7624 7625 // Second of three parts 7626 sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) 7627 if err != nil { 7628 logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) 7629 } 7630 7631 // Last of three parts 7632 lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) 7633 if err != nil { 7634 logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) 7635 } 7636 7637 // Complete the multipart upload 7638 _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) 7639 if err != nil { 7640 logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) 7641 } 7642 7643 // Stat the object and check its length matches 7644 objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) 7645 if err != nil { 7646 logError(testName, function, args, startTime, "", "StatObject call failed", err) 7647 } 7648 7649 if objInfo.Size != (5*1024*1024)*2+1 { 7650 logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) 7651 } 7652 7653 // Now we read the data back 7654 getOpts := minio.GetObjectOptions{} 7655 getOpts.SetRange(0, 5*1024*1024-1) 7656 r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) 7657 if err != nil { 7658 logError(testName, function, args, startTime, "", "GetObject call failed", err) 7659 } 7660 getBuf := make([]byte, 5*1024*1024) 7661 _, err = io.ReadFull(r, getBuf) 7662 if err != nil { 7663 logError(testName, function, args, startTime, "", "Read buffer failed", err) 7664 } 7665 if !bytes.Equal(getBuf, buf) { 7666 logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) 7667 } 7668 7669 getOpts.SetRange(5*1024*1024, 0) 7670 r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) 7671 if err != nil { 7672 logError(testName, function, args, startTime, "", "GetObject call failed", err) 7673 } 7674 getBuf = make([]byte, 5*1024*1024+1) 7675 _, err = io.ReadFull(r, getBuf) 7676 if err != nil { 7677 logError(testName, function, args, startTime, "", "Read buffer failed", err) 7678 } 7679 if !bytes.Equal(getBuf[:5*1024*1024], buf) { 7680 logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) 7681 } 7682 if getBuf[5*1024*1024] != buf[0] { 7683 logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) 7684 } 7685 7686 successLogger(testName, function, args, startTime).Info() 7687 7688 // Do not need to remove destBucketName its same as bucketName. 7689 } 7690 7691 // Test Core CopyObjectPart implementation for unencrypted to SSE-S3 encrypted copy 7692 func testUnencryptedToSSES3CopyObjectPart() { 7693 // initialize logging params 7694 startTime := time.Now() 7695 testName := getFuncName() 7696 function := "CopyObjectPart(destination, source)" 7697 args := map[string]interface{}{} 7698 7699 // Instantiate new minio client object 7700 client, err := minio.NewV4( 7701 os.Getenv(serverEndpoint), 7702 os.Getenv(accessKey), 7703 os.Getenv(secretKey), 7704 mustParseBool(os.Getenv(enableHTTPS)), 7705 ) 7706 if err != nil { 7707 logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) 7708 return 7709 } 7710 7711 // Instantiate new core client object. 7712 c := minio.Core{client} 7713 7714 // Enable tracing, write to stderr. 7715 // c.TraceOn(os.Stderr) 7716 7717 // Set user agent. 7718 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 7719 7720 // Generate a new random bucket name. 7721 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") 7722 7723 // Make a new bucket. 7724 err = c.MakeBucket(bucketName, "us-east-1") 7725 if err != nil { 7726 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 7727 } 7728 defer cleanupBucket(bucketName, client) 7729 // Make a buffer with 5MB of data 7730 buf := bytes.Repeat([]byte("abcde"), 1024*1024) 7731 7732 // Save the data 7733 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 7734 opts := minio.PutObjectOptions{ 7735 UserMetadata: map[string]string{ 7736 "Content-Type": "binary/octet-stream", 7737 }, 7738 } 7739 objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) 7740 if err != nil { 7741 logError(testName, function, args, startTime, "", "PutObject call failed", err) 7742 } 7743 7744 if objInfo.Size != int64(len(buf)) { 7745 logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) 7746 } 7747 7748 destBucketName := bucketName 7749 destObjectName := objectName + "-dest" 7750 dstencryption := encrypt.NewSSE() 7751 7752 uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) 7753 if err != nil { 7754 logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) 7755 } 7756 7757 // Content of the destination object will be two copies of 7758 // `objectName` concatenated, followed by first byte of 7759 // `objectName`. 7760 metadata := make(map[string]string) 7761 header := make(http.Header) 7762 dstencryption.Marshal(header) 7763 7764 for k, v := range header { 7765 metadata[k] = v[0] 7766 } 7767 7768 metadata["x-amz-copy-source-if-match"] = objInfo.ETag 7769 7770 // First of three parts 7771 fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) 7772 if err != nil { 7773 logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) 7774 } 7775 7776 // Second of three parts 7777 sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) 7778 if err != nil { 7779 logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) 7780 } 7781 7782 // Last of three parts 7783 lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) 7784 if err != nil { 7785 logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) 7786 } 7787 7788 // Complete the multipart upload 7789 _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) 7790 if err != nil { 7791 logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) 7792 } 7793 7794 // Stat the object and check its length matches 7795 objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) 7796 if err != nil { 7797 logError(testName, function, args, startTime, "", "StatObject call failed", err) 7798 } 7799 7800 if objInfo.Size != (5*1024*1024)*2+1 { 7801 logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) 7802 } 7803 7804 // Now we read the data back 7805 getOpts := minio.GetObjectOptions{} 7806 getOpts.SetRange(0, 5*1024*1024-1) 7807 r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) 7808 if err != nil { 7809 logError(testName, function, args, startTime, "", "GetObject call failed", err) 7810 } 7811 getBuf := make([]byte, 5*1024*1024) 7812 _, err = io.ReadFull(r, getBuf) 7813 if err != nil { 7814 logError(testName, function, args, startTime, "", "Read buffer failed", err) 7815 } 7816 if !bytes.Equal(getBuf, buf) { 7817 logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) 7818 } 7819 7820 getOpts.SetRange(5*1024*1024, 0) 7821 r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) 7822 if err != nil { 7823 logError(testName, function, args, startTime, "", "GetObject call failed", err) 7824 } 7825 getBuf = make([]byte, 5*1024*1024+1) 7826 _, err = io.ReadFull(r, getBuf) 7827 if err != nil { 7828 logError(testName, function, args, startTime, "", "Read buffer failed", err) 7829 } 7830 if !bytes.Equal(getBuf[:5*1024*1024], buf) { 7831 logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) 7832 } 7833 if getBuf[5*1024*1024] != buf[0] { 7834 logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) 7835 } 7836 7837 successLogger(testName, function, args, startTime).Info() 7838 7839 // Do not need to remove destBucketName its same as bucketName. 7840 } 7841 7842 // Test Core CopyObjectPart implementation for SSE-S3 to SSEC encryption copy part 7843 func testSSES3EncryptedToSSECCopyObjectPart() { 7844 // initialize logging params 7845 startTime := time.Now() 7846 testName := getFuncName() 7847 function := "CopyObjectPart(destination, source)" 7848 args := map[string]interface{}{} 7849 7850 // Instantiate new minio client object 7851 client, err := minio.NewV4( 7852 os.Getenv(serverEndpoint), 7853 os.Getenv(accessKey), 7854 os.Getenv(secretKey), 7855 mustParseBool(os.Getenv(enableHTTPS)), 7856 ) 7857 if err != nil { 7858 logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) 7859 return 7860 } 7861 7862 // Instantiate new core client object. 7863 c := minio.Core{client} 7864 7865 // Enable tracing, write to stderr. 7866 // c.TraceOn(os.Stderr) 7867 7868 // Set user agent. 7869 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 7870 7871 // Generate a new random bucket name. 7872 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") 7873 7874 // Make a new bucket. 7875 err = c.MakeBucket(bucketName, "us-east-1") 7876 if err != nil { 7877 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 7878 } 7879 defer cleanupBucket(bucketName, client) 7880 // Make a buffer with 5MB of data 7881 buf := bytes.Repeat([]byte("abcde"), 1024*1024) 7882 7883 // Save the data 7884 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 7885 password := "correct horse battery staple" 7886 srcEncryption := encrypt.NewSSE() 7887 opts := minio.PutObjectOptions{ 7888 UserMetadata: map[string]string{ 7889 "Content-Type": "binary/octet-stream", 7890 }, 7891 ServerSideEncryption: srcEncryption, 7892 } 7893 objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) 7894 if err != nil { 7895 logError(testName, function, args, startTime, "", "PutObject call failed", err) 7896 } 7897 7898 if objInfo.Size != int64(len(buf)) { 7899 logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) 7900 } 7901 7902 destBucketName := bucketName 7903 destObjectName := objectName + "-dest" 7904 dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName)) 7905 7906 uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) 7907 if err != nil { 7908 logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) 7909 } 7910 7911 // Content of the destination object will be two copies of 7912 // `objectName` concatenated, followed by first byte of 7913 // `objectName`. 7914 metadata := make(map[string]string) 7915 header := make(http.Header) 7916 dstencryption.Marshal(header) 7917 for k, v := range header { 7918 metadata[k] = v[0] 7919 } 7920 7921 metadata["x-amz-copy-source-if-match"] = objInfo.ETag 7922 7923 // First of three parts 7924 fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) 7925 if err != nil { 7926 logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) 7927 } 7928 7929 // Second of three parts 7930 sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) 7931 if err != nil { 7932 logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) 7933 } 7934 7935 // Last of three parts 7936 lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) 7937 if err != nil { 7938 logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) 7939 } 7940 7941 // Complete the multipart upload 7942 _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) 7943 if err != nil { 7944 logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) 7945 } 7946 7947 // Stat the object and check its length matches 7948 objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: dstencryption}}) 7949 if err != nil { 7950 logError(testName, function, args, startTime, "", "StatObject call failed", err) 7951 } 7952 7953 if objInfo.Size != (5*1024*1024)*2+1 { 7954 logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) 7955 } 7956 7957 // Now we read the data back 7958 getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption} 7959 getOpts.SetRange(0, 5*1024*1024-1) 7960 r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) 7961 if err != nil { 7962 logError(testName, function, args, startTime, "", "GetObject call failed", err) 7963 } 7964 getBuf := make([]byte, 5*1024*1024) 7965 _, err = io.ReadFull(r, getBuf) 7966 if err != nil { 7967 logError(testName, function, args, startTime, "", "Read buffer failed", err) 7968 } 7969 if !bytes.Equal(getBuf, buf) { 7970 logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) 7971 } 7972 7973 getOpts.SetRange(5*1024*1024, 0) 7974 r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) 7975 if err != nil { 7976 logError(testName, function, args, startTime, "", "GetObject call failed", err) 7977 } 7978 getBuf = make([]byte, 5*1024*1024+1) 7979 _, err = io.ReadFull(r, getBuf) 7980 if err != nil { 7981 logError(testName, function, args, startTime, "", "Read buffer failed", err) 7982 } 7983 if !bytes.Equal(getBuf[:5*1024*1024], buf) { 7984 logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) 7985 } 7986 if getBuf[5*1024*1024] != buf[0] { 7987 logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) 7988 } 7989 7990 successLogger(testName, function, args, startTime).Info() 7991 7992 // Do not need to remove destBucketName its same as bucketName. 7993 } 7994 7995 // Test Core CopyObjectPart implementation for unencrypted to unencrypted copy 7996 func testSSES3EncryptedToUnencryptedCopyPart() { 7997 // initialize logging params 7998 startTime := time.Now() 7999 testName := getFuncName() 8000 function := "CopyObjectPart(destination, source)" 8001 args := map[string]interface{}{} 8002 8003 // Instantiate new minio client object 8004 client, err := minio.NewV4( 8005 os.Getenv(serverEndpoint), 8006 os.Getenv(accessKey), 8007 os.Getenv(secretKey), 8008 mustParseBool(os.Getenv(enableHTTPS)), 8009 ) 8010 if err != nil { 8011 logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) 8012 return 8013 } 8014 8015 // Instantiate new core client object. 8016 c := minio.Core{client} 8017 8018 // Enable tracing, write to stderr. 8019 // c.TraceOn(os.Stderr) 8020 8021 // Set user agent. 8022 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 8023 8024 // Generate a new random bucket name. 8025 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") 8026 8027 // Make a new bucket. 8028 err = c.MakeBucket(bucketName, "us-east-1") 8029 if err != nil { 8030 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 8031 } 8032 defer cleanupBucket(bucketName, client) 8033 // Make a buffer with 5MB of data 8034 buf := bytes.Repeat([]byte("abcde"), 1024*1024) 8035 8036 // Save the data 8037 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 8038 srcEncryption := encrypt.NewSSE() 8039 opts := minio.PutObjectOptions{ 8040 UserMetadata: map[string]string{ 8041 "Content-Type": "binary/octet-stream", 8042 }, 8043 ServerSideEncryption: srcEncryption, 8044 } 8045 objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) 8046 if err != nil { 8047 logError(testName, function, args, startTime, "", "PutObject call failed", err) 8048 } 8049 8050 if objInfo.Size != int64(len(buf)) { 8051 logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) 8052 } 8053 8054 destBucketName := bucketName 8055 destObjectName := objectName + "-dest" 8056 8057 uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{}) 8058 if err != nil { 8059 logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) 8060 } 8061 8062 // Content of the destination object will be two copies of 8063 // `objectName` concatenated, followed by first byte of 8064 // `objectName`. 8065 metadata := make(map[string]string) 8066 header := make(http.Header) 8067 for k, v := range header { 8068 metadata[k] = v[0] 8069 } 8070 8071 metadata["x-amz-copy-source-if-match"] = objInfo.ETag 8072 8073 // First of three parts 8074 fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) 8075 if err != nil { 8076 logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) 8077 } 8078 8079 // Second of three parts 8080 sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) 8081 if err != nil { 8082 logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) 8083 } 8084 8085 // Last of three parts 8086 lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) 8087 if err != nil { 8088 logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) 8089 } 8090 8091 // Complete the multipart upload 8092 _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) 8093 if err != nil { 8094 logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) 8095 } 8096 8097 // Stat the object and check its length matches 8098 objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) 8099 if err != nil { 8100 logError(testName, function, args, startTime, "", "StatObject call failed", err) 8101 } 8102 8103 if objInfo.Size != (5*1024*1024)*2+1 { 8104 logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) 8105 } 8106 8107 // Now we read the data back 8108 getOpts := minio.GetObjectOptions{} 8109 getOpts.SetRange(0, 5*1024*1024-1) 8110 r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) 8111 if err != nil { 8112 logError(testName, function, args, startTime, "", "GetObject call failed", err) 8113 } 8114 getBuf := make([]byte, 5*1024*1024) 8115 _, err = io.ReadFull(r, getBuf) 8116 if err != nil { 8117 logError(testName, function, args, startTime, "", "Read buffer failed", err) 8118 } 8119 if !bytes.Equal(getBuf, buf) { 8120 logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) 8121 } 8122 8123 getOpts.SetRange(5*1024*1024, 0) 8124 r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) 8125 if err != nil { 8126 logError(testName, function, args, startTime, "", "GetObject call failed", err) 8127 } 8128 getBuf = make([]byte, 5*1024*1024+1) 8129 _, err = io.ReadFull(r, getBuf) 8130 if err != nil { 8131 logError(testName, function, args, startTime, "", "Read buffer failed", err) 8132 } 8133 if !bytes.Equal(getBuf[:5*1024*1024], buf) { 8134 logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) 8135 } 8136 if getBuf[5*1024*1024] != buf[0] { 8137 logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) 8138 } 8139 8140 successLogger(testName, function, args, startTime).Info() 8141 8142 // Do not need to remove destBucketName its same as bucketName. 8143 } 8144 8145 // Test Core CopyObjectPart implementation for unencrypted to SSE-S3 encrypted copy 8146 func testSSES3EncryptedToSSES3CopyObjectPart() { 8147 // initialize logging params 8148 startTime := time.Now() 8149 testName := getFuncName() 8150 function := "CopyObjectPart(destination, source)" 8151 args := map[string]interface{}{} 8152 8153 // Instantiate new minio client object 8154 client, err := minio.NewV4( 8155 os.Getenv(serverEndpoint), 8156 os.Getenv(accessKey), 8157 os.Getenv(secretKey), 8158 mustParseBool(os.Getenv(enableHTTPS)), 8159 ) 8160 if err != nil { 8161 logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) 8162 return 8163 } 8164 8165 // Instantiate new core client object. 8166 c := minio.Core{client} 8167 8168 // Enable tracing, write to stderr. 8169 // c.TraceOn(os.Stderr) 8170 8171 // Set user agent. 8172 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 8173 8174 // Generate a new random bucket name. 8175 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") 8176 8177 // Make a new bucket. 8178 err = c.MakeBucket(bucketName, "us-east-1") 8179 if err != nil { 8180 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 8181 } 8182 defer cleanupBucket(bucketName, client) 8183 // Make a buffer with 5MB of data 8184 buf := bytes.Repeat([]byte("abcde"), 1024*1024) 8185 8186 // Save the data 8187 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 8188 srcEncryption := encrypt.NewSSE() 8189 opts := minio.PutObjectOptions{ 8190 UserMetadata: map[string]string{ 8191 "Content-Type": "binary/octet-stream", 8192 }, 8193 ServerSideEncryption: srcEncryption, 8194 } 8195 8196 objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) 8197 if err != nil { 8198 logError(testName, function, args, startTime, "", "PutObject call failed", err) 8199 } 8200 8201 if objInfo.Size != int64(len(buf)) { 8202 logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) 8203 } 8204 8205 destBucketName := bucketName 8206 destObjectName := objectName + "-dest" 8207 dstencryption := encrypt.NewSSE() 8208 8209 uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) 8210 if err != nil { 8211 logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) 8212 } 8213 8214 // Content of the destination object will be two copies of 8215 // `objectName` concatenated, followed by first byte of 8216 // `objectName`. 8217 metadata := make(map[string]string) 8218 header := make(http.Header) 8219 dstencryption.Marshal(header) 8220 8221 for k, v := range header { 8222 metadata[k] = v[0] 8223 } 8224 8225 metadata["x-amz-copy-source-if-match"] = objInfo.ETag 8226 8227 // First of three parts 8228 fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) 8229 if err != nil { 8230 logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) 8231 } 8232 8233 // Second of three parts 8234 sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) 8235 if err != nil { 8236 logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) 8237 } 8238 8239 // Last of three parts 8240 lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) 8241 if err != nil { 8242 logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) 8243 } 8244 8245 // Complete the multipart upload 8246 _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) 8247 if err != nil { 8248 logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) 8249 } 8250 8251 // Stat the object and check its length matches 8252 objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) 8253 if err != nil { 8254 logError(testName, function, args, startTime, "", "StatObject call failed", err) 8255 } 8256 8257 if objInfo.Size != (5*1024*1024)*2+1 { 8258 logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) 8259 } 8260 8261 // Now we read the data back 8262 getOpts := minio.GetObjectOptions{} 8263 getOpts.SetRange(0, 5*1024*1024-1) 8264 r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) 8265 if err != nil { 8266 logError(testName, function, args, startTime, "", "GetObject call failed", err) 8267 } 8268 getBuf := make([]byte, 5*1024*1024) 8269 _, err = io.ReadFull(r, getBuf) 8270 if err != nil { 8271 logError(testName, function, args, startTime, "", "Read buffer failed", err) 8272 } 8273 if !bytes.Equal(getBuf, buf) { 8274 logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) 8275 } 8276 8277 getOpts.SetRange(5*1024*1024, 0) 8278 r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) 8279 if err != nil { 8280 logError(testName, function, args, startTime, "", "GetObject call failed", err) 8281 } 8282 getBuf = make([]byte, 5*1024*1024+1) 8283 _, err = io.ReadFull(r, getBuf) 8284 if err != nil { 8285 logError(testName, function, args, startTime, "", "Read buffer failed", err) 8286 } 8287 if !bytes.Equal(getBuf[:5*1024*1024], buf) { 8288 logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) 8289 } 8290 if getBuf[5*1024*1024] != buf[0] { 8291 logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) 8292 } 8293 8294 successLogger(testName, function, args, startTime).Info() 8295 8296 // Do not need to remove destBucketName its same as bucketName. 8297 } 8298 func testUserMetadataCopying() { 8299 // initialize logging params 8300 startTime := time.Now() 8301 testName := getFuncName() 8302 function := "CopyObject(destination, source)" 8303 args := map[string]interface{}{} 8304 8305 // Instantiate new minio client object 8306 c, err := minio.NewV4( 8307 os.Getenv(serverEndpoint), 8308 os.Getenv(accessKey), 8309 os.Getenv(secretKey), 8310 mustParseBool(os.Getenv(enableHTTPS)), 8311 ) 8312 if err != nil { 8313 logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) 8314 return 8315 } 8316 8317 // c.TraceOn(os.Stderr) 8318 testUserMetadataCopyingWrapper(c) 8319 } 8320 8321 func testUserMetadataCopyingWrapper(c *minio.Client) { 8322 // initialize logging params 8323 startTime := time.Now() 8324 testName := getFuncName() 8325 function := "CopyObject(destination, source)" 8326 args := map[string]interface{}{} 8327 8328 // Generate a new random bucket name. 8329 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 8330 // Make a new bucket in 'us-east-1' (source bucket). 8331 err := c.MakeBucket(bucketName, "us-east-1") 8332 if err != nil { 8333 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 8334 return 8335 } 8336 8337 fetchMeta := func(object string) (h http.Header) { 8338 objInfo, err := c.StatObject(bucketName, object, minio.StatObjectOptions{}) 8339 if err != nil { 8340 logError(testName, function, args, startTime, "", "Stat failed", err) 8341 return 8342 } 8343 h = make(http.Header) 8344 for k, vs := range objInfo.Metadata { 8345 if strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") { 8346 for _, v := range vs { 8347 h.Add(k, v) 8348 } 8349 } 8350 } 8351 return h 8352 } 8353 8354 // 1. create a client encrypted object to copy by uploading 8355 const srcSize = 1024 * 1024 8356 buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 5MiB 8357 metadata := make(http.Header) 8358 metadata.Set("x-amz-meta-myheader", "myvalue") 8359 m := make(map[string]string) 8360 m["x-amz-meta-myheader"] = "myvalue" 8361 _, err = c.PutObject(bucketName, "srcObject", 8362 bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{UserMetadata: m}) 8363 if err != nil { 8364 logError(testName, function, args, startTime, "", "PutObjectWithMetadata failed", err) 8365 return 8366 } 8367 if !reflect.DeepEqual(metadata, fetchMeta("srcObject")) { 8368 logError(testName, function, args, startTime, "", "Metadata match failed", err) 8369 return 8370 } 8371 8372 // 2. create source 8373 src := minio.NewSourceInfo(bucketName, "srcObject", nil) 8374 // 2.1 create destination with metadata set 8375 dst1, err := minio.NewDestinationInfo(bucketName, "dstObject-1", nil, map[string]string{"notmyheader": "notmyvalue"}) 8376 if err != nil { 8377 logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) 8378 return 8379 } 8380 8381 // 3. Check that copying to an object with metadata set resets 8382 // the headers on the copy. 8383 args["source"] = src 8384 args["destination"] = dst1 8385 err = c.CopyObject(dst1, src) 8386 if err != nil { 8387 logError(testName, function, args, startTime, "", "CopyObject failed", err) 8388 return 8389 } 8390 8391 expectedHeaders := make(http.Header) 8392 expectedHeaders.Set("x-amz-meta-notmyheader", "notmyvalue") 8393 if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-1")) { 8394 logError(testName, function, args, startTime, "", "Metadata match failed", err) 8395 return 8396 } 8397 8398 // 4. create destination with no metadata set and same source 8399 dst2, err := minio.NewDestinationInfo(bucketName, "dstObject-2", nil, nil) 8400 if err != nil { 8401 logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) 8402 return 8403 } 8404 src = minio.NewSourceInfo(bucketName, "srcObject", nil) 8405 8406 // 5. Check that copying to an object with no metadata set, 8407 // copies metadata. 8408 args["source"] = src 8409 args["destination"] = dst2 8410 err = c.CopyObject(dst2, src) 8411 if err != nil { 8412 logError(testName, function, args, startTime, "", "CopyObject failed", err) 8413 return 8414 } 8415 8416 expectedHeaders = metadata 8417 if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-2")) { 8418 logError(testName, function, args, startTime, "", "Metadata match failed", err) 8419 return 8420 } 8421 8422 // 6. Compose a pair of sources. 8423 srcs := []minio.SourceInfo{ 8424 minio.NewSourceInfo(bucketName, "srcObject", nil), 8425 minio.NewSourceInfo(bucketName, "srcObject", nil), 8426 } 8427 dst3, err := minio.NewDestinationInfo(bucketName, "dstObject-3", nil, nil) 8428 if err != nil { 8429 logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) 8430 return 8431 } 8432 8433 function = "ComposeObject(destination, sources)" 8434 args["source"] = srcs 8435 args["destination"] = dst3 8436 err = c.ComposeObject(dst3, srcs) 8437 if err != nil { 8438 logError(testName, function, args, startTime, "", "ComposeObject failed", err) 8439 return 8440 } 8441 8442 // Check that no headers are copied in this case 8443 if !reflect.DeepEqual(make(http.Header), fetchMeta("dstObject-3")) { 8444 logError(testName, function, args, startTime, "", "Metadata match failed", err) 8445 return 8446 } 8447 8448 // 7. Compose a pair of sources with dest user metadata set. 8449 srcs = []minio.SourceInfo{ 8450 minio.NewSourceInfo(bucketName, "srcObject", nil), 8451 minio.NewSourceInfo(bucketName, "srcObject", nil), 8452 } 8453 dst4, err := minio.NewDestinationInfo(bucketName, "dstObject-4", nil, map[string]string{"notmyheader": "notmyvalue"}) 8454 if err != nil { 8455 logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) 8456 return 8457 } 8458 8459 function = "ComposeObject(destination, sources)" 8460 args["source"] = srcs 8461 args["destination"] = dst4 8462 err = c.ComposeObject(dst4, srcs) 8463 if err != nil { 8464 logError(testName, function, args, startTime, "", "ComposeObject failed", err) 8465 return 8466 } 8467 8468 // Check that no headers are copied in this case 8469 expectedHeaders = make(http.Header) 8470 expectedHeaders.Set("x-amz-meta-notmyheader", "notmyvalue") 8471 if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-4")) { 8472 logError(testName, function, args, startTime, "", "Metadata match failed", err) 8473 return 8474 } 8475 8476 // Delete all objects and buckets 8477 if err = cleanupBucket(bucketName, c); err != nil { 8478 logError(testName, function, args, startTime, "", "Cleanup failed", err) 8479 return 8480 } 8481 8482 successLogger(testName, function, args, startTime).Info() 8483 } 8484 8485 func testUserMetadataCopyingV2() { 8486 // initialize logging params 8487 startTime := time.Now() 8488 testName := getFuncName() 8489 function := "CopyObject(destination, source)" 8490 args := map[string]interface{}{} 8491 8492 // Instantiate new minio client object 8493 c, err := minio.NewV2( 8494 os.Getenv(serverEndpoint), 8495 os.Getenv(accessKey), 8496 os.Getenv(secretKey), 8497 mustParseBool(os.Getenv(enableHTTPS)), 8498 ) 8499 if err != nil { 8500 logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) 8501 return 8502 } 8503 8504 // c.TraceOn(os.Stderr) 8505 testUserMetadataCopyingWrapper(c) 8506 } 8507 8508 func testStorageClassMetadataPutObject() { 8509 // initialize logging params 8510 startTime := time.Now() 8511 function := "testStorageClassMetadataPutObject()" 8512 args := map[string]interface{}{} 8513 testName := getFuncName() 8514 8515 // Instantiate new minio client object 8516 c, err := minio.NewV4( 8517 os.Getenv(serverEndpoint), 8518 os.Getenv(accessKey), 8519 os.Getenv(secretKey), 8520 mustParseBool(os.Getenv(enableHTTPS)), 8521 ) 8522 if err != nil { 8523 logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) 8524 return 8525 } 8526 8527 // Generate a new random bucket name. 8528 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") 8529 // Make a new bucket in 'us-east-1' (source bucket). 8530 err = c.MakeBucket(bucketName, "us-east-1") 8531 if err != nil { 8532 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 8533 return 8534 } 8535 8536 fetchMeta := func(object string) (h http.Header) { 8537 objInfo, err := c.StatObject(bucketName, object, minio.StatObjectOptions{}) 8538 if err != nil { 8539 logError(testName, function, args, startTime, "", "Stat failed", err) 8540 return 8541 } 8542 h = make(http.Header) 8543 for k, vs := range objInfo.Metadata { 8544 if strings.HasPrefix(strings.ToLower(k), "x-amz-storage-class") { 8545 for _, v := range vs { 8546 h.Add(k, v) 8547 } 8548 } 8549 } 8550 return h 8551 } 8552 8553 metadata := make(http.Header) 8554 metadata.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") 8555 8556 emptyMetadata := make(http.Header) 8557 8558 const srcSize = 1024 * 1024 8559 buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 1MiB 8560 8561 _, err = c.PutObject(bucketName, "srcObjectRRSClass", 8562 bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "REDUCED_REDUNDANCY"}) 8563 if err != nil { 8564 logError(testName, function, args, startTime, "", "PutObject failed", err) 8565 return 8566 } 8567 8568 // Get the returned metadata 8569 returnedMeta := fetchMeta("srcObjectRRSClass") 8570 8571 // The response metada should either be equal to metadata (with REDUCED_REDUNDANCY) or emptyMetadata (in case of gateways) 8572 if !reflect.DeepEqual(metadata, returnedMeta) && !reflect.DeepEqual(emptyMetadata, returnedMeta) { 8573 logError(testName, function, args, startTime, "", "Metadata match failed", err) 8574 return 8575 } 8576 8577 metadata = make(http.Header) 8578 metadata.Set("x-amz-storage-class", "STANDARD") 8579 8580 _, err = c.PutObject(bucketName, "srcObjectSSClass", 8581 bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "STANDARD"}) 8582 if err != nil { 8583 logError(testName, function, args, startTime, "", "PutObject failed", err) 8584 return 8585 } 8586 if reflect.DeepEqual(metadata, fetchMeta("srcObjectSSClass")) { 8587 logError(testName, function, args, startTime, "", "Metadata verification failed, STANDARD storage class should not be a part of response metadata", err) 8588 return 8589 } 8590 // Delete all objects and buckets 8591 if err = cleanupBucket(bucketName, c); err != nil { 8592 logError(testName, function, args, startTime, "", "Cleanup failed", err) 8593 return 8594 } 8595 successLogger(testName, function, args, startTime).Info() 8596 } 8597 8598 func testStorageClassInvalidMetadataPutObject() { 8599 // initialize logging params 8600 startTime := time.Now() 8601 function := "testStorageClassInvalidMetadataPutObject()" 8602 args := map[string]interface{}{} 8603 testName := getFuncName() 8604 8605 // Instantiate new minio client object 8606 c, err := minio.NewV4( 8607 os.Getenv(serverEndpoint), 8608 os.Getenv(accessKey), 8609 os.Getenv(secretKey), 8610 mustParseBool(os.Getenv(enableHTTPS)), 8611 ) 8612 if err != nil { 8613 logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) 8614 return 8615 } 8616 8617 // Generate a new random bucket name. 8618 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") 8619 // Make a new bucket in 'us-east-1' (source bucket). 8620 err = c.MakeBucket(bucketName, "us-east-1") 8621 if err != nil { 8622 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 8623 return 8624 } 8625 8626 const srcSize = 1024 * 1024 8627 buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 1MiB 8628 8629 _, err = c.PutObject(bucketName, "srcObjectRRSClass", 8630 bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "INVALID_STORAGE_CLASS"}) 8631 if err == nil { 8632 logError(testName, function, args, startTime, "", "PutObject with invalid storage class passed, was expected to fail", err) 8633 return 8634 } 8635 // Delete all objects and buckets 8636 if err = cleanupBucket(bucketName, c); err != nil { 8637 logError(testName, function, args, startTime, "", "Cleanup failed", err) 8638 return 8639 } 8640 successLogger(testName, function, args, startTime).Info() 8641 } 8642 8643 func testStorageClassMetadataCopyObject() { 8644 // initialize logging params 8645 startTime := time.Now() 8646 function := "testStorageClassMetadataCopyObject()" 8647 args := map[string]interface{}{} 8648 testName := getFuncName() 8649 8650 // Instantiate new minio client object 8651 c, err := minio.NewV4( 8652 os.Getenv(serverEndpoint), 8653 os.Getenv(accessKey), 8654 os.Getenv(secretKey), 8655 mustParseBool(os.Getenv(enableHTTPS)), 8656 ) 8657 if err != nil { 8658 logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) 8659 return 8660 } 8661 8662 // Generate a new random bucket name. 8663 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") 8664 // Make a new bucket in 'us-east-1' (source bucket). 8665 err = c.MakeBucket(bucketName, "us-east-1") 8666 if err != nil { 8667 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 8668 return 8669 } 8670 8671 fetchMeta := func(object string) (h http.Header) { 8672 objInfo, err := c.StatObject(bucketName, object, minio.StatObjectOptions{}) 8673 args["bucket"] = bucketName 8674 args["object"] = object 8675 if err != nil { 8676 logError(testName, function, args, startTime, "", "Stat failed", err) 8677 return 8678 } 8679 h = make(http.Header) 8680 for k, vs := range objInfo.Metadata { 8681 if strings.HasPrefix(strings.ToLower(k), "x-amz-storage-class") { 8682 for _, v := range vs { 8683 h.Add(k, v) 8684 } 8685 } 8686 } 8687 return h 8688 } 8689 8690 metadata := make(http.Header) 8691 metadata.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") 8692 8693 emptyMetadata := make(http.Header) 8694 8695 const srcSize = 1024 * 1024 8696 buf := bytes.Repeat([]byte("abcde"), srcSize) 8697 8698 // Put an object with RRS Storage class 8699 _, err = c.PutObject(bucketName, "srcObjectRRSClass", 8700 bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "REDUCED_REDUNDANCY"}) 8701 if err != nil { 8702 logError(testName, function, args, startTime, "", "PutObject failed", err) 8703 return 8704 } 8705 8706 // Make server side copy of object uploaded in previous step 8707 src := minio.NewSourceInfo(bucketName, "srcObjectRRSClass", nil) 8708 dst, err := minio.NewDestinationInfo(bucketName, "srcObjectRRSClassCopy", nil, nil) 8709 if err = c.CopyObject(dst, src); err != nil { 8710 logError(testName, function, args, startTime, "", "CopyObject failed on RRS", err) 8711 } 8712 8713 // Get the returned metadata 8714 returnedMeta := fetchMeta("srcObjectRRSClassCopy") 8715 8716 // The response metada should either be equal to metadata (with REDUCED_REDUNDANCY) or emptyMetadata (in case of gateways) 8717 if !reflect.DeepEqual(metadata, returnedMeta) && !reflect.DeepEqual(emptyMetadata, returnedMeta) { 8718 logError(testName, function, args, startTime, "", "Metadata match failed", err) 8719 return 8720 } 8721 8722 metadata = make(http.Header) 8723 metadata.Set("x-amz-storage-class", "STANDARD") 8724 8725 // Put an object with Standard Storage class 8726 _, err = c.PutObject(bucketName, "srcObjectSSClass", 8727 bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "STANDARD"}) 8728 if err != nil { 8729 logError(testName, function, args, startTime, "", "PutObject failed", err) 8730 return 8731 } 8732 8733 // Make server side copy of object uploaded in previous step 8734 src = minio.NewSourceInfo(bucketName, "srcObjectSSClass", nil) 8735 dst, err = minio.NewDestinationInfo(bucketName, "srcObjectSSClassCopy", nil, nil) 8736 if err = c.CopyObject(dst, src); err != nil { 8737 logError(testName, function, args, startTime, "", "CopyObject failed on SS", err) 8738 } 8739 // Fetch the meta data of copied object 8740 if reflect.DeepEqual(metadata, fetchMeta("srcObjectSSClassCopy")) { 8741 logError(testName, function, args, startTime, "", "Metadata verification failed, STANDARD storage class should not be a part of response metadata", err) 8742 return 8743 } 8744 // Delete all objects and buckets 8745 if err = cleanupBucket(bucketName, c); err != nil { 8746 logError(testName, function, args, startTime, "", "Cleanup failed", err) 8747 return 8748 } 8749 successLogger(testName, function, args, startTime).Info() 8750 } 8751 8752 // Test put object with size -1 byte object. 8753 func testPutObjectNoLengthV2() { 8754 // initialize logging params 8755 startTime := time.Now() 8756 testName := getFuncName() 8757 function := "PutObject(bucketName, objectName, reader, size, opts)" 8758 args := map[string]interface{}{ 8759 "bucketName": "", 8760 "objectName": "", 8761 "size": -1, 8762 "opts": "", 8763 } 8764 8765 // Seed random based on current time. 8766 rand.Seed(time.Now().Unix()) 8767 8768 // Instantiate new minio client object. 8769 c, err := minio.NewV2( 8770 os.Getenv(serverEndpoint), 8771 os.Getenv(accessKey), 8772 os.Getenv(secretKey), 8773 mustParseBool(os.Getenv(enableHTTPS)), 8774 ) 8775 if err != nil { 8776 logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) 8777 return 8778 } 8779 8780 // Enable tracing, write to stderr. 8781 // c.TraceOn(os.Stderr) 8782 8783 // Set user agent. 8784 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 8785 8786 // Generate a new random bucket name. 8787 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 8788 args["bucketName"] = bucketName 8789 8790 // Make a new bucket. 8791 err = c.MakeBucket(bucketName, "us-east-1") 8792 if err != nil { 8793 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 8794 return 8795 } 8796 8797 objectName := bucketName + "unique" 8798 args["objectName"] = objectName 8799 8800 bufSize := dataFileMap["datafile-129-MB"] 8801 var reader = getDataReader("datafile-129-MB") 8802 defer reader.Close() 8803 args["size"] = bufSize 8804 8805 // Upload an object. 8806 n, err := c.PutObject(bucketName, objectName, reader, -1, minio.PutObjectOptions{}) 8807 8808 if err != nil { 8809 logError(testName, function, args, startTime, "", "PutObjectWithSize failed", err) 8810 return 8811 } 8812 if n != int64(bufSize) { 8813 logError(testName, function, args, startTime, "", "Expected upload object size "+string(bufSize)+" got "+string(n), err) 8814 return 8815 } 8816 8817 // Delete all objects and buckets 8818 if err = cleanupBucket(bucketName, c); err != nil { 8819 logError(testName, function, args, startTime, "", "Cleanup failed", err) 8820 return 8821 } 8822 8823 successLogger(testName, function, args, startTime).Info() 8824 } 8825 8826 // Test put objects of unknown size. 8827 func testPutObjectsUnknownV2() { 8828 // initialize logging params 8829 startTime := time.Now() 8830 testName := getFuncName() 8831 function := "PutObject(bucketName, objectName, reader,size,opts)" 8832 args := map[string]interface{}{ 8833 "bucketName": "", 8834 "objectName": "", 8835 "size": "", 8836 "opts": "", 8837 } 8838 8839 // Seed random based on current time. 8840 rand.Seed(time.Now().Unix()) 8841 8842 // Instantiate new minio client object. 8843 c, err := minio.NewV2( 8844 os.Getenv(serverEndpoint), 8845 os.Getenv(accessKey), 8846 os.Getenv(secretKey), 8847 mustParseBool(os.Getenv(enableHTTPS)), 8848 ) 8849 if err != nil { 8850 logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) 8851 return 8852 } 8853 8854 // Enable tracing, write to stderr. 8855 // c.TraceOn(os.Stderr) 8856 8857 // Set user agent. 8858 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 8859 8860 // Generate a new random bucket name. 8861 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 8862 args["bucketName"] = bucketName 8863 8864 // Make a new bucket. 8865 err = c.MakeBucket(bucketName, "us-east-1") 8866 if err != nil { 8867 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 8868 return 8869 } 8870 8871 // Issues are revealed by trying to upload multiple files of unknown size 8872 // sequentially (on 4GB machines) 8873 for i := 1; i <= 4; i++ { 8874 // Simulate that we could be receiving byte slices of data that we want 8875 // to upload as a file 8876 rpipe, wpipe := io.Pipe() 8877 defer rpipe.Close() 8878 go func() { 8879 b := []byte("test") 8880 wpipe.Write(b) 8881 wpipe.Close() 8882 }() 8883 8884 // Upload the object. 8885 objectName := fmt.Sprintf("%sunique%d", bucketName, i) 8886 args["objectName"] = objectName 8887 8888 n, err := c.PutObject(bucketName, objectName, rpipe, -1, minio.PutObjectOptions{}) 8889 if err != nil { 8890 logError(testName, function, args, startTime, "", "PutObjectStreaming failed", err) 8891 return 8892 } 8893 args["size"] = n 8894 if n != int64(4) { 8895 logError(testName, function, args, startTime, "", "Expected upload object size "+string(4)+" got "+string(n), err) 8896 return 8897 } 8898 8899 } 8900 8901 // Delete all objects and buckets 8902 if err = cleanupBucket(bucketName, c); err != nil { 8903 logError(testName, function, args, startTime, "", "Cleanup failed", err) 8904 return 8905 } 8906 8907 successLogger(testName, function, args, startTime).Info() 8908 } 8909 8910 // Test put object with 0 byte object. 8911 func testPutObject0ByteV2() { 8912 // initialize logging params 8913 startTime := time.Now() 8914 testName := getFuncName() 8915 function := "PutObject(bucketName, objectName, reader, size, opts)" 8916 args := map[string]interface{}{ 8917 "bucketName": "", 8918 "objectName": "", 8919 "size": 0, 8920 "opts": "", 8921 } 8922 8923 // Seed random based on current time. 8924 rand.Seed(time.Now().Unix()) 8925 8926 // Instantiate new minio client object. 8927 c, err := minio.NewV2( 8928 os.Getenv(serverEndpoint), 8929 os.Getenv(accessKey), 8930 os.Getenv(secretKey), 8931 mustParseBool(os.Getenv(enableHTTPS)), 8932 ) 8933 if err != nil { 8934 logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) 8935 return 8936 } 8937 8938 // Enable tracing, write to stderr. 8939 // c.TraceOn(os.Stderr) 8940 8941 // Set user agent. 8942 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 8943 8944 // Generate a new random bucket name. 8945 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 8946 args["bucketName"] = bucketName 8947 8948 // Make a new bucket. 8949 err = c.MakeBucket(bucketName, "us-east-1") 8950 if err != nil { 8951 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 8952 return 8953 } 8954 8955 objectName := bucketName + "unique" 8956 args["objectName"] = objectName 8957 args["opts"] = minio.PutObjectOptions{} 8958 8959 // Upload an object. 8960 n, err := c.PutObject(bucketName, objectName, bytes.NewReader([]byte("")), 0, minio.PutObjectOptions{}) 8961 8962 if err != nil { 8963 logError(testName, function, args, startTime, "", "PutObjectWithSize failed", err) 8964 return 8965 } 8966 if n != 0 { 8967 logError(testName, function, args, startTime, "", "Expected upload object size 0 but got "+string(n), err) 8968 return 8969 } 8970 8971 // Delete all objects and buckets 8972 if err = cleanupBucket(bucketName, c); err != nil { 8973 logError(testName, function, args, startTime, "", "Cleanup failed", err) 8974 return 8975 } 8976 8977 successLogger(testName, function, args, startTime).Info() 8978 } 8979 8980 // Test expected error cases 8981 func testComposeObjectErrorCases() { 8982 // initialize logging params 8983 startTime := time.Now() 8984 testName := getFuncName() 8985 function := "ComposeObject(destination, sourceList)" 8986 args := map[string]interface{}{} 8987 8988 // Instantiate new minio client object 8989 c, err := minio.NewV4( 8990 os.Getenv(serverEndpoint), 8991 os.Getenv(accessKey), 8992 os.Getenv(secretKey), 8993 mustParseBool(os.Getenv(enableHTTPS)), 8994 ) 8995 if err != nil { 8996 logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) 8997 return 8998 } 8999 9000 testComposeObjectErrorCasesWrapper(c) 9001 } 9002 9003 // Test concatenating 10K objects 9004 func testCompose10KSources() { 9005 // initialize logging params 9006 startTime := time.Now() 9007 testName := getFuncName() 9008 function := "ComposeObject(destination, sourceList)" 9009 args := map[string]interface{}{} 9010 9011 // Instantiate new minio client object 9012 c, err := minio.NewV4( 9013 os.Getenv(serverEndpoint), 9014 os.Getenv(accessKey), 9015 os.Getenv(secretKey), 9016 mustParseBool(os.Getenv(enableHTTPS)), 9017 ) 9018 if err != nil { 9019 logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) 9020 return 9021 } 9022 9023 testComposeMultipleSources(c) 9024 } 9025 9026 // Tests comprehensive list of all methods. 9027 func testFunctionalV2() { 9028 // initialize logging params 9029 startTime := time.Now() 9030 testName := getFuncName() 9031 function := "testFunctionalV2()" 9032 functionAll := "" 9033 args := map[string]interface{}{} 9034 9035 // Seed random based on current time. 9036 rand.Seed(time.Now().Unix()) 9037 9038 c, err := minio.NewV2( 9039 os.Getenv(serverEndpoint), 9040 os.Getenv(accessKey), 9041 os.Getenv(secretKey), 9042 mustParseBool(os.Getenv(enableHTTPS)), 9043 ) 9044 if err != nil { 9045 logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) 9046 return 9047 } 9048 9049 // Enable to debug 9050 // c.TraceOn(os.Stderr) 9051 9052 // Set user agent. 9053 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 9054 9055 // Generate a new random bucket name. 9056 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 9057 location := "us-east-1" 9058 // Make a new bucket. 9059 function = "MakeBucket(bucketName, location)" 9060 functionAll = "MakeBucket(bucketName, location)" 9061 args = map[string]interface{}{ 9062 "bucketName": bucketName, 9063 "location": location, 9064 } 9065 err = c.MakeBucket(bucketName, location) 9066 if err != nil { 9067 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 9068 return 9069 } 9070 9071 // Generate a random file name. 9072 fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 9073 file, err := os.Create(fileName) 9074 if err != nil { 9075 logError(testName, function, args, startTime, "", "file create failed", err) 9076 return 9077 } 9078 for i := 0; i < 3; i++ { 9079 buf := make([]byte, rand.Intn(1<<19)) 9080 _, err = file.Write(buf) 9081 if err != nil { 9082 logError(testName, function, args, startTime, "", "file write failed", err) 9083 return 9084 } 9085 } 9086 file.Close() 9087 9088 // Verify if bucket exits and you have access. 9089 var exists bool 9090 function = "BucketExists(bucketName)" 9091 functionAll += ", " + function 9092 args = map[string]interface{}{ 9093 "bucketName": bucketName, 9094 } 9095 exists, err = c.BucketExists(bucketName) 9096 if err != nil { 9097 logError(testName, function, args, startTime, "", "BucketExists failed", err) 9098 return 9099 } 9100 if !exists { 9101 logError(testName, function, args, startTime, "", "Could not find existing bucket "+bucketName, err) 9102 return 9103 } 9104 9105 // Make the bucket 'public read/write'. 9106 function = "SetBucketPolicy(bucketName, bucketPolicy)" 9107 functionAll += ", " + function 9108 9109 readWritePolicy := `{"Version": "2012-10-17","Statement": [{"Action": ["s3:ListBucketMultipartUploads", "s3:ListBucket"],"Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::` + bucketName + `"],"Sid": ""}]}` 9110 9111 args = map[string]interface{}{ 9112 "bucketName": bucketName, 9113 "bucketPolicy": readWritePolicy, 9114 } 9115 err = c.SetBucketPolicy(bucketName, readWritePolicy) 9116 9117 if err != nil { 9118 logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) 9119 return 9120 } 9121 9122 // List all buckets. 9123 function = "ListBuckets()" 9124 functionAll += ", " + function 9125 args = nil 9126 buckets, err := c.ListBuckets() 9127 if len(buckets) == 0 { 9128 logError(testName, function, args, startTime, "", "List buckets cannot be empty", err) 9129 return 9130 } 9131 if err != nil { 9132 logError(testName, function, args, startTime, "", "ListBuckets failed", err) 9133 return 9134 } 9135 9136 // Verify if previously created bucket is listed in list buckets. 9137 bucketFound := false 9138 for _, bucket := range buckets { 9139 if bucket.Name == bucketName { 9140 bucketFound = true 9141 } 9142 } 9143 9144 // If bucket not found error out. 9145 if !bucketFound { 9146 logError(testName, function, args, startTime, "", "Bucket "+bucketName+"not found", err) 9147 return 9148 } 9149 9150 objectName := bucketName + "unique" 9151 9152 // Generate data 9153 buf := bytes.Repeat([]byte("n"), rand.Intn(1<<19)) 9154 9155 args = map[string]interface{}{ 9156 "bucketName": bucketName, 9157 "objectName": objectName, 9158 "contentType": "", 9159 } 9160 n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) 9161 if err != nil { 9162 logError(testName, function, args, startTime, "", "PutObject failed", err) 9163 return 9164 } 9165 if n != int64(len(buf)) { 9166 logError(testName, function, args, startTime, "", "Expected uploaded object length "+string(len(buf))+" got "+string(n), err) 9167 return 9168 } 9169 9170 objectNameNoLength := objectName + "-nolength" 9171 args["objectName"] = objectNameNoLength 9172 n, err = c.PutObject(bucketName, objectNameNoLength, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) 9173 if err != nil { 9174 logError(testName, function, args, startTime, "", "PutObject failed", err) 9175 return 9176 } 9177 9178 if n != int64(len(buf)) { 9179 logError(testName, function, args, startTime, "", "Expected uploaded object length "+string(len(buf))+" got "+string(n), err) 9180 return 9181 } 9182 9183 // Instantiate a done channel to close all listing. 9184 doneCh := make(chan struct{}) 9185 defer close(doneCh) 9186 9187 objFound := false 9188 isRecursive := true // Recursive is true. 9189 function = "ListObjects(bucketName, objectName, isRecursive, doneCh)" 9190 functionAll += ", " + function 9191 args = map[string]interface{}{ 9192 "bucketName": bucketName, 9193 "objectName": objectName, 9194 "isRecursive": isRecursive, 9195 } 9196 for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) { 9197 if obj.Key == objectName { 9198 objFound = true 9199 break 9200 } 9201 } 9202 if !objFound { 9203 logError(testName, function, args, startTime, "", "Could not find existing object "+objectName, err) 9204 return 9205 } 9206 9207 incompObjNotFound := true 9208 function = "ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh)" 9209 functionAll += ", " + function 9210 args = map[string]interface{}{ 9211 "bucketName": bucketName, 9212 "objectName": objectName, 9213 "isRecursive": isRecursive, 9214 } 9215 for objIncompl := range c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) { 9216 if objIncompl.Key != "" { 9217 incompObjNotFound = false 9218 break 9219 } 9220 } 9221 if !incompObjNotFound { 9222 logError(testName, function, args, startTime, "", "Unexpected dangling incomplete upload found", err) 9223 return 9224 } 9225 9226 function = "GetObject(bucketName, objectName)" 9227 functionAll += ", " + function 9228 args = map[string]interface{}{ 9229 "bucketName": bucketName, 9230 "objectName": objectName, 9231 } 9232 newReader, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) 9233 if err != nil { 9234 logError(testName, function, args, startTime, "", "GetObject failed", err) 9235 return 9236 } 9237 9238 newReadBytes, err := ioutil.ReadAll(newReader) 9239 if err != nil { 9240 logError(testName, function, args, startTime, "", "ReadAll failed", err) 9241 return 9242 } 9243 newReader.Close() 9244 9245 if !bytes.Equal(newReadBytes, buf) { 9246 logError(testName, function, args, startTime, "", "Bytes mismatch", err) 9247 return 9248 } 9249 9250 function = "FGetObject(bucketName, objectName, fileName)" 9251 functionAll += ", " + function 9252 args = map[string]interface{}{ 9253 "bucketName": bucketName, 9254 "objectName": objectName, 9255 "fileName": fileName + "-f", 9256 } 9257 err = c.FGetObject(bucketName, objectName, fileName+"-f", minio.GetObjectOptions{}) 9258 if err != nil { 9259 logError(testName, function, args, startTime, "", "FgetObject failed", err) 9260 return 9261 } 9262 9263 // Generate presigned HEAD object url. 9264 function = "PresignedHeadObject(bucketName, objectName, expires, reqParams)" 9265 functionAll += ", " + function 9266 args = map[string]interface{}{ 9267 "bucketName": bucketName, 9268 "objectName": objectName, 9269 "expires": 3600 * time.Second, 9270 } 9271 presignedHeadURL, err := c.PresignedHeadObject(bucketName, objectName, 3600*time.Second, nil) 9272 if err != nil { 9273 logError(testName, function, args, startTime, "", "PresignedHeadObject failed", err) 9274 return 9275 } 9276 9277 transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS))) 9278 if err != nil { 9279 logError(testName, function, args, startTime, "", "DefaultTransport failed", err) 9280 return 9281 } 9282 9283 httpClient := &http.Client{ 9284 // Setting a sensible time out of 30secs to wait for response 9285 // headers. Request is pro-actively canceled after 30secs 9286 // with no response. 9287 Timeout: 30 * time.Second, 9288 Transport: transport, 9289 } 9290 9291 req, err := http.NewRequest(http.MethodHead, presignedHeadURL.String(), nil) 9292 if err != nil { 9293 logError(testName, function, args, startTime, "", "PresignedHeadObject URL head request failed", err) 9294 return 9295 } 9296 9297 // Verify if presigned url works. 9298 resp, err := httpClient.Do(req) 9299 if err != nil { 9300 logError(testName, function, args, startTime, "", "PresignedHeadObject URL head request failed", err) 9301 return 9302 } 9303 if resp.StatusCode != http.StatusOK { 9304 logError(testName, function, args, startTime, "", "PresignedHeadObject URL returns status "+string(resp.StatusCode), err) 9305 return 9306 } 9307 if resp.Header.Get("ETag") == "" { 9308 logError(testName, function, args, startTime, "", "Got empty ETag", err) 9309 return 9310 } 9311 resp.Body.Close() 9312 9313 // Generate presigned GET object url. 9314 function = "PresignedGetObject(bucketName, objectName, expires, reqParams)" 9315 functionAll += ", " + function 9316 args = map[string]interface{}{ 9317 "bucketName": bucketName, 9318 "objectName": objectName, 9319 "expires": 3600 * time.Second, 9320 } 9321 presignedGetURL, err := c.PresignedGetObject(bucketName, objectName, 3600*time.Second, nil) 9322 if err != nil { 9323 logError(testName, function, args, startTime, "", "PresignedGetObject failed", err) 9324 return 9325 } 9326 9327 // Verify if presigned url works. 9328 req, err = http.NewRequest(http.MethodGet, presignedGetURL.String(), nil) 9329 if err != nil { 9330 logError(testName, function, args, startTime, "", "PresignedGetObject request incorrect", err) 9331 return 9332 } 9333 9334 resp, err = httpClient.Do(req) 9335 if err != nil { 9336 logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) 9337 return 9338 } 9339 9340 if resp.StatusCode != http.StatusOK { 9341 logError(testName, function, args, startTime, "", "PresignedGetObject URL returns status "+string(resp.StatusCode), err) 9342 return 9343 } 9344 newPresignedBytes, err := ioutil.ReadAll(resp.Body) 9345 if err != nil { 9346 logError(testName, function, args, startTime, "", "ReadAll failed", err) 9347 return 9348 } 9349 resp.Body.Close() 9350 if !bytes.Equal(newPresignedBytes, buf) { 9351 logError(testName, function, args, startTime, "", "Bytes mismatch", err) 9352 return 9353 } 9354 9355 // Set request parameters. 9356 reqParams := make(url.Values) 9357 reqParams.Set("response-content-disposition", "attachment; filename=\"test.txt\"") 9358 // Generate presigned GET object url. 9359 args["reqParams"] = reqParams 9360 presignedGetURL, err = c.PresignedGetObject(bucketName, objectName, 3600*time.Second, reqParams) 9361 if err != nil { 9362 logError(testName, function, args, startTime, "", "PresignedGetObject failed", err) 9363 return 9364 } 9365 9366 // Verify if presigned url works. 9367 req, err = http.NewRequest(http.MethodGet, presignedGetURL.String(), nil) 9368 if err != nil { 9369 logError(testName, function, args, startTime, "", "PresignedGetObject request incorrect", err) 9370 return 9371 } 9372 9373 resp, err = httpClient.Do(req) 9374 if err != nil { 9375 logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) 9376 return 9377 } 9378 9379 if resp.StatusCode != http.StatusOK { 9380 logError(testName, function, args, startTime, "", "PresignedGetObject URL returns status "+string(resp.StatusCode), err) 9381 return 9382 } 9383 newPresignedBytes, err = ioutil.ReadAll(resp.Body) 9384 if err != nil { 9385 logError(testName, function, args, startTime, "", "ReadAll failed", err) 9386 return 9387 } 9388 if !bytes.Equal(newPresignedBytes, buf) { 9389 logError(testName, function, args, startTime, "", "Bytes mismatch", err) 9390 return 9391 } 9392 // Verify content disposition. 9393 if resp.Header.Get("Content-Disposition") != "attachment; filename=\"test.txt\"" { 9394 logError(testName, function, args, startTime, "", "wrong Content-Disposition received ", err) 9395 return 9396 } 9397 9398 function = "PresignedPutObject(bucketName, objectName, expires)" 9399 functionAll += ", " + function 9400 args = map[string]interface{}{ 9401 "bucketName": bucketName, 9402 "objectName": objectName + "-presigned", 9403 "expires": 3600 * time.Second, 9404 } 9405 presignedPutURL, err := c.PresignedPutObject(bucketName, objectName+"-presigned", 3600*time.Second) 9406 if err != nil { 9407 logError(testName, function, args, startTime, "", "PresignedPutObject failed", err) 9408 return 9409 } 9410 9411 // Generate data more than 32K 9412 buf = bytes.Repeat([]byte("1"), rand.Intn(1<<10)+32*1024) 9413 9414 req, err = http.NewRequest(http.MethodPut, presignedPutURL.String(), bytes.NewReader(buf)) 9415 if err != nil { 9416 logError(testName, function, args, startTime, "", "HTTP request to PresignedPutObject URL failed", err) 9417 return 9418 } 9419 9420 resp, err = httpClient.Do(req) 9421 if err != nil { 9422 logError(testName, function, args, startTime, "", "HTTP request to PresignedPutObject URL failed", err) 9423 return 9424 } 9425 9426 function = "GetObject(bucketName, objectName)" 9427 functionAll += ", " + function 9428 args = map[string]interface{}{ 9429 "bucketName": bucketName, 9430 "objectName": objectName + "-presigned", 9431 } 9432 newReader, err = c.GetObject(bucketName, objectName+"-presigned", minio.GetObjectOptions{}) 9433 if err != nil { 9434 logError(testName, function, args, startTime, "", "GetObject failed", err) 9435 return 9436 } 9437 9438 newReadBytes, err = ioutil.ReadAll(newReader) 9439 if err != nil { 9440 logError(testName, function, args, startTime, "", "ReadAll failed", err) 9441 return 9442 } 9443 newReader.Close() 9444 9445 if !bytes.Equal(newReadBytes, buf) { 9446 logError(testName, function, args, startTime, "", "Bytes mismatch", err) 9447 return 9448 } 9449 9450 // Delete all objects and buckets 9451 if err = cleanupBucket(bucketName, c); err != nil { 9452 logError(testName, function, args, startTime, "", "Cleanup failed", err) 9453 return 9454 } 9455 9456 os.Remove(fileName) 9457 os.Remove(fileName + "-f") 9458 successLogger(testName, functionAll, args, startTime).Info() 9459 } 9460 9461 // Test get object with GetObjectWithContext 9462 func testGetObjectWithContext() { 9463 // initialize logging params 9464 startTime := time.Now() 9465 testName := getFuncName() 9466 function := "GetObjectWithContext(ctx, bucketName, objectName)" 9467 args := map[string]interface{}{ 9468 "ctx": "", 9469 "bucketName": "", 9470 "objectName": "", 9471 } 9472 // Seed random based on current time. 9473 rand.Seed(time.Now().Unix()) 9474 9475 // Instantiate new minio client object. 9476 c, err := minio.NewV4( 9477 os.Getenv(serverEndpoint), 9478 os.Getenv(accessKey), 9479 os.Getenv(secretKey), 9480 mustParseBool(os.Getenv(enableHTTPS)), 9481 ) 9482 if err != nil { 9483 logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) 9484 return 9485 } 9486 9487 // Enable tracing, write to stderr. 9488 // c.TraceOn(os.Stderr) 9489 9490 // Set user agent. 9491 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 9492 9493 // Generate a new random bucket name. 9494 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 9495 args["bucketName"] = bucketName 9496 9497 // Make a new bucket. 9498 err = c.MakeBucket(bucketName, "us-east-1") 9499 if err != nil { 9500 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 9501 return 9502 } 9503 9504 bufSize := dataFileMap["datafile-33-kB"] 9505 var reader = getDataReader("datafile-33-kB") 9506 defer reader.Close() 9507 // Save the data 9508 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 9509 args["objectName"] = objectName 9510 9511 _, err = c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) 9512 if err != nil { 9513 logError(testName, function, args, startTime, "", "PutObject failed", err) 9514 return 9515 } 9516 9517 ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) 9518 args["ctx"] = ctx 9519 defer cancel() 9520 9521 r, err := c.GetObjectWithContext(ctx, bucketName, objectName, minio.GetObjectOptions{}) 9522 if err != nil { 9523 logError(testName, function, args, startTime, "", "GetObjectWithContext failed unexpectedly", err) 9524 return 9525 } 9526 9527 if _, err = r.Stat(); err == nil { 9528 logError(testName, function, args, startTime, "", "GetObjectWithContext should fail on short timeout", err) 9529 return 9530 } 9531 r.Close() 9532 9533 ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) 9534 args["ctx"] = ctx 9535 defer cancel() 9536 9537 // Read the data back 9538 r, err = c.GetObjectWithContext(ctx, bucketName, objectName, minio.GetObjectOptions{}) 9539 if err != nil { 9540 logError(testName, function, args, startTime, "", "GetObjectWithContext failed", err) 9541 return 9542 } 9543 9544 st, err := r.Stat() 9545 if err != nil { 9546 logError(testName, function, args, startTime, "", "object Stat call failed", err) 9547 return 9548 } 9549 if st.Size != int64(bufSize) { 9550 logError(testName, function, args, startTime, "", "Number of bytes in stat does not match: want "+string(bufSize)+", got"+string(st.Size), err) 9551 return 9552 } 9553 if err := r.Close(); err != nil { 9554 logError(testName, function, args, startTime, "", "object Close() call failed", err) 9555 return 9556 } 9557 9558 // Delete all objects and buckets 9559 if err = cleanupBucket(bucketName, c); err != nil { 9560 logError(testName, function, args, startTime, "", "Cleanup failed", err) 9561 return 9562 } 9563 9564 successLogger(testName, function, args, startTime).Info() 9565 9566 } 9567 9568 // Test get object with FGetObjectWithContext 9569 func testFGetObjectWithContext() { 9570 // initialize logging params 9571 startTime := time.Now() 9572 testName := getFuncName() 9573 function := "FGetObjectWithContext(ctx, bucketName, objectName, fileName)" 9574 args := map[string]interface{}{ 9575 "ctx": "", 9576 "bucketName": "", 9577 "objectName": "", 9578 "fileName": "", 9579 } 9580 // Seed random based on current time. 9581 rand.Seed(time.Now().Unix()) 9582 9583 // Instantiate new minio client object. 9584 c, err := minio.NewV4( 9585 os.Getenv(serverEndpoint), 9586 os.Getenv(accessKey), 9587 os.Getenv(secretKey), 9588 mustParseBool(os.Getenv(enableHTTPS)), 9589 ) 9590 if err != nil { 9591 logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) 9592 return 9593 } 9594 9595 // Enable tracing, write to stderr. 9596 // c.TraceOn(os.Stderr) 9597 9598 // Set user agent. 9599 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 9600 9601 // Generate a new random bucket name. 9602 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 9603 args["bucketName"] = bucketName 9604 9605 // Make a new bucket. 9606 err = c.MakeBucket(bucketName, "us-east-1") 9607 if err != nil { 9608 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 9609 return 9610 } 9611 9612 bufSize := dataFileMap["datafile-1-MB"] 9613 var reader = getDataReader("datafile-1-MB") 9614 defer reader.Close() 9615 // Save the data 9616 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 9617 args["objectName"] = objectName 9618 9619 _, err = c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) 9620 if err != nil { 9621 logError(testName, function, args, startTime, "", "PutObject failed", err) 9622 return 9623 } 9624 9625 ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) 9626 args["ctx"] = ctx 9627 defer cancel() 9628 9629 fileName := "tempfile-context" 9630 args["fileName"] = fileName 9631 // Read the data back 9632 err = c.FGetObjectWithContext(ctx, bucketName, objectName, fileName+"-f", minio.GetObjectOptions{}) 9633 if err == nil { 9634 logError(testName, function, args, startTime, "", "FGetObjectWithContext should fail on short timeout", err) 9635 return 9636 } 9637 ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) 9638 defer cancel() 9639 9640 // Read the data back 9641 err = c.FGetObjectWithContext(ctx, bucketName, objectName, fileName+"-fcontext", minio.GetObjectOptions{}) 9642 if err != nil { 9643 logError(testName, function, args, startTime, "", "FGetObjectWithContext with long timeout failed", err) 9644 return 9645 } 9646 if err = os.Remove(fileName + "-fcontext"); err != nil { 9647 logError(testName, function, args, startTime, "", "Remove file failed", err) 9648 return 9649 } 9650 // Delete all objects and buckets 9651 if err = cleanupBucket(bucketName, c); err != nil { 9652 logError(testName, function, args, startTime, "", "Cleanup failed", err) 9653 return 9654 } 9655 9656 successLogger(testName, function, args, startTime).Info() 9657 9658 } 9659 9660 // Test get object ACLs with GetObjectACLWithContext 9661 func testGetObjectACLWithContext() { 9662 // initialize logging params 9663 startTime := time.Now() 9664 testName := getFuncName() 9665 function := "GetObjectACLWithContext(ctx, bucketName, objectName)" 9666 args := map[string]interface{}{ 9667 "ctx": "", 9668 "bucketName": "", 9669 "objectName": "", 9670 } 9671 // Seed random based on current time. 9672 rand.Seed(time.Now().Unix()) 9673 9674 // skipping region functional tests for non s3 runs 9675 if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { 9676 ignoredLog(testName, function, args, startTime, "Skipped region functional tests for non s3 runs").Info() 9677 return 9678 } 9679 9680 // Instantiate new minio client object. 9681 c, err := minio.NewV4( 9682 os.Getenv(serverEndpoint), 9683 os.Getenv(accessKey), 9684 os.Getenv(secretKey), 9685 mustParseBool(os.Getenv(enableHTTPS)), 9686 ) 9687 if err != nil { 9688 logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) 9689 return 9690 } 9691 9692 // Enable tracing, write to stderr. 9693 // c.TraceOn(os.Stderr) 9694 9695 // Set user agent. 9696 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 9697 9698 // Generate a new random bucket name. 9699 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 9700 args["bucketName"] = bucketName 9701 9702 // Make a new bucket. 9703 err = c.MakeBucket(bucketName, "us-east-1") 9704 if err != nil { 9705 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 9706 return 9707 } 9708 9709 bufSize := dataFileMap["datafile-1-MB"] 9710 var reader = getDataReader("datafile-1-MB") 9711 defer reader.Close() 9712 // Save the data 9713 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 9714 args["objectName"] = objectName 9715 9716 // Add meta data to add a canned acl 9717 metaData := map[string]string{ 9718 "X-Amz-Acl": "public-read-write", 9719 } 9720 9721 _, err = c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream", UserMetadata: metaData}) 9722 if err != nil { 9723 logError(testName, function, args, startTime, "", "PutObject failed", err) 9724 return 9725 } 9726 9727 ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) 9728 args["ctx"] = ctx 9729 defer cancel() 9730 9731 // Read the data back 9732 objectInfo, getObjectACLErr := c.GetObjectACLWithContext(ctx, bucketName, objectName) 9733 if getObjectACLErr == nil { 9734 logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail", getObjectACLErr) 9735 return 9736 } 9737 9738 s, ok := objectInfo.Metadata["X-Amz-Acl"] 9739 if !ok { 9740 logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail unable to find \"X-Amz-Acl\"", nil) 9741 return 9742 } 9743 9744 if len(s) != 1 { 9745 logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail \"X-Amz-Acl\" canned acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil) 9746 return 9747 } 9748 9749 if s[0] != "public-read-write" { 9750 logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail \"X-Amz-Acl\" expected \"public-read-write\" but got"+fmt.Sprintf("%q", s[0]), nil) 9751 return 9752 } 9753 9754 bufSize = dataFileMap["datafile-1-MB"] 9755 var reader2 = getDataReader("datafile-1-MB") 9756 defer reader2.Close() 9757 // Save the data 9758 objectName = randString(60, rand.NewSource(time.Now().UnixNano()), "") 9759 args["objectName"] = objectName 9760 9761 // Add meta data to add a canned acl 9762 metaData = map[string]string{ 9763 "X-Amz-Grant-Read": "id=fooread@minio.go", 9764 "X-Amz-Grant-Write": "id=foowrite@minio.go", 9765 } 9766 9767 _, err = c.PutObject(bucketName, objectName, reader2, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream", UserMetadata: metaData}) 9768 if err != nil { 9769 logError(testName, function, args, startTime, "", "PutObject failed", err) 9770 return 9771 } 9772 9773 ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) 9774 args["ctx"] = ctx 9775 defer cancel() 9776 9777 // Read the data back 9778 objectInfo, getObjectACLErr = c.GetObjectACLWithContext(ctx, bucketName, objectName) 9779 if getObjectACLErr == nil { 9780 logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail", getObjectACLErr) 9781 return 9782 } 9783 9784 if len(objectInfo.Metadata) != 3 { 9785 logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail expected \"3\" ACLs but got "+fmt.Sprintf(`"%d"`, len(objectInfo.Metadata)), nil) 9786 return 9787 } 9788 9789 s, ok = objectInfo.Metadata["X-Amz-Grant-Read"] 9790 if !ok { 9791 logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail unable to find \"X-Amz-Grant-Read\"", nil) 9792 return 9793 } 9794 9795 if len(s) != 1 { 9796 logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail \"X-Amz-Grant-Read\" acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil) 9797 return 9798 } 9799 9800 if s[0] != "fooread@minio.go" { 9801 logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail \"X-Amz-Grant-Read\" acl expected \"fooread@minio.go\" got "+fmt.Sprintf("%q", s), nil) 9802 return 9803 } 9804 9805 s, ok = objectInfo.Metadata["X-Amz-Grant-Write"] 9806 if !ok { 9807 logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail unable to find \"X-Amz-Grant-Write\"", nil) 9808 return 9809 } 9810 9811 if len(s) != 1 { 9812 logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail \"X-Amz-Grant-Write\" acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil) 9813 return 9814 } 9815 9816 if s[0] != "foowrite@minio.go" { 9817 logError(testName, function, args, startTime, "", "GetObjectACLWithContext fail \"X-Amz-Grant-Write\" acl expected \"foowrite@minio.go\" got "+fmt.Sprintf("%q", s), nil) 9818 return 9819 } 9820 9821 // Delete all objects and buckets 9822 if err = cleanupBucket(bucketName, c); err != nil { 9823 logError(testName, function, args, startTime, "", "Cleanup failed", err) 9824 return 9825 } 9826 9827 successLogger(testName, function, args, startTime).Info() 9828 } 9829 9830 // Test validates putObject with context to see if request cancellation is honored for V2. 9831 func testPutObjectWithContextV2() { 9832 // initialize logging params 9833 startTime := time.Now() 9834 testName := getFuncName() 9835 function := "PutObjectWithContext(ctx, bucketName, objectName, reader, size, opts)" 9836 args := map[string]interface{}{ 9837 "ctx": "", 9838 "bucketName": "", 9839 "objectName": "", 9840 "size": "", 9841 "opts": "", 9842 } 9843 // Instantiate new minio client object. 9844 c, err := minio.NewV2( 9845 os.Getenv(serverEndpoint), 9846 os.Getenv(accessKey), 9847 os.Getenv(secretKey), 9848 mustParseBool(os.Getenv(enableHTTPS)), 9849 ) 9850 if err != nil { 9851 logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) 9852 return 9853 } 9854 9855 // Enable tracing, write to stderr. 9856 // c.TraceOn(os.Stderr) 9857 9858 // Set user agent. 9859 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 9860 9861 // Make a new bucket. 9862 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 9863 args["bucketName"] = bucketName 9864 9865 err = c.MakeBucket(bucketName, "us-east-1") 9866 if err != nil { 9867 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 9868 return 9869 } 9870 defer c.RemoveBucket(bucketName) 9871 bufSize := dataFileMap["datatfile-33-kB"] 9872 var reader = getDataReader("datafile-33-kB") 9873 defer reader.Close() 9874 9875 objectName := fmt.Sprintf("test-file-%v", rand.Uint32()) 9876 args["objectName"] = objectName 9877 9878 ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) 9879 args["ctx"] = ctx 9880 args["size"] = bufSize 9881 defer cancel() 9882 9883 _, err = c.PutObjectWithContext(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) 9884 if err != nil { 9885 logError(testName, function, args, startTime, "", "PutObjectWithContext with short timeout failed", err) 9886 return 9887 } 9888 9889 ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) 9890 args["ctx"] = ctx 9891 9892 defer cancel() 9893 reader = getDataReader("datafile-33-kB") 9894 defer reader.Close() 9895 _, err = c.PutObjectWithContext(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) 9896 if err != nil { 9897 logError(testName, function, args, startTime, "", "PutObjectWithContext with long timeout failed", err) 9898 return 9899 } 9900 9901 // Delete all objects and buckets 9902 if err = cleanupBucket(bucketName, c); err != nil { 9903 logError(testName, function, args, startTime, "", "Cleanup failed", err) 9904 return 9905 } 9906 9907 successLogger(testName, function, args, startTime).Info() 9908 9909 } 9910 9911 // Test get object with GetObjectWithContext 9912 func testGetObjectWithContextV2() { 9913 // initialize logging params 9914 startTime := time.Now() 9915 testName := getFuncName() 9916 function := "GetObjectWithContext(ctx, bucketName, objectName)" 9917 args := map[string]interface{}{ 9918 "ctx": "", 9919 "bucketName": "", 9920 "objectName": "", 9921 } 9922 // Seed random based on current time. 9923 rand.Seed(time.Now().Unix()) 9924 9925 // Instantiate new minio client object. 9926 c, err := minio.NewV2( 9927 os.Getenv(serverEndpoint), 9928 os.Getenv(accessKey), 9929 os.Getenv(secretKey), 9930 mustParseBool(os.Getenv(enableHTTPS)), 9931 ) 9932 if err != nil { 9933 logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) 9934 return 9935 } 9936 9937 // Enable tracing, write to stderr. 9938 // c.TraceOn(os.Stderr) 9939 9940 // Set user agent. 9941 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 9942 9943 // Generate a new random bucket name. 9944 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 9945 args["bucketName"] = bucketName 9946 9947 // Make a new bucket. 9948 err = c.MakeBucket(bucketName, "us-east-1") 9949 if err != nil { 9950 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 9951 return 9952 } 9953 9954 bufSize := dataFileMap["datafile-33-kB"] 9955 var reader = getDataReader("datafile-33-kB") 9956 defer reader.Close() 9957 // Save the data 9958 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 9959 args["objectName"] = objectName 9960 9961 _, err = c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) 9962 if err != nil { 9963 logError(testName, function, args, startTime, "", "PutObject call failed", err) 9964 return 9965 } 9966 9967 ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) 9968 args["ctx"] = ctx 9969 defer cancel() 9970 9971 r, err := c.GetObjectWithContext(ctx, bucketName, objectName, minio.GetObjectOptions{}) 9972 if err != nil { 9973 logError(testName, function, args, startTime, "", "GetObjectWithContext failed unexpectedly", err) 9974 return 9975 } 9976 if _, err = r.Stat(); err == nil { 9977 logError(testName, function, args, startTime, "", "GetObjectWithContext should fail on short timeout", err) 9978 return 9979 } 9980 r.Close() 9981 9982 ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) 9983 defer cancel() 9984 9985 // Read the data back 9986 r, err = c.GetObjectWithContext(ctx, bucketName, objectName, minio.GetObjectOptions{}) 9987 if err != nil { 9988 logError(testName, function, args, startTime, "", "GetObjectWithContext shouldn't fail on longer timeout", err) 9989 return 9990 } 9991 9992 st, err := r.Stat() 9993 if err != nil { 9994 logError(testName, function, args, startTime, "", "object Stat call failed", err) 9995 return 9996 } 9997 if st.Size != int64(bufSize) { 9998 logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(bufSize)+" got "+string(st.Size), err) 9999 return 10000 } 10001 if err := r.Close(); err != nil { 10002 logError(testName, function, args, startTime, "", " object Close() call failed", err) 10003 return 10004 } 10005 10006 // Delete all objects and buckets 10007 if err = cleanupBucket(bucketName, c); err != nil { 10008 logError(testName, function, args, startTime, "", "Cleanup failed", err) 10009 return 10010 } 10011 10012 successLogger(testName, function, args, startTime).Info() 10013 10014 } 10015 10016 // Test get object with FGetObjectWithContext 10017 func testFGetObjectWithContextV2() { 10018 // initialize logging params 10019 startTime := time.Now() 10020 testName := getFuncName() 10021 function := "FGetObjectWithContext(ctx, bucketName, objectName,fileName)" 10022 args := map[string]interface{}{ 10023 "ctx": "", 10024 "bucketName": "", 10025 "objectName": "", 10026 "fileName": "", 10027 } 10028 // Seed random based on current time. 10029 rand.Seed(time.Now().Unix()) 10030 10031 // Instantiate new minio client object. 10032 c, err := minio.NewV2( 10033 os.Getenv(serverEndpoint), 10034 os.Getenv(accessKey), 10035 os.Getenv(secretKey), 10036 mustParseBool(os.Getenv(enableHTTPS)), 10037 ) 10038 if err != nil { 10039 logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) 10040 return 10041 } 10042 10043 // Enable tracing, write to stderr. 10044 // c.TraceOn(os.Stderr) 10045 10046 // Set user agent. 10047 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 10048 10049 // Generate a new random bucket name. 10050 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 10051 args["bucketName"] = bucketName 10052 10053 // Make a new bucket. 10054 err = c.MakeBucket(bucketName, "us-east-1") 10055 if err != nil { 10056 logError(testName, function, args, startTime, "", "MakeBucket call failed", err) 10057 return 10058 } 10059 10060 bufSize := dataFileMap["datatfile-1-MB"] 10061 var reader = getDataReader("datafile-1-MB") 10062 defer reader.Close() 10063 // Save the data 10064 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 10065 args["objectName"] = objectName 10066 10067 _, err = c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) 10068 if err != nil { 10069 logError(testName, function, args, startTime, "", "PutObject call failed", err) 10070 return 10071 } 10072 10073 ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) 10074 args["ctx"] = ctx 10075 defer cancel() 10076 10077 fileName := "tempfile-context" 10078 args["fileName"] = fileName 10079 10080 // Read the data back 10081 err = c.FGetObjectWithContext(ctx, bucketName, objectName, fileName+"-f", minio.GetObjectOptions{}) 10082 if err == nil { 10083 logError(testName, function, args, startTime, "", "FGetObjectWithContext should fail on short timeout", err) 10084 return 10085 } 10086 ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) 10087 defer cancel() 10088 10089 // Read the data back 10090 err = c.FGetObjectWithContext(ctx, bucketName, objectName, fileName+"-fcontext", minio.GetObjectOptions{}) 10091 if err != nil { 10092 logError(testName, function, args, startTime, "", "FGetObjectWithContext call shouldn't fail on long timeout", err) 10093 return 10094 } 10095 10096 if err = os.Remove(fileName + "-fcontext"); err != nil { 10097 logError(testName, function, args, startTime, "", "Remove file failed", err) 10098 return 10099 } 10100 // Delete all objects and buckets 10101 if err = cleanupBucket(bucketName, c); err != nil { 10102 logError(testName, function, args, startTime, "", "Cleanup failed", err) 10103 return 10104 } 10105 10106 successLogger(testName, function, args, startTime).Info() 10107 10108 } 10109 10110 // Test list object v1 and V2 10111 func testListObjects() { 10112 // initialize logging params 10113 startTime := time.Now() 10114 testName := getFuncName() 10115 function := "ListObjects(bucketName, objectPrefix, recursive, doneCh)" 10116 args := map[string]interface{}{ 10117 "bucketName": "", 10118 "objectPrefix": "", 10119 "recursive": "true", 10120 } 10121 // Seed random based on current time. 10122 rand.Seed(time.Now().Unix()) 10123 10124 // Instantiate new minio client object. 10125 c, err := minio.New( 10126 os.Getenv(serverEndpoint), 10127 os.Getenv(accessKey), 10128 os.Getenv(secretKey), 10129 mustParseBool(os.Getenv(enableHTTPS)), 10130 ) 10131 if err != nil { 10132 logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) 10133 return 10134 } 10135 10136 // Enable tracing, write to stderr. 10137 // c.TraceOn(os.Stderr) 10138 10139 // Set user agent. 10140 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 10141 10142 // Generate a new random bucket name. 10143 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 10144 args["bucketName"] = bucketName 10145 10146 // Make a new bucket. 10147 err = c.MakeBucket(bucketName, "us-east-1") 10148 if err != nil { 10149 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 10150 return 10151 } 10152 10153 testObjects := []struct { 10154 name string 10155 storageClass string 10156 }{ 10157 // Special characters 10158 {"foo bar", "STANDARD"}, 10159 {"foo-%", "STANDARD"}, 10160 {"random-object-1", "STANDARD"}, 10161 {"random-object-2", "REDUCED_REDUNDANCY"}, 10162 } 10163 10164 for i, object := range testObjects { 10165 bufSize := dataFileMap["datafile-33-kB"] 10166 var reader = getDataReader("datafile-33-kB") 10167 defer reader.Close() 10168 _, err = c.PutObject(bucketName, object.name, reader, int64(bufSize), 10169 minio.PutObjectOptions{ContentType: "binary/octet-stream", StorageClass: object.storageClass}) 10170 if err != nil { 10171 logError(testName, function, args, startTime, "", fmt.Sprintf("PutObject %d call failed", i+1), err) 10172 return 10173 } 10174 } 10175 10176 testList := func(listFn func(string, string, bool, <-chan struct{}) <-chan minio.ObjectInfo, bucket string) { 10177 // Create a done channel to control 'ListObjects' go routine. 10178 doneCh := make(chan struct{}) 10179 // Exit cleanly upon return. 10180 defer close(doneCh) 10181 10182 var objCursor int 10183 10184 // check for object name and storage-class from listing object result 10185 for objInfo := range listFn(bucket, "", true, doneCh) { 10186 if objInfo.Err != nil { 10187 logError(testName, function, args, startTime, "", "ListObjects failed unexpectedly", err) 10188 return 10189 } 10190 if objInfo.Key != testObjects[objCursor].name { 10191 logError(testName, function, args, startTime, "", "ListObjects does not return expected object name", err) 10192 } 10193 if objInfo.StorageClass != testObjects[objCursor].storageClass { 10194 // Ignored as Gateways (Azure/GCS etc) wont return storage class 10195 ignoredLog(testName, function, args, startTime, "ListObjects doesn't return expected storage class").Info() 10196 } 10197 objCursor++ 10198 } 10199 10200 if objCursor != len(testObjects) { 10201 logError(testName, function, args, startTime, "", "ListObjects returned unexpected number of items", errors.New("")) 10202 } 10203 } 10204 10205 testList(c.ListObjects, bucketName) 10206 testList(c.ListObjectsV2, bucketName) 10207 10208 // Delete all objects and buckets 10209 if err = cleanupBucket(bucketName, c); err != nil { 10210 logError(testName, function, args, startTime, "", "Cleanup failed", err) 10211 return 10212 } 10213 10214 successLogger(testName, function, args, startTime).Info() 10215 } 10216 10217 // Test deleting multiple objects with object retention set in Governance mode 10218 func testRemoveObjectsWithOptions() { 10219 // initialize logging params 10220 startTime := time.Now() 10221 testName := getFuncName() 10222 function := "RemoveObjectsWithOptions(bucketName, objectsCh, opts)" 10223 args := map[string]interface{}{ 10224 "bucketName": "", 10225 "objectPrefix": "", 10226 "recursive": "true", 10227 } 10228 // Seed random based on current time. 10229 rand.Seed(time.Now().Unix()) 10230 10231 // Instantiate new minio client object. 10232 c, err := minio.New( 10233 os.Getenv(serverEndpoint), 10234 os.Getenv(accessKey), 10235 os.Getenv(secretKey), 10236 mustParseBool(os.Getenv(enableHTTPS)), 10237 ) 10238 if err != nil { 10239 logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) 10240 return 10241 } 10242 10243 // Enable tracing, write to stderr. 10244 // c.TraceOn(os.Stderr) 10245 10246 // Set user agent. 10247 c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") 10248 10249 // Generate a new random bucket name. 10250 bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") 10251 args["bucketName"] = bucketName 10252 objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") 10253 args["objectName"] = objectName 10254 10255 // Make a new bucket. 10256 err = c.MakeBucketWithObjectLock(bucketName, "us-east-1") 10257 if err != nil { 10258 logError(testName, function, args, startTime, "", "MakeBucket failed", err) 10259 return 10260 } 10261 10262 bufSize := dataFileMap["datafile-129-MB"] 10263 var reader = getDataReader("datafile-129-MB") 10264 defer reader.Close() 10265 10266 n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{}) 10267 if err != nil { 10268 log.Fatalln(err) 10269 } 10270 log.Println("Uploaded", objectName, " of size: ", n, "to bucket: ", bucketName, "Successfully.") 10271 10272 t := time.Date(2030, time.April, 25, 14, 0, 0, 0, time.UTC) 10273 m := minio.RetentionMode(minio.Governance) 10274 opts := minio.PutObjectRetentionOptions{ 10275 GovernanceBypass: false, 10276 RetainUntilDate: &t, 10277 Mode: &m, 10278 } 10279 err = c.PutObjectRetention(bucketName, objectName, opts) 10280 if err != nil { 10281 log.Fatalln(err) 10282 } 10283 10284 objectsCh := make(chan string) 10285 // Send object names that are needed to be removed to objectsCh 10286 go func() { 10287 defer close(objectsCh) 10288 // List all objects from a bucket-name with a matching prefix. 10289 for object := range c.ListObjects(bucketName, "", true, nil) { 10290 if object.Err != nil { 10291 log.Fatalln(object.Err) 10292 } 10293 objectsCh <- object.Key 10294 } 10295 }() 10296 10297 for rErr := range c.RemoveObjects(bucketName, objectsCh) { 10298 // Error is expected here because Retention is set on the object 10299 // and RemoveObjects is called without Bypass Governance 10300 if rErr.Err == nil { 10301 logError(testName, function, args, startTime, "", "Expected error during deletion", nil) 10302 return 10303 } 10304 } 10305 10306 objectsCh1 := make(chan string) 10307 10308 // Send object names that are needed to be removed to objectsCh 10309 go func() { 10310 defer close(objectsCh1) 10311 // List all objects from a bucket-name with a matching prefix. 10312 for object := range c.ListObjects(bucketName, "", true, nil) { 10313 if object.Err != nil { 10314 log.Fatalln(object.Err) 10315 } 10316 objectsCh1 <- object.Key 10317 } 10318 }() 10319 10320 opts1 := minio.RemoveObjectsOptions{ 10321 GovernanceBypass: true, 10322 } 10323 10324 for rErr := range c.RemoveObjectsWithOptions(bucketName, objectsCh1, opts1) { 10325 // Error is not expected here because Retention is set on the object 10326 // and RemoveObjects is called with Bypass Governance 10327 logError(testName, function, args, startTime, "", "Error detected during deletion", rErr.Err) 10328 return 10329 } 10330 10331 // Delete all objects and buckets 10332 if err = cleanupBucket(bucketName, c); err != nil { 10333 logError(testName, function, args, startTime, "", "Cleanup failed", err) 10334 return 10335 } 10336 10337 successLogger(testName, function, args, startTime).Info() 10338 } 10339 10340 // Convert string to bool and always return false if any error 10341 func mustParseBool(str string) bool { 10342 b, err := strconv.ParseBool(str) 10343 if err != nil { 10344 return false 10345 } 10346 return b 10347 } 10348 10349 func main() { 10350 // Output to stdout instead of the default stderr 10351 log.SetOutput(os.Stdout) 10352 // create custom formatter 10353 mintFormatter := mintJSONFormatter{} 10354 // set custom formatter 10355 log.SetFormatter(&mintFormatter) 10356 // log Info or above -- success cases are Info level, failures are Fatal level 10357 log.SetLevel(log.InfoLevel) 10358 10359 tls := mustParseBool(os.Getenv(enableHTTPS)) 10360 kmsEnabled := mustParseBool(os.Getenv(enableKMS)) 10361 // execute tests 10362 if isFullMode() { 10363 testMakeBucketErrorV2() 10364 testGetObjectClosedTwiceV2() 10365 testFPutObjectV2() 10366 testMakeBucketRegionsV2() 10367 testGetObjectReadSeekFunctionalV2() 10368 testGetObjectReadAtFunctionalV2() 10369 testCopyObjectV2() 10370 testFunctionalV2() 10371 testComposeObjectErrorCasesV2() 10372 testCompose10KSourcesV2() 10373 testUserMetadataCopyingV2() 10374 testPutObject0ByteV2() 10375 testPutObjectNoLengthV2() 10376 testPutObjectsUnknownV2() 10377 testGetObjectWithContextV2() 10378 testFPutObjectWithContextV2() 10379 testFGetObjectWithContextV2() 10380 testPutObjectWithContextV2() 10381 testMakeBucketError() 10382 testMakeBucketRegions() 10383 testPutObjectWithMetadata() 10384 testPutObjectReadAt() 10385 testPutObjectStreaming() 10386 testGetObjectSeekEnd() 10387 testGetObjectClosedTwice() 10388 testRemoveMultipleObjects() 10389 testFPutObjectMultipart() 10390 testFPutObject() 10391 testGetObjectReadSeekFunctional() 10392 testGetObjectReadAtFunctional() 10393 testGetObjectReadAtWhenEOFWasReached() 10394 testPresignedPostPolicy() 10395 testCopyObject() 10396 testComposeObjectErrorCases() 10397 testCompose10KSources() 10398 testUserMetadataCopying() 10399 testBucketNotification() 10400 testFunctional() 10401 testGetObjectModified() 10402 testPutObjectUploadSeekedObject() 10403 testGetObjectWithContext() 10404 testFPutObjectWithContext() 10405 testFGetObjectWithContext() 10406 testGetObjectACLWithContext() 10407 testPutObjectWithContext() 10408 testStorageClassMetadataPutObject() 10409 testStorageClassInvalidMetadataPutObject() 10410 testStorageClassMetadataCopyObject() 10411 testPutObjectWithContentLanguage() 10412 testListObjects() 10413 testRemoveObjectsWithOptions() 10414 10415 // SSE-C tests will only work over TLS connection. 10416 if tls { 10417 testSSECEncryptionPutGet() 10418 testSSECEncryptionFPut() 10419 testSSECEncryptedGetObjectReadAtFunctional() 10420 testSSECEncryptedGetObjectReadSeekFunctional() 10421 testEncryptedCopyObjectV2() 10422 testEncryptedSSECToSSECCopyObject() 10423 testEncryptedSSECToUnencryptedCopyObject() 10424 testUnencryptedToSSECCopyObject() 10425 testUnencryptedToUnencryptedCopyObject() 10426 testEncryptedEmptyObject() 10427 testDecryptedCopyObject() 10428 testSSECEncryptedToSSECCopyObjectPart() 10429 testSSECMultipartEncryptedToSSECCopyObjectPart() 10430 testSSECEncryptedToUnencryptedCopyPart() 10431 testUnencryptedToSSECCopyObjectPart() 10432 testUnencryptedToUnencryptedCopyPart() 10433 if kmsEnabled { 10434 testSSES3EncryptionPutGet() 10435 testSSES3EncryptionFPut() 10436 testSSES3EncryptedGetObjectReadAtFunctional() 10437 testSSES3EncryptedGetObjectReadSeekFunctional() 10438 testEncryptedSSECToSSES3CopyObject() 10439 testEncryptedSSES3ToSSECCopyObject() 10440 testEncryptedSSES3ToSSES3CopyObject() 10441 testEncryptedSSES3ToUnencryptedCopyObject() 10442 testUnencryptedToSSES3CopyObject() 10443 testSSECEncryptedToSSES3CopyObjectPart() 10444 testUnencryptedToSSES3CopyObjectPart() 10445 testSSES3EncryptedToSSECCopyObjectPart() 10446 testSSES3EncryptedToUnencryptedCopyPart() 10447 testSSES3EncryptedToSSES3CopyObjectPart() 10448 } 10449 } 10450 } else { 10451 testFunctional() 10452 testFunctionalV2() 10453 } 10454 }