github.com/minio/minio-go/v6@v6.0.57/api-put-object-common.go (about) 1 /* 2 * MinIO Go Library for Amazon S3 Compatible Cloud Storage 3 * Copyright 2015-2017 MinIO, Inc. 4 * 5 * Licensed under the Apache License, Version 2.0 (the "License"); 6 * you may not use this file except in compliance with the License. 7 * You may obtain a copy of the License at 8 * 9 * http://www.apache.org/licenses/LICENSE-2.0 10 * 11 * Unless required by applicable law or agreed to in writing, software 12 * distributed under the License is distributed on an "AS IS" BASIS, 13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 * See the License for the specific language governing permissions and 15 * limitations under the License. 16 */ 17 18 package minio 19 20 import ( 21 "context" 22 "io" 23 "math" 24 "os" 25 26 "github.com/minio/minio-go/v6/pkg/s3utils" 27 ) 28 29 // Verify if reader is *minio.Object 30 func isObject(reader io.Reader) (ok bool) { 31 _, ok = reader.(*Object) 32 return 33 } 34 35 // Verify if reader is a generic ReaderAt 36 func isReadAt(reader io.Reader) (ok bool) { 37 var v *os.File 38 v, ok = reader.(*os.File) 39 if ok { 40 // Stdin, Stdout and Stderr all have *os.File type 41 // which happen to also be io.ReaderAt compatible 42 // we need to add special conditions for them to 43 // be ignored by this function. 44 for _, f := range []string{ 45 "/dev/stdin", 46 "/dev/stdout", 47 "/dev/stderr", 48 } { 49 if f == v.Name() { 50 ok = false 51 break 52 } 53 } 54 } else { 55 _, ok = reader.(io.ReaderAt) 56 } 57 return 58 } 59 60 // optimalPartInfo - calculate the optimal part info for a given 61 // object size. 62 // 63 // NOTE: Assumption here is that for any object to be uploaded to any S3 compatible 64 // object storage it will have the following parameters as constants. 65 // 66 // maxPartsCount - 10000 67 // minPartSize - 128MiB 68 // maxMultipartPutObjectSize - 5TiB 69 // 70 func optimalPartInfo(objectSize int64, configuredPartSize uint64) (totalPartsCount int, partSize int64, lastPartSize int64, err error) { 71 // object size is '-1' set it to 5TiB. 72 var unknownSize bool 73 if objectSize == -1 { 74 unknownSize = true 75 objectSize = maxMultipartPutObjectSize 76 } 77 78 // object size is larger than supported maximum. 79 if objectSize > maxMultipartPutObjectSize { 80 err = ErrEntityTooLarge(objectSize, maxMultipartPutObjectSize, "", "") 81 return 82 } 83 84 var partSizeFlt float64 85 if configuredPartSize > 0 { 86 if int64(configuredPartSize) > objectSize { 87 err = ErrEntityTooLarge(int64(configuredPartSize), objectSize, "", "") 88 return 89 } 90 91 if !unknownSize { 92 if objectSize > (int64(configuredPartSize) * maxPartsCount) { 93 err = ErrInvalidArgument("Part size * max_parts(10000) is lesser than input objectSize.") 94 return 95 } 96 } 97 98 if configuredPartSize < absMinPartSize { 99 err = ErrInvalidArgument("Input part size is smaller than allowed minimum of 5MiB.") 100 return 101 } 102 103 if configuredPartSize > maxPartSize { 104 err = ErrInvalidArgument("Input part size is bigger than allowed maximum of 5GiB.") 105 return 106 } 107 108 partSizeFlt = float64(configuredPartSize) 109 if unknownSize { 110 // If input has unknown size and part size is configured 111 // keep it to maximum allowed as per 10000 parts. 112 objectSize = int64(configuredPartSize) * maxPartsCount 113 } 114 } else { 115 configuredPartSize = minPartSize 116 // Use floats for part size for all calculations to avoid 117 // overflows during float64 to int64 conversions. 118 partSizeFlt = float64(objectSize / maxPartsCount) 119 partSizeFlt = math.Ceil(partSizeFlt/float64(configuredPartSize)) * float64(configuredPartSize) 120 } 121 122 // Total parts count. 123 totalPartsCount = int(math.Ceil(float64(objectSize) / partSizeFlt)) 124 // Part size. 125 partSize = int64(partSizeFlt) 126 // Last part size. 127 lastPartSize = objectSize - int64(totalPartsCount-1)*partSize 128 return totalPartsCount, partSize, lastPartSize, nil 129 } 130 131 // getUploadID - fetch upload id if already present for an object name 132 // or initiate a new request to fetch a new upload id. 133 func (c Client) newUploadID(ctx context.Context, bucketName, objectName string, opts PutObjectOptions) (uploadID string, err error) { 134 // Input validation. 135 if err := s3utils.CheckValidBucketName(bucketName); err != nil { 136 return "", err 137 } 138 if err := s3utils.CheckValidObjectName(objectName); err != nil { 139 return "", err 140 } 141 142 // Initiate multipart upload for an object. 143 initMultipartUploadResult, err := c.initiateMultipartUpload(ctx, bucketName, objectName, opts) 144 if err != nil { 145 return "", err 146 } 147 return initMultipartUploadResult.UploadID, nil 148 }