github.com/minio/minio-go/v6@v6.0.57/api-put-object-streaming.go (about) 1 /* 2 * MinIO Go Library for Amazon S3 Compatible Cloud Storage 3 * Copyright 2017 MinIO, Inc. 4 * 5 * Licensed under the Apache License, Version 2.0 (the "License"); 6 * you may not use this file except in compliance with the License. 7 * You may obtain a copy of the License at 8 * 9 * http://www.apache.org/licenses/LICENSE-2.0 10 * 11 * Unless required by applicable law or agreed to in writing, software 12 * distributed under the License is distributed on an "AS IS" BASIS, 13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 * See the License for the specific language governing permissions and 15 * limitations under the License. 16 */ 17 18 package minio 19 20 import ( 21 "bytes" 22 "context" 23 "encoding/base64" 24 "fmt" 25 "io" 26 "net/http" 27 "sort" 28 "strings" 29 30 "github.com/minio/minio-go/v6/pkg/s3utils" 31 ) 32 33 // putObjectMultipartStream - upload a large object using 34 // multipart upload and streaming signature for signing payload. 35 // Comprehensive put object operation involving multipart uploads. 36 // 37 // Following code handles these types of readers. 38 // 39 // - *minio.Object 40 // - Any reader which has a method 'ReadAt()' 41 // 42 func (c Client) putObjectMultipartStream(ctx context.Context, bucketName, objectName string, 43 reader io.Reader, size int64, opts PutObjectOptions) (n int64, err error) { 44 45 if !isObject(reader) && isReadAt(reader) && !opts.SendContentMd5 { 46 // Verify if the reader implements ReadAt and it is not a *minio.Object then we will use parallel uploader. 47 n, err = c.putObjectMultipartStreamFromReadAt(ctx, bucketName, objectName, reader.(io.ReaderAt), size, opts) 48 } else { 49 n, err = c.putObjectMultipartStreamOptionalChecksum(ctx, bucketName, objectName, reader, size, opts) 50 } 51 if err != nil { 52 errResp := ToErrorResponse(err) 53 // Verify if multipart functionality is not available, if not 54 // fall back to single PutObject operation. 55 if errResp.Code == "AccessDenied" && strings.Contains(errResp.Message, "Access Denied") { 56 // Verify if size of reader is greater than '5GiB'. 57 if size > maxSinglePutObjectSize { 58 return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName) 59 } 60 // Fall back to uploading as single PutObject operation. 61 return c.putObject(ctx, bucketName, objectName, reader, size, opts) 62 } 63 } 64 return n, err 65 } 66 67 // uploadedPartRes - the response received from a part upload. 68 type uploadedPartRes struct { 69 Error error // Any error encountered while uploading the part. 70 PartNum int // Number of the part uploaded. 71 Size int64 // Size of the part uploaded. 72 Part ObjectPart 73 } 74 75 type uploadPartReq struct { 76 PartNum int // Number of the part uploaded. 77 Part ObjectPart // Size of the part uploaded. 78 } 79 80 // putObjectMultipartFromReadAt - Uploads files bigger than 128MiB. 81 // Supports all readers which implements io.ReaderAt interface 82 // (ReadAt method). 83 // 84 // NOTE: This function is meant to be used for all readers which 85 // implement io.ReaderAt which allows us for resuming multipart 86 // uploads but reading at an offset, which would avoid re-read the 87 // data which was already uploaded. Internally this function uses 88 // temporary files for staging all the data, these temporary files are 89 // cleaned automatically when the caller i.e http client closes the 90 // stream after uploading all the contents successfully. 91 func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketName, objectName string, 92 reader io.ReaderAt, size int64, opts PutObjectOptions) (n int64, err error) { 93 // Input validation. 94 if err = s3utils.CheckValidBucketName(bucketName); err != nil { 95 return 0, err 96 } 97 if err = s3utils.CheckValidObjectName(objectName); err != nil { 98 return 0, err 99 } 100 101 // Calculate the optimal parts info for a given size. 102 totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(size, opts.PartSize) 103 if err != nil { 104 return 0, err 105 } 106 107 // Initiate a new multipart upload. 108 uploadID, err := c.newUploadID(ctx, bucketName, objectName, opts) 109 if err != nil { 110 return 0, err 111 } 112 113 // Aborts the multipart upload in progress, if the 114 // function returns any error, since we do not resume 115 // we should purge the parts which have been uploaded 116 // to relinquish storage space. 117 defer func() { 118 if err != nil { 119 c.abortMultipartUpload(ctx, bucketName, objectName, uploadID) 120 } 121 }() 122 123 // Total data read and written to server. should be equal to 'size' at the end of the call. 124 var totalUploadedSize int64 125 126 // Complete multipart upload. 127 var complMultipartUpload completeMultipartUpload 128 129 // Declare a channel that sends the next part number to be uploaded. 130 // Buffered to 10000 because thats the maximum number of parts allowed 131 // by S3. 132 uploadPartsCh := make(chan uploadPartReq, 10000) 133 134 // Declare a channel that sends back the response of a part upload. 135 // Buffered to 10000 because thats the maximum number of parts allowed 136 // by S3. 137 uploadedPartsCh := make(chan uploadedPartRes, 10000) 138 139 // Used for readability, lastPartNumber is always totalPartsCount. 140 lastPartNumber := totalPartsCount 141 142 // Send each part number to the channel to be processed. 143 for p := 1; p <= totalPartsCount; p++ { 144 uploadPartsCh <- uploadPartReq{PartNum: p} 145 } 146 close(uploadPartsCh) 147 // Receive each part number from the channel allowing three parallel uploads. 148 for w := 1; w <= opts.getNumThreads(); w++ { 149 go func(partSize int64) { 150 // Each worker will draw from the part channel and upload in parallel. 151 for uploadReq := range uploadPartsCh { 152 153 // If partNumber was not uploaded we calculate the missing 154 // part offset and size. For all other part numbers we 155 // calculate offset based on multiples of partSize. 156 readOffset := int64(uploadReq.PartNum-1) * partSize 157 158 // As a special case if partNumber is lastPartNumber, we 159 // calculate the offset based on the last part size. 160 if uploadReq.PartNum == lastPartNumber { 161 readOffset = (size - lastPartSize) 162 partSize = lastPartSize 163 } 164 165 // Get a section reader on a particular offset. 166 sectionReader := newHook(io.NewSectionReader(reader, readOffset, partSize), opts.Progress) 167 168 // Proceed to upload the part. 169 objPart, err := c.uploadPart(ctx, bucketName, objectName, uploadID, 170 sectionReader, uploadReq.PartNum, 171 "", "", partSize, opts.ServerSideEncryption) 172 if err != nil { 173 uploadedPartsCh <- uploadedPartRes{ 174 Error: err, 175 } 176 // Exit the goroutine. 177 return 178 } 179 180 // Save successfully uploaded part metadata. 181 uploadReq.Part = objPart 182 183 // Send successful part info through the channel. 184 uploadedPartsCh <- uploadedPartRes{ 185 Size: objPart.Size, 186 PartNum: uploadReq.PartNum, 187 Part: uploadReq.Part, 188 } 189 } 190 }(partSize) 191 } 192 193 // Gather the responses as they occur and update any 194 // progress bar. 195 for u := 1; u <= totalPartsCount; u++ { 196 uploadRes := <-uploadedPartsCh 197 if uploadRes.Error != nil { 198 return totalUploadedSize, uploadRes.Error 199 } 200 // Update the totalUploadedSize. 201 totalUploadedSize += uploadRes.Size 202 // Store the parts to be completed in order. 203 complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{ 204 ETag: uploadRes.Part.ETag, 205 PartNumber: uploadRes.Part.PartNumber, 206 }) 207 } 208 209 // Verify if we uploaded all the data. 210 if totalUploadedSize != size { 211 return totalUploadedSize, ErrUnexpectedEOF(totalUploadedSize, size, bucketName, objectName) 212 } 213 214 // Sort all completed parts. 215 sort.Sort(completedParts(complMultipartUpload.Parts)) 216 _, err = c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload) 217 if err != nil { 218 return totalUploadedSize, err 219 } 220 221 // Return final size. 222 return totalUploadedSize, nil 223 } 224 225 func (c Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, bucketName, objectName string, 226 reader io.Reader, size int64, opts PutObjectOptions) (n int64, err error) { 227 // Input validation. 228 if err = s3utils.CheckValidBucketName(bucketName); err != nil { 229 return 0, err 230 } 231 if err = s3utils.CheckValidObjectName(objectName); err != nil { 232 return 0, err 233 } 234 235 // Calculate the optimal parts info for a given size. 236 totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(size, opts.PartSize) 237 if err != nil { 238 return 0, err 239 } 240 // Initiates a new multipart request 241 uploadID, err := c.newUploadID(ctx, bucketName, objectName, opts) 242 if err != nil { 243 return 0, err 244 } 245 246 // Aborts the multipart upload if the function returns 247 // any error, since we do not resume we should purge 248 // the parts which have been uploaded to relinquish 249 // storage space. 250 defer func() { 251 if err != nil { 252 c.abortMultipartUpload(ctx, bucketName, objectName, uploadID) 253 } 254 }() 255 256 // Total data read and written to server. should be equal to 'size' at the end of the call. 257 var totalUploadedSize int64 258 259 // Initialize parts uploaded map. 260 partsInfo := make(map[int]ObjectPart) 261 262 // Create a buffer. 263 buf := make([]byte, partSize) 264 265 // Avoid declaring variables in the for loop 266 var md5Base64 string 267 var hookReader io.Reader 268 269 // Part number always starts with '1'. 270 var partNumber int 271 for partNumber = 1; partNumber <= totalPartsCount; partNumber++ { 272 273 // Proceed to upload the part. 274 if partNumber == totalPartsCount { 275 partSize = lastPartSize 276 } 277 278 if opts.SendContentMd5 { 279 length, rerr := io.ReadFull(reader, buf) 280 if rerr == io.EOF && partNumber > 1 { 281 break 282 } 283 if rerr != nil && rerr != io.ErrUnexpectedEOF && rerr != io.EOF { 284 return 0, rerr 285 } 286 // Calculate md5sum. 287 hash := c.md5Hasher() 288 hash.Write(buf[:length]) 289 md5Base64 = base64.StdEncoding.EncodeToString(hash.Sum(nil)) 290 hash.Close() 291 292 // Update progress reader appropriately to the latest offset 293 // as we read from the source. 294 hookReader = newHook(bytes.NewReader(buf[:length]), opts.Progress) 295 } else { 296 // Update progress reader appropriately to the latest offset 297 // as we read from the source. 298 hookReader = newHook(reader, opts.Progress) 299 } 300 301 objPart, uerr := c.uploadPart(ctx, bucketName, objectName, uploadID, 302 io.LimitReader(hookReader, partSize), 303 partNumber, md5Base64, "", partSize, opts.ServerSideEncryption) 304 if uerr != nil { 305 return totalUploadedSize, uerr 306 } 307 308 // Save successfully uploaded part metadata. 309 partsInfo[partNumber] = objPart 310 311 // Save successfully uploaded size. 312 totalUploadedSize += partSize 313 } 314 315 // Verify if we uploaded all the data. 316 if size > 0 { 317 if totalUploadedSize != size { 318 return totalUploadedSize, ErrUnexpectedEOF(totalUploadedSize, size, bucketName, objectName) 319 } 320 } 321 322 // Complete multipart upload. 323 var complMultipartUpload completeMultipartUpload 324 325 // Loop over total uploaded parts to save them in 326 // Parts array before completing the multipart request. 327 for i := 1; i < partNumber; i++ { 328 part, ok := partsInfo[i] 329 if !ok { 330 return 0, ErrInvalidArgument(fmt.Sprintf("Missing part number %d", i)) 331 } 332 complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{ 333 ETag: part.ETag, 334 PartNumber: part.PartNumber, 335 }) 336 } 337 338 // Sort all completed parts. 339 sort.Sort(completedParts(complMultipartUpload.Parts)) 340 _, err = c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload) 341 if err != nil { 342 return totalUploadedSize, err 343 } 344 345 // Return final size. 346 return totalUploadedSize, nil 347 } 348 349 // putObject special function used Google Cloud Storage. This special function 350 // is used for Google Cloud Storage since Google's multipart API is not S3 compatible. 351 func (c Client) putObject(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, opts PutObjectOptions) (n int64, err error) { 352 // Input validation. 353 if err := s3utils.CheckValidBucketName(bucketName); err != nil { 354 return 0, err 355 } 356 if err := s3utils.CheckValidObjectName(objectName); err != nil { 357 return 0, err 358 } 359 360 // Size -1 is only supported on Google Cloud Storage, we error 361 // out in all other situations. 362 if size < 0 && !s3utils.IsGoogleEndpoint(*c.endpointURL) { 363 return 0, ErrEntityTooSmall(size, bucketName, objectName) 364 } 365 366 if opts.SendContentMd5 && s3utils.IsGoogleEndpoint(*c.endpointURL) && size < 0 { 367 return 0, ErrInvalidArgument("MD5Sum cannot be calculated with size '-1'") 368 } 369 370 if size > 0 { 371 if isReadAt(reader) && !isObject(reader) { 372 seeker, ok := reader.(io.Seeker) 373 if ok { 374 offset, err := seeker.Seek(0, io.SeekCurrent) 375 if err != nil { 376 return 0, ErrInvalidArgument(err.Error()) 377 } 378 reader = io.NewSectionReader(reader.(io.ReaderAt), offset, size) 379 } 380 } 381 } 382 383 var md5Base64 string 384 if opts.SendContentMd5 { 385 // Create a buffer. 386 buf := make([]byte, size) 387 388 length, rErr := io.ReadFull(reader, buf) 389 if rErr != nil && rErr != io.ErrUnexpectedEOF { 390 return 0, rErr 391 } 392 393 // Calculate md5sum. 394 hash := c.md5Hasher() 395 hash.Write(buf[:length]) 396 md5Base64 = base64.StdEncoding.EncodeToString(hash.Sum(nil)) 397 reader = bytes.NewReader(buf[:length]) 398 hash.Close() 399 } 400 401 // Update progress reader appropriately to the latest offset as we 402 // read from the source. 403 readSeeker := newHook(reader, opts.Progress) 404 405 // This function does not calculate sha256 and md5sum for payload. 406 // Execute put object. 407 st, err := c.putObjectDo(ctx, bucketName, objectName, readSeeker, md5Base64, "", size, opts) 408 if err != nil { 409 return 0, err 410 } 411 if st.Size != size { 412 return 0, ErrUnexpectedEOF(st.Size, size, bucketName, objectName) 413 } 414 return size, nil 415 } 416 417 // putObjectDo - executes the put object http operation. 418 // NOTE: You must have WRITE permissions on a bucket to add an object to it. 419 func (c Client) putObjectDo(ctx context.Context, bucketName, objectName string, reader io.Reader, md5Base64, sha256Hex string, size int64, opts PutObjectOptions) (ObjectInfo, error) { 420 // Input validation. 421 if err := s3utils.CheckValidBucketName(bucketName); err != nil { 422 return ObjectInfo{}, err 423 } 424 if err := s3utils.CheckValidObjectName(objectName); err != nil { 425 return ObjectInfo{}, err 426 } 427 // Set headers. 428 customHeader := opts.Header() 429 430 // Populate request metadata. 431 reqMetadata := requestMetadata{ 432 bucketName: bucketName, 433 objectName: objectName, 434 customHeader: customHeader, 435 contentBody: reader, 436 contentLength: size, 437 contentMD5Base64: md5Base64, 438 contentSHA256Hex: sha256Hex, 439 } 440 441 // Execute PUT an objectName. 442 resp, err := c.executeMethod(ctx, "PUT", reqMetadata) 443 defer closeResponse(resp) 444 if err != nil { 445 return ObjectInfo{}, err 446 } 447 if resp != nil { 448 if resp.StatusCode != http.StatusOK { 449 return ObjectInfo{}, httpRespToErrorResponse(resp, bucketName, objectName) 450 } 451 } 452 453 var objInfo ObjectInfo 454 // Trim off the odd double quotes from ETag in the beginning and end. 455 objInfo.ETag = trimEtag(resp.Header.Get("ETag")) 456 // A success here means data was written to server successfully. 457 objInfo.Size = size 458 459 // Return here. 460 return objInfo, nil 461 }