github.com/divyam234/rclone@v1.64.1/fs/chunksize/chunksize.go (about) 1 // Package chunksize calculates a suitable chunk size for large uploads 2 package chunksize 3 4 import ( 5 "github.com/divyam234/rclone/fs" 6 ) 7 8 // Calculator calculates the minimum chunk size needed to fit within 9 // the maximum number of parts, rounded up to the nearest fs.Mebi. 10 // 11 // For most backends, (chunk_size) * (concurrent_upload_routines) 12 // memory will be required so we want to use the smallest possible 13 // chunk size that's going to allow the upload to proceed. Rounding up 14 // to the nearest fs.Mebi on the assumption that some backends may 15 // only allow integer type parameters when specifying the chunk size. 16 // 17 // Returns the default chunk size if it is sufficiently large enough 18 // to support the given file size otherwise returns the smallest chunk 19 // size necessary to allow the upload to proceed. 20 func Calculator(o interface{}, size int64, maxParts int, defaultChunkSize fs.SizeSuffix) fs.SizeSuffix { 21 // If streaming then use default chunk size 22 if size < 0 { 23 fs.Debugf(o, "Streaming upload with chunk_size %s allows uploads of up to %s and will fail only when that limit is reached.", defaultChunkSize, fs.SizeSuffix(maxParts)*defaultChunkSize) 24 25 return defaultChunkSize 26 } 27 fileSize := fs.SizeSuffix(size) 28 requiredChunks := fileSize / defaultChunkSize 29 if requiredChunks < fs.SizeSuffix(maxParts) || (requiredChunks == fs.SizeSuffix(maxParts) && fileSize%defaultChunkSize == 0) { 30 return defaultChunkSize 31 } 32 33 minChunk := fileSize / fs.SizeSuffix(maxParts) 34 remainder := minChunk % fs.Mebi 35 if remainder != 0 { 36 minChunk += fs.Mebi - remainder 37 } 38 if fileSize/minChunk == fs.SizeSuffix(maxParts) && fileSize%fs.SizeSuffix(maxParts) != 0 { // when right on the boundary, we need to add a MiB 39 minChunk += fs.Mebi 40 } 41 42 fs.Debugf(o, "size: %v, parts: %v, default: %v, new: %v; default chunk size insufficient, returned new chunk size", fileSize, maxParts, defaultChunkSize, minChunk) 43 return minChunk 44 }