github.com/rclone/rclone@v1.66.1-0.20240517100346-7b89735ae726/backend/sharefile/upload.go (about) 1 // Upload large files for sharefile 2 // 3 // Docs - https://api.sharefile.com/rest/docs/resource.aspx?name=Items#Upload_File 4 5 package sharefile 6 7 import ( 8 "bytes" 9 "context" 10 "crypto/md5" 11 "encoding/hex" 12 "encoding/json" 13 "fmt" 14 "io" 15 "strings" 16 "sync" 17 18 "github.com/rclone/rclone/backend/sharefile/api" 19 "github.com/rclone/rclone/fs" 20 "github.com/rclone/rclone/fs/accounting" 21 "github.com/rclone/rclone/lib/readers" 22 "github.com/rclone/rclone/lib/rest" 23 ) 24 25 // largeUpload is used to control the upload of large files which need chunking 26 type largeUpload struct { 27 ctx context.Context 28 f *Fs // parent Fs 29 o *Object // object being uploaded 30 in io.Reader // read the data from here 31 wrap accounting.WrapFn // account parts being transferred 32 size int64 // total size 33 parts int64 // calculated number of parts, if known 34 info *api.UploadSpecification // where to post chunks, etc. 35 threads int // number of threads to use in upload 36 streamed bool // set if using streamed upload 37 } 38 39 // newLargeUpload starts an upload of object o from in with metadata in src 40 func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo, info *api.UploadSpecification) (up *largeUpload, err error) { 41 size := src.Size() 42 parts := int64(-1) 43 if size >= 0 { 44 parts = size / int64(o.fs.opt.ChunkSize) 45 if size%int64(o.fs.opt.ChunkSize) != 0 { 46 parts++ 47 } 48 } 49 50 var streamed bool 51 switch strings.ToLower(info.Method) { 52 case "streamed": 53 streamed = true 54 case "threaded": 55 streamed = false 56 default: 57 return nil, fmt.Errorf("can't use method %q with newLargeUpload", info.Method) 58 } 59 60 threads := f.ci.Transfers 61 if threads > info.MaxNumberOfThreads { 62 threads = info.MaxNumberOfThreads 63 } 64 65 // unwrap the accounting from the input, we use wrap to put it 66 // back on after the buffering 67 in, wrap := accounting.UnWrap(in) 68 up = &largeUpload{ 69 ctx: ctx, 70 f: f, 71 o: o, 72 in: in, 73 wrap: wrap, 74 size: size, 75 threads: threads, 76 info: info, 77 parts: parts, 78 streamed: streamed, 79 } 80 return up, nil 81 } 82 83 // parse the api.UploadFinishResponse in respBody 84 func (up *largeUpload) parseUploadFinishResponse(respBody []byte) (err error) { 85 var finish api.UploadFinishResponse 86 err = json.Unmarshal(respBody, &finish) 87 if err != nil { 88 // Sometimes the unmarshal fails in which case return the body 89 return fmt.Errorf("upload: bad response: %q", bytes.TrimSpace(respBody)) 90 } 91 return up.o.checkUploadResponse(up.ctx, &finish) 92 } 93 94 // Transfer a chunk 95 func (up *largeUpload) transferChunk(ctx context.Context, part int64, offset int64, body []byte, fileHash string) error { 96 md5sumRaw := md5.Sum(body) 97 md5sum := hex.EncodeToString(md5sumRaw[:]) 98 size := int64(len(body)) 99 100 // Add some more parameters to the ChunkURI 101 u := up.info.ChunkURI 102 u += fmt.Sprintf("&index=%d&byteOffset=%d&hash=%s&fmt=json", 103 part, offset, md5sum, 104 ) 105 if fileHash != "" { 106 u += fmt.Sprintf("&finish=true&fileSize=%d&fileHash=%s", 107 offset+int64(len(body)), 108 fileHash, 109 ) 110 } 111 opts := rest.Opts{ 112 Method: "POST", 113 RootURL: u, 114 ContentLength: &size, 115 } 116 var respBody []byte 117 err := up.f.pacer.Call(func() (bool, error) { 118 fs.Debugf(up.o, "Sending chunk %d length %d", part, len(body)) 119 opts.Body = up.wrap(bytes.NewReader(body)) 120 resp, err := up.f.srv.Call(ctx, &opts) 121 if err != nil { 122 fs.Debugf(up.o, "Error sending chunk %d: %v", part, err) 123 } else { 124 respBody, err = rest.ReadBody(resp) 125 } 126 // retry all errors now that the multipart upload has started 127 return err != nil, err 128 }) 129 if err != nil { 130 fs.Debugf(up.o, "Error sending chunk %d: %v", part, err) 131 return err 132 } 133 // If last chunk and using "streamed" transfer, get the response back now 134 if up.streamed && fileHash != "" { 135 return up.parseUploadFinishResponse(respBody) 136 } 137 fs.Debugf(up.o, "Done sending chunk %d", part) 138 return nil 139 } 140 141 // finish closes off the large upload and reads the metadata 142 func (up *largeUpload) finish(ctx context.Context) error { 143 fs.Debugf(up.o, "Finishing large file upload") 144 // For a streamed transfer we will already have read the info 145 if up.streamed { 146 return nil 147 } 148 149 opts := rest.Opts{ 150 Method: "POST", 151 RootURL: up.info.FinishURI, 152 } 153 var respBody []byte 154 err := up.f.pacer.Call(func() (bool, error) { 155 resp, err := up.f.srv.Call(ctx, &opts) 156 if err != nil { 157 return shouldRetry(ctx, resp, err) 158 } 159 respBody, err = rest.ReadBody(resp) 160 // retry all errors now that the multipart upload has started 161 return err != nil, err 162 }) 163 if err != nil { 164 return err 165 } 166 return up.parseUploadFinishResponse(respBody) 167 } 168 169 // Upload uploads the chunks from the input 170 func (up *largeUpload) Upload(ctx context.Context) error { 171 if up.parts >= 0 { 172 fs.Debugf(up.o, "Starting upload of large file in %d chunks", up.parts) 173 } else { 174 fs.Debugf(up.o, "Starting streaming upload of large file") 175 } 176 var ( 177 offset int64 178 errs = make(chan error, 1) 179 wg sync.WaitGroup 180 err error 181 wholeFileHash = md5.New() 182 eof = false 183 ) 184 outer: 185 for part := int64(0); !eof; part++ { 186 // Check any errors 187 select { 188 case err = <-errs: 189 break outer 190 default: 191 } 192 193 // Get a block of memory 194 buf := up.f.getUploadBlock() 195 196 // Read the chunk 197 var n int 198 n, err = readers.ReadFill(up.in, buf) 199 if err == io.EOF { 200 eof = true 201 buf = buf[:n] 202 err = nil 203 } else if err != nil { 204 up.f.putUploadBlock(buf) 205 break outer 206 } 207 208 // Hash it 209 _, _ = io.Copy(wholeFileHash, bytes.NewBuffer(buf)) 210 211 // Get file hash if was last chunk 212 fileHash := "" 213 if eof { 214 fileHash = hex.EncodeToString(wholeFileHash.Sum(nil)) 215 } 216 217 // Transfer the chunk 218 wg.Add(1) 219 transferChunk := func(part, offset int64, buf []byte, fileHash string) { 220 defer wg.Done() 221 defer up.f.putUploadBlock(buf) 222 err := up.transferChunk(ctx, part, offset, buf, fileHash) 223 if err != nil { 224 select { 225 case errs <- err: 226 default: 227 } 228 } 229 } 230 if up.streamed { 231 transferChunk(part, offset, buf, fileHash) // streamed 232 } else { 233 go transferChunk(part, offset, buf, fileHash) // multithreaded 234 } 235 236 offset += int64(n) 237 } 238 wg.Wait() 239 240 // check size read is correct 241 if eof && err == nil && up.size >= 0 && up.size != offset { 242 err = fmt.Errorf("upload: short read: read %d bytes expected %d", up.size, offset) 243 } 244 245 // read any errors 246 if err == nil { 247 select { 248 case err = <-errs: 249 default: 250 } 251 } 252 253 // finish regardless of errors 254 finishErr := up.finish(ctx) 255 if err == nil { 256 err = finishErr 257 } 258 259 return err 260 }