github.com/rclone/rclone@v1.66.1-0.20240517100346-7b89735ae726/backend/drive/upload.go (about) 1 // Upload for drive 2 // 3 // Docs 4 // Resumable upload: https://developers.google.com/drive/web/manage-uploads#resumable 5 // Best practices: https://developers.google.com/drive/web/manage-uploads#best-practices 6 // Files insert: https://developers.google.com/drive/v2/reference/files/insert 7 // Files update: https://developers.google.com/drive/v2/reference/files/update 8 // 9 // This contains code adapted from google.golang.org/api (C) the GO AUTHORS 10 11 package drive 12 13 import ( 14 "bytes" 15 "context" 16 "encoding/json" 17 "fmt" 18 "io" 19 "net/http" 20 "net/url" 21 "strconv" 22 23 "github.com/rclone/rclone/fs" 24 "github.com/rclone/rclone/fs/fserrors" 25 "github.com/rclone/rclone/lib/readers" 26 "google.golang.org/api/drive/v3" 27 "google.golang.org/api/googleapi" 28 ) 29 30 const ( 31 // statusResumeIncomplete is the code returned by the Google uploader when the transfer is not yet complete. 32 statusResumeIncomplete = 308 33 ) 34 35 // resumableUpload is used by the generated APIs to provide resumable uploads. 36 // It is not used by developers directly. 37 type resumableUpload struct { 38 f *Fs 39 remote string 40 // URI is the resumable resource destination provided by the server after specifying "&uploadType=resumable". 41 URI string 42 // Media is the object being uploaded. 43 Media io.Reader 44 // MediaType defines the media type, e.g. "image/jpeg". 45 MediaType string 46 // ContentLength is the full size of the object being uploaded. 47 ContentLength int64 48 // Return value 49 ret *drive.File 50 } 51 52 // Upload the io.Reader in of size bytes with contentType and info 53 func (f *Fs) Upload(ctx context.Context, in io.Reader, size int64, contentType, fileID, remote string, info *drive.File) (*drive.File, error) { 54 params := url.Values{ 55 "alt": {"json"}, 56 "uploadType": {"resumable"}, 57 "fields": {partialFields}, 58 } 59 params.Set("supportsAllDrives", "true") 60 if f.opt.KeepRevisionForever { 61 params.Set("keepRevisionForever", "true") 62 } 63 urls := "https://www.googleapis.com/upload/drive/v3/files" 64 method := "POST" 65 if fileID != "" { 66 params.Set("setModifiedDate", "true") 67 urls += "/{fileId}" 68 method = "PATCH" 69 } 70 urls += "?" + params.Encode() 71 var res *http.Response 72 var err error 73 err = f.pacer.Call(func() (bool, error) { 74 var body io.Reader 75 body, err = googleapi.WithoutDataWrapper.JSONReader(info) 76 if err != nil { 77 return false, err 78 } 79 var req *http.Request 80 req, err = http.NewRequestWithContext(ctx, method, urls, body) 81 if err != nil { 82 return false, err 83 } 84 googleapi.Expand(req.URL, map[string]string{ 85 "fileId": fileID, 86 }) 87 req.Header.Set("Content-Type", "application/json; charset=UTF-8") 88 req.Header.Set("X-Upload-Content-Type", contentType) 89 if size >= 0 { 90 req.Header.Set("X-Upload-Content-Length", fmt.Sprintf("%v", size)) 91 } 92 res, err = f.client.Do(req) 93 if err == nil { 94 defer googleapi.CloseBody(res) 95 err = googleapi.CheckResponse(res) 96 } 97 return f.shouldRetry(ctx, err) 98 }) 99 if err != nil { 100 return nil, err 101 } 102 loc := res.Header.Get("Location") 103 rx := &resumableUpload{ 104 f: f, 105 remote: remote, 106 URI: loc, 107 Media: in, 108 MediaType: contentType, 109 ContentLength: size, 110 } 111 return rx.Upload(ctx) 112 } 113 114 // Make an http.Request for the range passed in 115 func (rx *resumableUpload) makeRequest(ctx context.Context, start int64, body io.ReadSeeker, reqSize int64) *http.Request { 116 req, _ := http.NewRequestWithContext(ctx, "POST", rx.URI, body) 117 req.ContentLength = reqSize 118 totalSize := "*" 119 if rx.ContentLength >= 0 { 120 totalSize = strconv.FormatInt(rx.ContentLength, 10) 121 } 122 if reqSize != 0 { 123 req.Header.Set("Content-Range", fmt.Sprintf("bytes %v-%v/%v", start, start+reqSize-1, totalSize)) 124 } else { 125 req.Header.Set("Content-Range", fmt.Sprintf("bytes */%v", totalSize)) 126 } 127 req.Header.Set("Content-Type", rx.MediaType) 128 return req 129 } 130 131 // Transfer a chunk - caller must call googleapi.CloseBody(res) if err == nil || res != nil 132 func (rx *resumableUpload) transferChunk(ctx context.Context, start int64, chunk io.ReadSeeker, chunkSize int64) (int, error) { 133 _, _ = chunk.Seek(0, io.SeekStart) 134 req := rx.makeRequest(ctx, start, chunk, chunkSize) 135 res, err := rx.f.client.Do(req) 136 if err != nil { 137 return 599, err 138 } 139 defer googleapi.CloseBody(res) 140 if res.StatusCode == statusResumeIncomplete { 141 return res.StatusCode, nil 142 } 143 err = googleapi.CheckResponse(res) 144 if err != nil { 145 return res.StatusCode, err 146 } 147 148 // When the entire file upload is complete, the server 149 // responds with an HTTP 201 Created along with any metadata 150 // associated with this resource. If this request had been 151 // updating an existing entity rather than creating a new one, 152 // the HTTP response code for a completed upload would have 153 // been 200 OK. 154 // 155 // So parse the response out of the body. We aren't expecting 156 // any other 2xx codes, so we parse it unconditionally on 157 // StatusCode 158 if err = json.NewDecoder(res.Body).Decode(&rx.ret); err != nil { 159 return 598, err 160 } 161 162 return res.StatusCode, nil 163 } 164 165 // Upload uploads the chunks from the input 166 // It retries each chunk using the pacer and --low-level-retries 167 func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) { 168 start := int64(0) 169 var StatusCode int 170 var err error 171 buf := make([]byte, int(rx.f.opt.ChunkSize)) 172 for finished := false; !finished; { 173 var reqSize int64 174 var chunk io.ReadSeeker 175 if rx.ContentLength >= 0 { 176 // If size known use repeatable reader for smoother bwlimit 177 if start >= rx.ContentLength { 178 break 179 } 180 reqSize = rx.ContentLength - start 181 if reqSize >= int64(rx.f.opt.ChunkSize) { 182 reqSize = int64(rx.f.opt.ChunkSize) 183 } 184 chunk = readers.NewRepeatableLimitReaderBuffer(rx.Media, buf, reqSize) 185 } else { 186 // If size unknown read into buffer 187 var n int 188 n, err = readers.ReadFill(rx.Media, buf) 189 if err == io.EOF { 190 // Send the last chunk with the correct ContentLength 191 // otherwise Google doesn't know we've finished 192 rx.ContentLength = start + int64(n) 193 finished = true 194 } else if err != nil { 195 return nil, err 196 } 197 reqSize = int64(n) 198 chunk = bytes.NewReader(buf[:reqSize]) 199 } 200 201 // Transfer the chunk 202 err = rx.f.pacer.Call(func() (bool, error) { 203 fs.Debugf(rx.remote, "Sending chunk %d length %d", start, reqSize) 204 StatusCode, err = rx.transferChunk(ctx, start, chunk, reqSize) 205 again, err := rx.f.shouldRetry(ctx, err) 206 if StatusCode == statusResumeIncomplete || StatusCode == http.StatusCreated || StatusCode == http.StatusOK { 207 again = false 208 err = nil 209 } 210 return again, err 211 }) 212 if err != nil { 213 return nil, err 214 } 215 216 start += reqSize 217 } 218 // Resume or retry uploads that fail due to connection interruptions or 219 // any 5xx errors, including: 220 // 221 // 500 Internal Server Error 222 // 502 Bad Gateway 223 // 503 Service Unavailable 224 // 504 Gateway Timeout 225 // 226 // Use an exponential backoff strategy if any 5xx server error is 227 // returned when resuming or retrying upload requests. These errors can 228 // occur if a server is getting overloaded. Exponential backoff can help 229 // alleviate these kinds of problems during periods of high volume of 230 // requests or heavy network traffic. Other kinds of requests should not 231 // be handled by exponential backoff but you can still retry a number of 232 // them. When retrying these requests, limit the number of times you 233 // retry them. For example your code could limit to ten retries or less 234 // before reporting an error. 235 // 236 // Handle 404 Not Found errors when doing resumable uploads by starting 237 // the entire upload over from the beginning. 238 if rx.ret == nil { 239 return nil, fserrors.RetryErrorf("Incomplete upload - retry, last error %d", StatusCode) 240 } 241 return rx.ret, nil 242 }