github.com/10XDev/rclone@v1.52.3-0.20200626220027-16af9ab76b2a/backend/drive/upload.go (about) 1 // Upload for drive 2 // 3 // Docs 4 // Resumable upload: https://developers.google.com/drive/web/manage-uploads#resumable 5 // Best practices: https://developers.google.com/drive/web/manage-uploads#best-practices 6 // Files insert: https://developers.google.com/drive/v2/reference/files/insert 7 // Files update: https://developers.google.com/drive/v2/reference/files/update 8 // 9 // This contains code adapted from google.golang.org/api (C) the GO AUTHORS 10 11 package drive 12 13 import ( 14 "bytes" 15 "context" 16 "encoding/json" 17 "fmt" 18 "io" 19 "net/http" 20 "net/url" 21 "strconv" 22 23 "github.com/rclone/rclone/fs" 24 "github.com/rclone/rclone/fs/fserrors" 25 "github.com/rclone/rclone/lib/readers" 26 "google.golang.org/api/drive/v3" 27 "google.golang.org/api/googleapi" 28 ) 29 30 const ( 31 // statusResumeIncomplete is the code returned by the Google uploader when the transfer is not yet complete. 32 statusResumeIncomplete = 308 33 ) 34 35 // resumableUpload is used by the generated APIs to provide resumable uploads. 36 // It is not used by developers directly. 37 type resumableUpload struct { 38 f *Fs 39 remote string 40 // URI is the resumable resource destination provided by the server after specifying "&uploadType=resumable". 41 URI string 42 // Media is the object being uploaded. 43 Media io.Reader 44 // MediaType defines the media type, e.g. "image/jpeg". 45 MediaType string 46 // ContentLength is the full size of the object being uploaded. 47 ContentLength int64 48 // Return value 49 ret *drive.File 50 } 51 52 // Upload the io.Reader in of size bytes with contentType and info 53 func (f *Fs) Upload(ctx context.Context, in io.Reader, size int64, contentType, fileID, remote string, info *drive.File) (*drive.File, error) { 54 params := url.Values{ 55 "alt": {"json"}, 56 "uploadType": {"resumable"}, 57 "fields": {partialFields}, 58 } 59 params.Set("supportsAllDrives", "true") 60 if f.opt.KeepRevisionForever { 61 params.Set("keepRevisionForever", "true") 62 } 63 urls := "https://www.googleapis.com/upload/drive/v3/files" 64 method := "POST" 65 if fileID != "" { 66 params.Set("setModifiedDate", "true") 67 urls += "/{fileId}" 68 method = "PATCH" 69 } 70 urls += "?" + params.Encode() 71 var res *http.Response 72 var err error 73 err = f.pacer.Call(func() (bool, error) { 74 var body io.Reader 75 body, err = googleapi.WithoutDataWrapper.JSONReader(info) 76 if err != nil { 77 return false, err 78 } 79 var req *http.Request 80 req, err = http.NewRequest(method, urls, body) 81 if err != nil { 82 return false, err 83 } 84 req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext 85 googleapi.Expand(req.URL, map[string]string{ 86 "fileId": fileID, 87 }) 88 req.Header.Set("Content-Type", "application/json; charset=UTF-8") 89 req.Header.Set("X-Upload-Content-Type", contentType) 90 if size >= 0 { 91 req.Header.Set("X-Upload-Content-Length", fmt.Sprintf("%v", size)) 92 } 93 res, err = f.client.Do(req) 94 if err == nil { 95 defer googleapi.CloseBody(res) 96 err = googleapi.CheckResponse(res) 97 } 98 return f.shouldRetry(err) 99 }) 100 if err != nil { 101 return nil, err 102 } 103 loc := res.Header.Get("Location") 104 rx := &resumableUpload{ 105 f: f, 106 remote: remote, 107 URI: loc, 108 Media: in, 109 MediaType: contentType, 110 ContentLength: size, 111 } 112 return rx.Upload(ctx) 113 } 114 115 // Make an http.Request for the range passed in 116 func (rx *resumableUpload) makeRequest(ctx context.Context, start int64, body io.ReadSeeker, reqSize int64) *http.Request { 117 req, _ := http.NewRequest("POST", rx.URI, body) 118 req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext 119 req.ContentLength = reqSize 120 totalSize := "*" 121 if rx.ContentLength >= 0 { 122 totalSize = strconv.FormatInt(rx.ContentLength, 10) 123 } 124 if reqSize != 0 { 125 req.Header.Set("Content-Range", fmt.Sprintf("bytes %v-%v/%v", start, start+reqSize-1, totalSize)) 126 } else { 127 req.Header.Set("Content-Range", fmt.Sprintf("bytes */%v", totalSize)) 128 } 129 req.Header.Set("Content-Type", rx.MediaType) 130 return req 131 } 132 133 // Transfer a chunk - caller must call googleapi.CloseBody(res) if err == nil || res != nil 134 func (rx *resumableUpload) transferChunk(ctx context.Context, start int64, chunk io.ReadSeeker, chunkSize int64) (int, error) { 135 _, _ = chunk.Seek(0, io.SeekStart) 136 req := rx.makeRequest(ctx, start, chunk, chunkSize) 137 res, err := rx.f.client.Do(req) 138 if err != nil { 139 return 599, err 140 } 141 defer googleapi.CloseBody(res) 142 if res.StatusCode == statusResumeIncomplete { 143 return res.StatusCode, nil 144 } 145 err = googleapi.CheckResponse(res) 146 if err != nil { 147 return res.StatusCode, err 148 } 149 150 // When the entire file upload is complete, the server 151 // responds with an HTTP 201 Created along with any metadata 152 // associated with this resource. If this request had been 153 // updating an existing entity rather than creating a new one, 154 // the HTTP response code for a completed upload would have 155 // been 200 OK. 156 // 157 // So parse the response out of the body. We aren't expecting 158 // any other 2xx codes, so we parse it unconditionally on 159 // StatusCode 160 if err = json.NewDecoder(res.Body).Decode(&rx.ret); err != nil { 161 return 598, err 162 } 163 164 return res.StatusCode, nil 165 } 166 167 // Upload uploads the chunks from the input 168 // It retries each chunk using the pacer and --low-level-retries 169 func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) { 170 start := int64(0) 171 var StatusCode int 172 var err error 173 buf := make([]byte, int(rx.f.opt.ChunkSize)) 174 for finished := false; !finished; { 175 var reqSize int64 176 var chunk io.ReadSeeker 177 if rx.ContentLength >= 0 { 178 // If size known use repeatable reader for smoother bwlimit 179 if start >= rx.ContentLength { 180 break 181 } 182 reqSize = rx.ContentLength - start 183 if reqSize >= int64(rx.f.opt.ChunkSize) { 184 reqSize = int64(rx.f.opt.ChunkSize) 185 } 186 chunk = readers.NewRepeatableLimitReaderBuffer(rx.Media, buf, reqSize) 187 } else { 188 // If size unknown read into buffer 189 var n int 190 n, err = readers.ReadFill(rx.Media, buf) 191 if err == io.EOF { 192 // Send the last chunk with the correct ContentLength 193 // otherwise Google doesn't know we've finished 194 rx.ContentLength = start + int64(n) 195 finished = true 196 } else if err != nil { 197 return nil, err 198 } 199 reqSize = int64(n) 200 chunk = bytes.NewReader(buf[:reqSize]) 201 } 202 203 // Transfer the chunk 204 err = rx.f.pacer.Call(func() (bool, error) { 205 fs.Debugf(rx.remote, "Sending chunk %d length %d", start, reqSize) 206 StatusCode, err = rx.transferChunk(ctx, start, chunk, reqSize) 207 again, err := rx.f.shouldRetry(err) 208 if StatusCode == statusResumeIncomplete || StatusCode == http.StatusCreated || StatusCode == http.StatusOK { 209 again = false 210 err = nil 211 } 212 return again, err 213 }) 214 if err != nil { 215 return nil, err 216 } 217 218 start += reqSize 219 } 220 // Resume or retry uploads that fail due to connection interruptions or 221 // any 5xx errors, including: 222 // 223 // 500 Internal Server Error 224 // 502 Bad Gateway 225 // 503 Service Unavailable 226 // 504 Gateway Timeout 227 // 228 // Use an exponential backoff strategy if any 5xx server error is 229 // returned when resuming or retrying upload requests. These errors can 230 // occur if a server is getting overloaded. Exponential backoff can help 231 // alleviate these kinds of problems during periods of high volume of 232 // requests or heavy network traffic. Other kinds of requests should not 233 // be handled by exponential backoff but you can still retry a number of 234 // them. When retrying these requests, limit the number of times you 235 // retry them. For example your code could limit to ten retries or less 236 // before reporting an error. 237 // 238 // Handle 404 Not Found errors when doing resumable uploads by starting 239 // the entire upload over from the beginning. 240 if rx.ret == nil { 241 return nil, fserrors.RetryErrorf("Incomplete upload - retry, last error %d", StatusCode) 242 } 243 return rx.ret, nil 244 }