github.com/10XDev/rclone@v1.52.3-0.20200626220027-16af9ab76b2a/backend/drive/upload.go (about)

     1  // Upload for drive
     2  //
     3  // Docs
     4  // Resumable upload: https://developers.google.com/drive/web/manage-uploads#resumable
     5  // Best practices: https://developers.google.com/drive/web/manage-uploads#best-practices
     6  // Files insert: https://developers.google.com/drive/v2/reference/files/insert
     7  // Files update: https://developers.google.com/drive/v2/reference/files/update
     8  //
     9  // This contains code adapted from google.golang.org/api (C) the GO AUTHORS
    10  
    11  package drive
    12  
    13  import (
    14  	"bytes"
    15  	"context"
    16  	"encoding/json"
    17  	"fmt"
    18  	"io"
    19  	"net/http"
    20  	"net/url"
    21  	"strconv"
    22  
    23  	"github.com/rclone/rclone/fs"
    24  	"github.com/rclone/rclone/fs/fserrors"
    25  	"github.com/rclone/rclone/lib/readers"
    26  	"google.golang.org/api/drive/v3"
    27  	"google.golang.org/api/googleapi"
    28  )
    29  
    30  const (
    31  	// statusResumeIncomplete is the code returned by the Google uploader when the transfer is not yet complete.
    32  	statusResumeIncomplete = 308
    33  )
    34  
    35  // resumableUpload is used by the generated APIs to provide resumable uploads.
    36  // It is not used by developers directly.
    37  type resumableUpload struct {
    38  	f      *Fs
    39  	remote string
    40  	// URI is the resumable resource destination provided by the server after specifying "&uploadType=resumable".
    41  	URI string
    42  	// Media is the object being uploaded.
    43  	Media io.Reader
    44  	// MediaType defines the media type, e.g. "image/jpeg".
    45  	MediaType string
    46  	// ContentLength is the full size of the object being uploaded.
    47  	ContentLength int64
    48  	// Return value
    49  	ret *drive.File
    50  }
    51  
    52  // Upload the io.Reader in of size bytes with contentType and info
    53  func (f *Fs) Upload(ctx context.Context, in io.Reader, size int64, contentType, fileID, remote string, info *drive.File) (*drive.File, error) {
    54  	params := url.Values{
    55  		"alt":        {"json"},
    56  		"uploadType": {"resumable"},
    57  		"fields":     {partialFields},
    58  	}
    59  	params.Set("supportsAllDrives", "true")
    60  	if f.opt.KeepRevisionForever {
    61  		params.Set("keepRevisionForever", "true")
    62  	}
    63  	urls := "https://www.googleapis.com/upload/drive/v3/files"
    64  	method := "POST"
    65  	if fileID != "" {
    66  		params.Set("setModifiedDate", "true")
    67  		urls += "/{fileId}"
    68  		method = "PATCH"
    69  	}
    70  	urls += "?" + params.Encode()
    71  	var res *http.Response
    72  	var err error
    73  	err = f.pacer.Call(func() (bool, error) {
    74  		var body io.Reader
    75  		body, err = googleapi.WithoutDataWrapper.JSONReader(info)
    76  		if err != nil {
    77  			return false, err
    78  		}
    79  		var req *http.Request
    80  		req, err = http.NewRequest(method, urls, body)
    81  		if err != nil {
    82  			return false, err
    83  		}
    84  		req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
    85  		googleapi.Expand(req.URL, map[string]string{
    86  			"fileId": fileID,
    87  		})
    88  		req.Header.Set("Content-Type", "application/json; charset=UTF-8")
    89  		req.Header.Set("X-Upload-Content-Type", contentType)
    90  		if size >= 0 {
    91  			req.Header.Set("X-Upload-Content-Length", fmt.Sprintf("%v", size))
    92  		}
    93  		res, err = f.client.Do(req)
    94  		if err == nil {
    95  			defer googleapi.CloseBody(res)
    96  			err = googleapi.CheckResponse(res)
    97  		}
    98  		return f.shouldRetry(err)
    99  	})
   100  	if err != nil {
   101  		return nil, err
   102  	}
   103  	loc := res.Header.Get("Location")
   104  	rx := &resumableUpload{
   105  		f:             f,
   106  		remote:        remote,
   107  		URI:           loc,
   108  		Media:         in,
   109  		MediaType:     contentType,
   110  		ContentLength: size,
   111  	}
   112  	return rx.Upload(ctx)
   113  }
   114  
   115  // Make an http.Request for the range passed in
   116  func (rx *resumableUpload) makeRequest(ctx context.Context, start int64, body io.ReadSeeker, reqSize int64) *http.Request {
   117  	req, _ := http.NewRequest("POST", rx.URI, body)
   118  	req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
   119  	req.ContentLength = reqSize
   120  	totalSize := "*"
   121  	if rx.ContentLength >= 0 {
   122  		totalSize = strconv.FormatInt(rx.ContentLength, 10)
   123  	}
   124  	if reqSize != 0 {
   125  		req.Header.Set("Content-Range", fmt.Sprintf("bytes %v-%v/%v", start, start+reqSize-1, totalSize))
   126  	} else {
   127  		req.Header.Set("Content-Range", fmt.Sprintf("bytes */%v", totalSize))
   128  	}
   129  	req.Header.Set("Content-Type", rx.MediaType)
   130  	return req
   131  }
   132  
   133  // Transfer a chunk - caller must call googleapi.CloseBody(res) if err == nil || res != nil
   134  func (rx *resumableUpload) transferChunk(ctx context.Context, start int64, chunk io.ReadSeeker, chunkSize int64) (int, error) {
   135  	_, _ = chunk.Seek(0, io.SeekStart)
   136  	req := rx.makeRequest(ctx, start, chunk, chunkSize)
   137  	res, err := rx.f.client.Do(req)
   138  	if err != nil {
   139  		return 599, err
   140  	}
   141  	defer googleapi.CloseBody(res)
   142  	if res.StatusCode == statusResumeIncomplete {
   143  		return res.StatusCode, nil
   144  	}
   145  	err = googleapi.CheckResponse(res)
   146  	if err != nil {
   147  		return res.StatusCode, err
   148  	}
   149  
   150  	// When the entire file upload is complete, the server
   151  	// responds with an HTTP 201 Created along with any metadata
   152  	// associated with this resource. If this request had been
   153  	// updating an existing entity rather than creating a new one,
   154  	// the HTTP response code for a completed upload would have
   155  	// been 200 OK.
   156  	//
   157  	// So parse the response out of the body.  We aren't expecting
   158  	// any other 2xx codes, so we parse it unconditionally on
   159  	// StatusCode
   160  	if err = json.NewDecoder(res.Body).Decode(&rx.ret); err != nil {
   161  		return 598, err
   162  	}
   163  
   164  	return res.StatusCode, nil
   165  }
   166  
   167  // Upload uploads the chunks from the input
   168  // It retries each chunk using the pacer and --low-level-retries
   169  func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) {
   170  	start := int64(0)
   171  	var StatusCode int
   172  	var err error
   173  	buf := make([]byte, int(rx.f.opt.ChunkSize))
   174  	for finished := false; !finished; {
   175  		var reqSize int64
   176  		var chunk io.ReadSeeker
   177  		if rx.ContentLength >= 0 {
   178  			// If size known use repeatable reader for smoother bwlimit
   179  			if start >= rx.ContentLength {
   180  				break
   181  			}
   182  			reqSize = rx.ContentLength - start
   183  			if reqSize >= int64(rx.f.opt.ChunkSize) {
   184  				reqSize = int64(rx.f.opt.ChunkSize)
   185  			}
   186  			chunk = readers.NewRepeatableLimitReaderBuffer(rx.Media, buf, reqSize)
   187  		} else {
   188  			// If size unknown read into buffer
   189  			var n int
   190  			n, err = readers.ReadFill(rx.Media, buf)
   191  			if err == io.EOF {
   192  				// Send the last chunk with the correct ContentLength
   193  				// otherwise Google doesn't know we've finished
   194  				rx.ContentLength = start + int64(n)
   195  				finished = true
   196  			} else if err != nil {
   197  				return nil, err
   198  			}
   199  			reqSize = int64(n)
   200  			chunk = bytes.NewReader(buf[:reqSize])
   201  		}
   202  
   203  		// Transfer the chunk
   204  		err = rx.f.pacer.Call(func() (bool, error) {
   205  			fs.Debugf(rx.remote, "Sending chunk %d length %d", start, reqSize)
   206  			StatusCode, err = rx.transferChunk(ctx, start, chunk, reqSize)
   207  			again, err := rx.f.shouldRetry(err)
   208  			if StatusCode == statusResumeIncomplete || StatusCode == http.StatusCreated || StatusCode == http.StatusOK {
   209  				again = false
   210  				err = nil
   211  			}
   212  			return again, err
   213  		})
   214  		if err != nil {
   215  			return nil, err
   216  		}
   217  
   218  		start += reqSize
   219  	}
   220  	// Resume or retry uploads that fail due to connection interruptions or
   221  	// any 5xx errors, including:
   222  	//
   223  	// 500 Internal Server Error
   224  	// 502 Bad Gateway
   225  	// 503 Service Unavailable
   226  	// 504 Gateway Timeout
   227  	//
   228  	// Use an exponential backoff strategy if any 5xx server error is
   229  	// returned when resuming or retrying upload requests. These errors can
   230  	// occur if a server is getting overloaded. Exponential backoff can help
   231  	// alleviate these kinds of problems during periods of high volume of
   232  	// requests or heavy network traffic.  Other kinds of requests should not
   233  	// be handled by exponential backoff but you can still retry a number of
   234  	// them. When retrying these requests, limit the number of times you
   235  	// retry them. For example your code could limit to ten retries or less
   236  	// before reporting an error.
   237  	//
   238  	// Handle 404 Not Found errors when doing resumable uploads by starting
   239  	// the entire upload over from the beginning.
   240  	if rx.ret == nil {
   241  		return nil, fserrors.RetryErrorf("Incomplete upload - retry, last error %d", StatusCode)
   242  	}
   243  	return rx.ret, nil
   244  }