github.com/rclone/rclone@v1.66.1-0.20240517100346-7b89735ae726/backend/sharefile/upload.go (about)

     1  // Upload large files for sharefile
     2  //
     3  // Docs - https://api.sharefile.com/rest/docs/resource.aspx?name=Items#Upload_File
     4  
     5  package sharefile
     6  
     7  import (
     8  	"bytes"
     9  	"context"
    10  	"crypto/md5"
    11  	"encoding/hex"
    12  	"encoding/json"
    13  	"fmt"
    14  	"io"
    15  	"strings"
    16  	"sync"
    17  
    18  	"github.com/rclone/rclone/backend/sharefile/api"
    19  	"github.com/rclone/rclone/fs"
    20  	"github.com/rclone/rclone/fs/accounting"
    21  	"github.com/rclone/rclone/lib/readers"
    22  	"github.com/rclone/rclone/lib/rest"
    23  )
    24  
    25  // largeUpload is used to control the upload of large files which need chunking
    26  type largeUpload struct {
    27  	ctx      context.Context
    28  	f        *Fs                      // parent Fs
    29  	o        *Object                  // object being uploaded
    30  	in       io.Reader                // read the data from here
    31  	wrap     accounting.WrapFn        // account parts being transferred
    32  	size     int64                    // total size
    33  	parts    int64                    // calculated number of parts, if known
    34  	info     *api.UploadSpecification // where to post chunks, etc.
    35  	threads  int                      // number of threads to use in upload
    36  	streamed bool                     // set if using streamed upload
    37  }
    38  
    39  // newLargeUpload starts an upload of object o from in with metadata in src
    40  func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo, info *api.UploadSpecification) (up *largeUpload, err error) {
    41  	size := src.Size()
    42  	parts := int64(-1)
    43  	if size >= 0 {
    44  		parts = size / int64(o.fs.opt.ChunkSize)
    45  		if size%int64(o.fs.opt.ChunkSize) != 0 {
    46  			parts++
    47  		}
    48  	}
    49  
    50  	var streamed bool
    51  	switch strings.ToLower(info.Method) {
    52  	case "streamed":
    53  		streamed = true
    54  	case "threaded":
    55  		streamed = false
    56  	default:
    57  		return nil, fmt.Errorf("can't use method %q with newLargeUpload", info.Method)
    58  	}
    59  
    60  	threads := f.ci.Transfers
    61  	if threads > info.MaxNumberOfThreads {
    62  		threads = info.MaxNumberOfThreads
    63  	}
    64  
    65  	// unwrap the accounting from the input, we use wrap to put it
    66  	// back on after the buffering
    67  	in, wrap := accounting.UnWrap(in)
    68  	up = &largeUpload{
    69  		ctx:      ctx,
    70  		f:        f,
    71  		o:        o,
    72  		in:       in,
    73  		wrap:     wrap,
    74  		size:     size,
    75  		threads:  threads,
    76  		info:     info,
    77  		parts:    parts,
    78  		streamed: streamed,
    79  	}
    80  	return up, nil
    81  }
    82  
    83  // parse the api.UploadFinishResponse in respBody
    84  func (up *largeUpload) parseUploadFinishResponse(respBody []byte) (err error) {
    85  	var finish api.UploadFinishResponse
    86  	err = json.Unmarshal(respBody, &finish)
    87  	if err != nil {
    88  		// Sometimes the unmarshal fails in which case return the body
    89  		return fmt.Errorf("upload: bad response: %q", bytes.TrimSpace(respBody))
    90  	}
    91  	return up.o.checkUploadResponse(up.ctx, &finish)
    92  }
    93  
    94  // Transfer a chunk
    95  func (up *largeUpload) transferChunk(ctx context.Context, part int64, offset int64, body []byte, fileHash string) error {
    96  	md5sumRaw := md5.Sum(body)
    97  	md5sum := hex.EncodeToString(md5sumRaw[:])
    98  	size := int64(len(body))
    99  
   100  	// Add some more parameters to the ChunkURI
   101  	u := up.info.ChunkURI
   102  	u += fmt.Sprintf("&index=%d&byteOffset=%d&hash=%s&fmt=json",
   103  		part, offset, md5sum,
   104  	)
   105  	if fileHash != "" {
   106  		u += fmt.Sprintf("&finish=true&fileSize=%d&fileHash=%s",
   107  			offset+int64(len(body)),
   108  			fileHash,
   109  		)
   110  	}
   111  	opts := rest.Opts{
   112  		Method:        "POST",
   113  		RootURL:       u,
   114  		ContentLength: &size,
   115  	}
   116  	var respBody []byte
   117  	err := up.f.pacer.Call(func() (bool, error) {
   118  		fs.Debugf(up.o, "Sending chunk %d length %d", part, len(body))
   119  		opts.Body = up.wrap(bytes.NewReader(body))
   120  		resp, err := up.f.srv.Call(ctx, &opts)
   121  		if err != nil {
   122  			fs.Debugf(up.o, "Error sending chunk %d: %v", part, err)
   123  		} else {
   124  			respBody, err = rest.ReadBody(resp)
   125  		}
   126  		// retry all errors now that the multipart upload has started
   127  		return err != nil, err
   128  	})
   129  	if err != nil {
   130  		fs.Debugf(up.o, "Error sending chunk %d: %v", part, err)
   131  		return err
   132  	}
   133  	// If last chunk and using "streamed" transfer, get the response back now
   134  	if up.streamed && fileHash != "" {
   135  		return up.parseUploadFinishResponse(respBody)
   136  	}
   137  	fs.Debugf(up.o, "Done sending chunk %d", part)
   138  	return nil
   139  }
   140  
   141  // finish closes off the large upload and reads the metadata
   142  func (up *largeUpload) finish(ctx context.Context) error {
   143  	fs.Debugf(up.o, "Finishing large file upload")
   144  	// For a streamed transfer we will already have read the info
   145  	if up.streamed {
   146  		return nil
   147  	}
   148  
   149  	opts := rest.Opts{
   150  		Method:  "POST",
   151  		RootURL: up.info.FinishURI,
   152  	}
   153  	var respBody []byte
   154  	err := up.f.pacer.Call(func() (bool, error) {
   155  		resp, err := up.f.srv.Call(ctx, &opts)
   156  		if err != nil {
   157  			return shouldRetry(ctx, resp, err)
   158  		}
   159  		respBody, err = rest.ReadBody(resp)
   160  		// retry all errors now that the multipart upload has started
   161  		return err != nil, err
   162  	})
   163  	if err != nil {
   164  		return err
   165  	}
   166  	return up.parseUploadFinishResponse(respBody)
   167  }
   168  
   169  // Upload uploads the chunks from the input
   170  func (up *largeUpload) Upload(ctx context.Context) error {
   171  	if up.parts >= 0 {
   172  		fs.Debugf(up.o, "Starting upload of large file in %d chunks", up.parts)
   173  	} else {
   174  		fs.Debugf(up.o, "Starting streaming upload of large file")
   175  	}
   176  	var (
   177  		offset        int64
   178  		errs          = make(chan error, 1)
   179  		wg            sync.WaitGroup
   180  		err           error
   181  		wholeFileHash = md5.New()
   182  		eof           = false
   183  	)
   184  outer:
   185  	for part := int64(0); !eof; part++ {
   186  		// Check any errors
   187  		select {
   188  		case err = <-errs:
   189  			break outer
   190  		default:
   191  		}
   192  
   193  		// Get a block of memory
   194  		buf := up.f.getUploadBlock()
   195  
   196  		// Read the chunk
   197  		var n int
   198  		n, err = readers.ReadFill(up.in, buf)
   199  		if err == io.EOF {
   200  			eof = true
   201  			buf = buf[:n]
   202  			err = nil
   203  		} else if err != nil {
   204  			up.f.putUploadBlock(buf)
   205  			break outer
   206  		}
   207  
   208  		// Hash it
   209  		_, _ = io.Copy(wholeFileHash, bytes.NewBuffer(buf))
   210  
   211  		// Get file hash if was last chunk
   212  		fileHash := ""
   213  		if eof {
   214  			fileHash = hex.EncodeToString(wholeFileHash.Sum(nil))
   215  		}
   216  
   217  		// Transfer the chunk
   218  		wg.Add(1)
   219  		transferChunk := func(part, offset int64, buf []byte, fileHash string) {
   220  			defer wg.Done()
   221  			defer up.f.putUploadBlock(buf)
   222  			err := up.transferChunk(ctx, part, offset, buf, fileHash)
   223  			if err != nil {
   224  				select {
   225  				case errs <- err:
   226  				default:
   227  				}
   228  			}
   229  		}
   230  		if up.streamed {
   231  			transferChunk(part, offset, buf, fileHash) // streamed
   232  		} else {
   233  			go transferChunk(part, offset, buf, fileHash) // multithreaded
   234  		}
   235  
   236  		offset += int64(n)
   237  	}
   238  	wg.Wait()
   239  
   240  	// check size read is correct
   241  	if eof && err == nil && up.size >= 0 && up.size != offset {
   242  		err = fmt.Errorf("upload: short read: read %d bytes expected %d", up.size, offset)
   243  	}
   244  
   245  	// read any errors
   246  	if err == nil {
   247  		select {
   248  		case err = <-errs:
   249  		default:
   250  		}
   251  	}
   252  
   253  	// finish regardless of errors
   254  	finishErr := up.finish(ctx)
   255  	if err == nil {
   256  		err = finishErr
   257  	}
   258  
   259  	return err
   260  }