github.com/artpar/rclone@v1.67.3/backend/dropbox/batcher.go (about)

     1  // This file contains the implementation of the sync batcher for uploads
     2  //
     3  // Dropbox rules say you can start as many batches as you want, but
     4  // you may only have one batch being committed and must wait for the
     5  // batch to be finished before committing another.
     6  
     7  package dropbox
     8  
     9  import (
    10  	"context"
    11  	"fmt"
    12  
    13  	"github.com/artpar/rclone/fs/fserrors"
    14  	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
    15  )
    16  
    17  // finishBatch commits the batch, returning a batch status to poll or maybe complete
    18  func (f *Fs) finishBatch(ctx context.Context, items []*files.UploadSessionFinishArg) (complete *files.UploadSessionFinishBatchResult, err error) {
    19  	var arg = &files.UploadSessionFinishBatchArg{
    20  		Entries: items,
    21  	}
    22  	err = f.pacer.Call(func() (bool, error) {
    23  		complete, err = f.srv.UploadSessionFinishBatchV2(arg)
    24  		// If error is insufficient space then don't retry
    25  		if e, ok := err.(files.UploadSessionFinishAPIError); ok {
    26  			if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.WriteErrorInsufficientSpace {
    27  				err = fserrors.NoRetryError(err)
    28  				return false, err
    29  			}
    30  		}
    31  		// after the first chunk is uploaded, we retry everything
    32  		return err != nil, err
    33  	})
    34  	if err != nil {
    35  		return nil, fmt.Errorf("batch commit failed: %w", err)
    36  	}
    37  	return complete, nil
    38  }
    39  
    40  // Called by the batcher to commit a batch
    41  func (f *Fs) commitBatch(ctx context.Context, items []*files.UploadSessionFinishArg, results []*files.FileMetadata, errors []error) (err error) {
    42  	// finalise the batch getting either a result or a job id to poll
    43  	complete, err := f.finishBatch(ctx, items)
    44  	if err != nil {
    45  		return err
    46  	}
    47  
    48  	// Check we got the right number of entries
    49  	entries := complete.Entries
    50  	if len(entries) != len(results) {
    51  		return fmt.Errorf("expecting %d items in batch but got %d", len(results), len(entries))
    52  	}
    53  
    54  	// Format results for return
    55  	for i := range results {
    56  		item := entries[i]
    57  		if item.Tag == "success" {
    58  			results[i] = item.Success
    59  		} else {
    60  			errorTag := item.Tag
    61  			if item.Failure != nil {
    62  				errorTag = item.Failure.Tag
    63  				if item.Failure.LookupFailed != nil {
    64  					errorTag += "/" + item.Failure.LookupFailed.Tag
    65  				}
    66  				if item.Failure.Path != nil {
    67  					errorTag += "/" + item.Failure.Path.Tag
    68  				}
    69  				if item.Failure.PropertiesError != nil {
    70  					errorTag += "/" + item.Failure.PropertiesError.Tag
    71  				}
    72  			}
    73  			errors[i] = fmt.Errorf("upload failed: %s", errorTag)
    74  		}
    75  	}
    76  
    77  	return nil
    78  }