github.com/10XDev/rclone@v1.52.3-0.20200626220027-16af9ab76b2a/backend/fichier/fichier.go (about)

     1  package fichier
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"io"
     7  	"net/http"
     8  	"strconv"
     9  	"strings"
    10  	"time"
    11  
    12  	"github.com/pkg/errors"
    13  	"github.com/rclone/rclone/fs"
    14  	"github.com/rclone/rclone/fs/config"
    15  	"github.com/rclone/rclone/fs/config/configmap"
    16  	"github.com/rclone/rclone/fs/config/configstruct"
    17  	"github.com/rclone/rclone/fs/fshttp"
    18  	"github.com/rclone/rclone/fs/hash"
    19  	"github.com/rclone/rclone/lib/dircache"
    20  	"github.com/rclone/rclone/lib/encoder"
    21  	"github.com/rclone/rclone/lib/pacer"
    22  	"github.com/rclone/rclone/lib/rest"
    23  )
    24  
    25  const (
    26  	rootID         = "0"
    27  	apiBaseURL     = "https://api.1fichier.com/v1"
    28  	minSleep       = 400 * time.Millisecond // api is extremely rate limited now
    29  	maxSleep       = 5 * time.Second
    30  	decayConstant  = 2 // bigger for slower decay, exponential
    31  	attackConstant = 0 // start with max sleep
    32  )
    33  
    34  func init() {
    35  	fs.Register(&fs.RegInfo{
    36  		Name:        "fichier",
    37  		Description: "1Fichier",
    38  		Config: func(name string, config configmap.Mapper) {
    39  		},
    40  		NewFs: NewFs,
    41  		Options: []fs.Option{{
    42  			Help: "Your API Key, get it from https://1fichier.com/console/params.pl",
    43  			Name: "api_key",
    44  		}, {
    45  			Help:     "If you want to download a shared folder, add this parameter",
    46  			Name:     "shared_folder",
    47  			Required: false,
    48  			Advanced: true,
    49  		}, {
    50  			Name:     config.ConfigEncoding,
    51  			Help:     config.ConfigEncodingHelp,
    52  			Advanced: true,
    53  			// Characters that need escaping
    54  			//
    55  			// 		'\\': '\', // FULLWIDTH REVERSE SOLIDUS
    56  			// 		'<':  '<', // FULLWIDTH LESS-THAN SIGN
    57  			// 		'>':  '>', // FULLWIDTH GREATER-THAN SIGN
    58  			// 		'"':  '"', // FULLWIDTH QUOTATION MARK - not on the list but seems to be reserved
    59  			// 		'\'': ''', // FULLWIDTH APOSTROPHE
    60  			// 		'$':  '$', // FULLWIDTH DOLLAR SIGN
    61  			// 		'`':  '`', // FULLWIDTH GRAVE ACCENT
    62  			//
    63  			// Leading space and trailing space
    64  			Default: (encoder.Display |
    65  				encoder.EncodeBackSlash |
    66  				encoder.EncodeSingleQuote |
    67  				encoder.EncodeBackQuote |
    68  				encoder.EncodeDoubleQuote |
    69  				encoder.EncodeLtGt |
    70  				encoder.EncodeDollar |
    71  				encoder.EncodeLeftSpace |
    72  				encoder.EncodeRightSpace |
    73  				encoder.EncodeInvalidUtf8),
    74  		}},
    75  	})
    76  }
    77  
    78  // Options defines the configuration for this backend
    79  type Options struct {
    80  	APIKey       string               `config:"api_key"`
    81  	SharedFolder string               `config:"shared_folder"`
    82  	Enc          encoder.MultiEncoder `config:"encoding"`
    83  }
    84  
    85  // Fs is the interface a cloud storage system must provide
    86  type Fs struct {
    87  	root       string
    88  	name       string
    89  	features   *fs.Features
    90  	opt        Options
    91  	dirCache   *dircache.DirCache
    92  	baseClient *http.Client
    93  	pacer      *fs.Pacer
    94  	rest       *rest.Client
    95  }
    96  
    97  // FindLeaf finds a directory of name leaf in the folder with ID pathID
    98  func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {
    99  	folderID, err := strconv.Atoi(pathID)
   100  	if err != nil {
   101  		return "", false, err
   102  	}
   103  	folders, err := f.listFolders(ctx, folderID)
   104  	if err != nil {
   105  		return "", false, err
   106  	}
   107  
   108  	for _, folder := range folders.SubFolders {
   109  		if folder.Name == leaf {
   110  			pathIDOut := strconv.Itoa(folder.ID)
   111  			return pathIDOut, true, nil
   112  		}
   113  	}
   114  
   115  	return "", false, nil
   116  }
   117  
   118  // CreateDir makes a directory with pathID as parent and name leaf
   119  func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {
   120  	folderID, err := strconv.Atoi(pathID)
   121  	if err != nil {
   122  		return "", err
   123  	}
   124  	resp, err := f.makeFolder(ctx, leaf, folderID)
   125  	if err != nil {
   126  		return "", err
   127  	}
   128  	return strconv.Itoa(resp.FolderID), err
   129  }
   130  
   131  // Name of the remote (as passed into NewFs)
   132  func (f *Fs) Name() string {
   133  	return f.name
   134  }
   135  
   136  // Root of the remote (as passed into NewFs)
   137  func (f *Fs) Root() string {
   138  	return f.root
   139  }
   140  
   141  // String returns a description of the FS
   142  func (f *Fs) String() string {
   143  	return fmt.Sprintf("1Fichier root '%s'", f.root)
   144  }
   145  
   146  // Precision of the ModTimes in this Fs
   147  func (f *Fs) Precision() time.Duration {
   148  	return fs.ModTimeNotSupported
   149  }
   150  
   151  // Hashes returns the supported hash types of the filesystem
   152  func (f *Fs) Hashes() hash.Set {
   153  	return hash.Set(hash.Whirlpool)
   154  }
   155  
   156  // Features returns the optional features of this Fs
   157  func (f *Fs) Features() *fs.Features {
   158  	return f.features
   159  }
   160  
   161  // NewFs makes a new Fs object from the path
   162  //
   163  // The path is of the form remote:path
   164  //
   165  // Remotes are looked up in the config file.  If the remote isn't
   166  // found then NotFoundInConfigFile will be returned.
   167  //
   168  // On Windows avoid single character remote names as they can be mixed
   169  // up with drive letters.
   170  func NewFs(name string, root string, config configmap.Mapper) (fs.Fs, error) {
   171  	opt := new(Options)
   172  	err := configstruct.Set(config, opt)
   173  	if err != nil {
   174  		return nil, err
   175  	}
   176  
   177  	// If using a Shared Folder override root
   178  	if opt.SharedFolder != "" {
   179  		root = ""
   180  	}
   181  
   182  	//workaround for wonky parser
   183  	root = strings.Trim(root, "/")
   184  
   185  	f := &Fs{
   186  		name:       name,
   187  		root:       root,
   188  		opt:        *opt,
   189  		pacer:      fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant), pacer.AttackConstant(attackConstant))),
   190  		baseClient: &http.Client{},
   191  	}
   192  
   193  	f.features = (&fs.Features{
   194  		DuplicateFiles:          true,
   195  		CanHaveEmptyDirectories: true,
   196  	}).Fill(f)
   197  
   198  	client := fshttp.NewClient(fs.Config)
   199  
   200  	f.rest = rest.NewClient(client).SetRoot(apiBaseURL)
   201  
   202  	f.rest.SetHeader("Authorization", "Bearer "+f.opt.APIKey)
   203  
   204  	f.dirCache = dircache.New(root, rootID, f)
   205  
   206  	ctx := context.Background()
   207  
   208  	// Find the current root
   209  	err = f.dirCache.FindRoot(ctx, false)
   210  	if err != nil {
   211  		// Assume it is a file
   212  		newRoot, remote := dircache.SplitPath(root)
   213  		tempF := *f
   214  		tempF.dirCache = dircache.New(newRoot, rootID, &tempF)
   215  		tempF.root = newRoot
   216  		// Make new Fs which is the parent
   217  		err = tempF.dirCache.FindRoot(ctx, false)
   218  		if err != nil {
   219  			// No root so return old f
   220  			return f, nil
   221  		}
   222  		_, err := tempF.NewObject(ctx, remote)
   223  		if err != nil {
   224  			if err == fs.ErrorObjectNotFound {
   225  				// File doesn't exist so return old f
   226  				return f, nil
   227  			}
   228  			return nil, err
   229  		}
   230  		f.features.Fill(&tempF)
   231  		// XXX: update the old f here instead of returning tempF, since
   232  		// `features` were already filled with functions having *f as a receiver.
   233  		// See https://github.com/rclone/rclone/issues/2182
   234  		f.dirCache = tempF.dirCache
   235  		f.root = tempF.root
   236  		// return an error with an fs which points to the parent
   237  		return f, fs.ErrorIsFile
   238  	}
   239  	return f, nil
   240  }
   241  
   242  // List the objects and directories in dir into entries.  The
   243  // entries can be returned in any order but should be for a
   244  // complete directory.
   245  //
   246  // dir should be "" to list the root, and should not have
   247  // trailing slashes.
   248  //
   249  // This should return ErrDirNotFound if the directory isn't
   250  // found.
   251  func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
   252  	if f.opt.SharedFolder != "" {
   253  		return f.listSharedFiles(ctx, f.opt.SharedFolder)
   254  	}
   255  
   256  	dirContent, err := f.listDir(ctx, dir)
   257  	if err != nil {
   258  		return nil, err
   259  	}
   260  
   261  	return dirContent, nil
   262  }
   263  
   264  // NewObject finds the Object at remote.  If it can't be found
   265  // it returns the error ErrorObjectNotFound.
   266  func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
   267  	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, false)
   268  	if err != nil {
   269  		if err == fs.ErrorDirNotFound {
   270  			return nil, fs.ErrorObjectNotFound
   271  		}
   272  		return nil, err
   273  	}
   274  
   275  	folderID, err := strconv.Atoi(directoryID)
   276  	if err != nil {
   277  		return nil, err
   278  	}
   279  	files, err := f.listFiles(ctx, folderID)
   280  	if err != nil {
   281  		return nil, err
   282  	}
   283  
   284  	for _, file := range files.Items {
   285  		if file.Filename == leaf {
   286  			path, ok := f.dirCache.GetInv(directoryID)
   287  
   288  			if !ok {
   289  				return nil, errors.New("Cannot find dir in dircache")
   290  			}
   291  
   292  			return f.newObjectFromFile(ctx, path, file), nil
   293  		}
   294  	}
   295  
   296  	return nil, fs.ErrorObjectNotFound
   297  }
   298  
   299  // Put in to the remote path with the modTime given of the given size
   300  //
   301  // When called from outside an Fs by rclone, src.Size() will always be >= 0.
   302  // But for unknown-sized objects (indicated by src.Size() == -1), Put should either
   303  // return an error or upload it properly (rather than e.g. calling panic).
   304  //
   305  // May create the object even if it returns an error - if so
   306  // will return the object and the error, otherwise will return
   307  // nil and the error
   308  func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
   309  	exisitingObj, err := f.NewObject(ctx, src.Remote())
   310  	switch err {
   311  	case nil:
   312  		return exisitingObj, exisitingObj.Update(ctx, in, src, options...)
   313  	case fs.ErrorObjectNotFound:
   314  		// Not found so create it
   315  		return f.PutUnchecked(ctx, in, src, options...)
   316  	default:
   317  		return nil, err
   318  	}
   319  }
   320  
   321  // putUnchecked uploads the object with the given name and size
   322  //
   323  // This will create a duplicate if we upload a new file without
   324  // checking to see if there is one already - use Put() for that.
   325  func (f *Fs) putUnchecked(ctx context.Context, in io.Reader, remote string, size int64, options ...fs.OpenOption) (fs.Object, error) {
   326  	if size > int64(100e9) {
   327  		return nil, errors.New("File too big, cant upload")
   328  	} else if size == 0 {
   329  		return nil, fs.ErrorCantUploadEmptyFiles
   330  	}
   331  
   332  	nodeResponse, err := f.getUploadNode(ctx)
   333  	if err != nil {
   334  		return nil, err
   335  	}
   336  
   337  	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, true)
   338  	if err != nil {
   339  		return nil, err
   340  	}
   341  
   342  	_, err = f.uploadFile(ctx, in, size, leaf, directoryID, nodeResponse.ID, nodeResponse.URL, options...)
   343  	if err != nil {
   344  		return nil, err
   345  	}
   346  
   347  	fileUploadResponse, err := f.endUpload(ctx, nodeResponse.ID, nodeResponse.URL)
   348  	if err != nil {
   349  		return nil, err
   350  	}
   351  
   352  	if len(fileUploadResponse.Links) != 1 {
   353  		return nil, errors.New("unexpected amount of files")
   354  	}
   355  
   356  	link := fileUploadResponse.Links[0]
   357  	fileSize, err := strconv.ParseInt(link.Size, 10, 64)
   358  
   359  	if err != nil {
   360  		return nil, err
   361  	}
   362  
   363  	return &Object{
   364  		fs:     f,
   365  		remote: remote,
   366  		file: File{
   367  			ACL:         0,
   368  			CDN:         0,
   369  			Checksum:    link.Whirlpool,
   370  			ContentType: "",
   371  			Date:        time.Now().Format("2006-01-02 15:04:05"),
   372  			Filename:    link.Filename,
   373  			Pass:        0,
   374  			Size:        fileSize,
   375  			URL:         link.Download,
   376  		},
   377  	}, nil
   378  }
   379  
   380  // PutUnchecked uploads the object
   381  //
   382  // This will create a duplicate if we upload a new file without
   383  // checking to see if there is one already - use Put() for that.
   384  func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
   385  	return f.putUnchecked(ctx, in, src.Remote(), src.Size(), options...)
   386  }
   387  
   388  // Mkdir makes the directory (container, bucket)
   389  //
   390  // Shouldn't return an error if it already exists
   391  func (f *Fs) Mkdir(ctx context.Context, dir string) error {
   392  	err := f.dirCache.FindRoot(ctx, true)
   393  	if err != nil {
   394  		return err
   395  	}
   396  	if dir != "" {
   397  		_, err = f.dirCache.FindDir(ctx, dir, true)
   398  	}
   399  	return err
   400  }
   401  
   402  // Rmdir removes the directory (container, bucket) if empty
   403  //
   404  // Return an error if it doesn't exist or isn't empty
   405  func (f *Fs) Rmdir(ctx context.Context, dir string) error {
   406  	err := f.dirCache.FindRoot(ctx, false)
   407  	if err != nil {
   408  		return err
   409  	}
   410  
   411  	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
   412  	if err != nil {
   413  		return err
   414  	}
   415  
   416  	folderID, err := strconv.Atoi(directoryID)
   417  	if err != nil {
   418  		return err
   419  	}
   420  
   421  	_, err = f.removeFolder(ctx, dir, folderID)
   422  	if err != nil {
   423  		return err
   424  	}
   425  
   426  	f.dirCache.FlushDir(dir)
   427  
   428  	return nil
   429  }
   430  
   431  // Check the interfaces are satisfied
   432  var (
   433  	_ fs.Fs              = (*Fs)(nil)
   434  	_ fs.PutUncheckeder  = (*Fs)(nil)
   435  	_ dircache.DirCacher = (*Fs)(nil)
   436  )