github.com/10XDev/rclone@v1.52.3-0.20200626220027-16af9ab76b2a/backend/fichier/fichier.go (about) 1 package fichier 2 3 import ( 4 "context" 5 "fmt" 6 "io" 7 "net/http" 8 "strconv" 9 "strings" 10 "time" 11 12 "github.com/pkg/errors" 13 "github.com/rclone/rclone/fs" 14 "github.com/rclone/rclone/fs/config" 15 "github.com/rclone/rclone/fs/config/configmap" 16 "github.com/rclone/rclone/fs/config/configstruct" 17 "github.com/rclone/rclone/fs/fshttp" 18 "github.com/rclone/rclone/fs/hash" 19 "github.com/rclone/rclone/lib/dircache" 20 "github.com/rclone/rclone/lib/encoder" 21 "github.com/rclone/rclone/lib/pacer" 22 "github.com/rclone/rclone/lib/rest" 23 ) 24 25 const ( 26 rootID = "0" 27 apiBaseURL = "https://api.1fichier.com/v1" 28 minSleep = 400 * time.Millisecond // api is extremely rate limited now 29 maxSleep = 5 * time.Second 30 decayConstant = 2 // bigger for slower decay, exponential 31 attackConstant = 0 // start with max sleep 32 ) 33 34 func init() { 35 fs.Register(&fs.RegInfo{ 36 Name: "fichier", 37 Description: "1Fichier", 38 Config: func(name string, config configmap.Mapper) { 39 }, 40 NewFs: NewFs, 41 Options: []fs.Option{{ 42 Help: "Your API Key, get it from https://1fichier.com/console/params.pl", 43 Name: "api_key", 44 }, { 45 Help: "If you want to download a shared folder, add this parameter", 46 Name: "shared_folder", 47 Required: false, 48 Advanced: true, 49 }, { 50 Name: config.ConfigEncoding, 51 Help: config.ConfigEncodingHelp, 52 Advanced: true, 53 // Characters that need escaping 54 // 55 // '\\': '\', // FULLWIDTH REVERSE SOLIDUS 56 // '<': '<', // FULLWIDTH LESS-THAN SIGN 57 // '>': '>', // FULLWIDTH GREATER-THAN SIGN 58 // '"': '"', // FULLWIDTH QUOTATION MARK - not on the list but seems to be reserved 59 // '\'': ''', // FULLWIDTH APOSTROPHE 60 // '$': '$', // FULLWIDTH DOLLAR SIGN 61 // '`': '`', // FULLWIDTH GRAVE ACCENT 62 // 63 // Leading space and trailing space 64 Default: (encoder.Display | 65 encoder.EncodeBackSlash | 66 encoder.EncodeSingleQuote | 67 encoder.EncodeBackQuote | 68 encoder.EncodeDoubleQuote | 69 encoder.EncodeLtGt | 70 encoder.EncodeDollar | 71 encoder.EncodeLeftSpace | 72 encoder.EncodeRightSpace | 73 encoder.EncodeInvalidUtf8), 74 }}, 75 }) 76 } 77 78 // Options defines the configuration for this backend 79 type Options struct { 80 APIKey string `config:"api_key"` 81 SharedFolder string `config:"shared_folder"` 82 Enc encoder.MultiEncoder `config:"encoding"` 83 } 84 85 // Fs is the interface a cloud storage system must provide 86 type Fs struct { 87 root string 88 name string 89 features *fs.Features 90 opt Options 91 dirCache *dircache.DirCache 92 baseClient *http.Client 93 pacer *fs.Pacer 94 rest *rest.Client 95 } 96 97 // FindLeaf finds a directory of name leaf in the folder with ID pathID 98 func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) { 99 folderID, err := strconv.Atoi(pathID) 100 if err != nil { 101 return "", false, err 102 } 103 folders, err := f.listFolders(ctx, folderID) 104 if err != nil { 105 return "", false, err 106 } 107 108 for _, folder := range folders.SubFolders { 109 if folder.Name == leaf { 110 pathIDOut := strconv.Itoa(folder.ID) 111 return pathIDOut, true, nil 112 } 113 } 114 115 return "", false, nil 116 } 117 118 // CreateDir makes a directory with pathID as parent and name leaf 119 func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) { 120 folderID, err := strconv.Atoi(pathID) 121 if err != nil { 122 return "", err 123 } 124 resp, err := f.makeFolder(ctx, leaf, folderID) 125 if err != nil { 126 return "", err 127 } 128 return strconv.Itoa(resp.FolderID), err 129 } 130 131 // Name of the remote (as passed into NewFs) 132 func (f *Fs) Name() string { 133 return f.name 134 } 135 136 // Root of the remote (as passed into NewFs) 137 func (f *Fs) Root() string { 138 return f.root 139 } 140 141 // String returns a description of the FS 142 func (f *Fs) String() string { 143 return fmt.Sprintf("1Fichier root '%s'", f.root) 144 } 145 146 // Precision of the ModTimes in this Fs 147 func (f *Fs) Precision() time.Duration { 148 return fs.ModTimeNotSupported 149 } 150 151 // Hashes returns the supported hash types of the filesystem 152 func (f *Fs) Hashes() hash.Set { 153 return hash.Set(hash.Whirlpool) 154 } 155 156 // Features returns the optional features of this Fs 157 func (f *Fs) Features() *fs.Features { 158 return f.features 159 } 160 161 // NewFs makes a new Fs object from the path 162 // 163 // The path is of the form remote:path 164 // 165 // Remotes are looked up in the config file. If the remote isn't 166 // found then NotFoundInConfigFile will be returned. 167 // 168 // On Windows avoid single character remote names as they can be mixed 169 // up with drive letters. 170 func NewFs(name string, root string, config configmap.Mapper) (fs.Fs, error) { 171 opt := new(Options) 172 err := configstruct.Set(config, opt) 173 if err != nil { 174 return nil, err 175 } 176 177 // If using a Shared Folder override root 178 if opt.SharedFolder != "" { 179 root = "" 180 } 181 182 //workaround for wonky parser 183 root = strings.Trim(root, "/") 184 185 f := &Fs{ 186 name: name, 187 root: root, 188 opt: *opt, 189 pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant), pacer.AttackConstant(attackConstant))), 190 baseClient: &http.Client{}, 191 } 192 193 f.features = (&fs.Features{ 194 DuplicateFiles: true, 195 CanHaveEmptyDirectories: true, 196 }).Fill(f) 197 198 client := fshttp.NewClient(fs.Config) 199 200 f.rest = rest.NewClient(client).SetRoot(apiBaseURL) 201 202 f.rest.SetHeader("Authorization", "Bearer "+f.opt.APIKey) 203 204 f.dirCache = dircache.New(root, rootID, f) 205 206 ctx := context.Background() 207 208 // Find the current root 209 err = f.dirCache.FindRoot(ctx, false) 210 if err != nil { 211 // Assume it is a file 212 newRoot, remote := dircache.SplitPath(root) 213 tempF := *f 214 tempF.dirCache = dircache.New(newRoot, rootID, &tempF) 215 tempF.root = newRoot 216 // Make new Fs which is the parent 217 err = tempF.dirCache.FindRoot(ctx, false) 218 if err != nil { 219 // No root so return old f 220 return f, nil 221 } 222 _, err := tempF.NewObject(ctx, remote) 223 if err != nil { 224 if err == fs.ErrorObjectNotFound { 225 // File doesn't exist so return old f 226 return f, nil 227 } 228 return nil, err 229 } 230 f.features.Fill(&tempF) 231 // XXX: update the old f here instead of returning tempF, since 232 // `features` were already filled with functions having *f as a receiver. 233 // See https://github.com/rclone/rclone/issues/2182 234 f.dirCache = tempF.dirCache 235 f.root = tempF.root 236 // return an error with an fs which points to the parent 237 return f, fs.ErrorIsFile 238 } 239 return f, nil 240 } 241 242 // List the objects and directories in dir into entries. The 243 // entries can be returned in any order but should be for a 244 // complete directory. 245 // 246 // dir should be "" to list the root, and should not have 247 // trailing slashes. 248 // 249 // This should return ErrDirNotFound if the directory isn't 250 // found. 251 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { 252 if f.opt.SharedFolder != "" { 253 return f.listSharedFiles(ctx, f.opt.SharedFolder) 254 } 255 256 dirContent, err := f.listDir(ctx, dir) 257 if err != nil { 258 return nil, err 259 } 260 261 return dirContent, nil 262 } 263 264 // NewObject finds the Object at remote. If it can't be found 265 // it returns the error ErrorObjectNotFound. 266 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { 267 leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, false) 268 if err != nil { 269 if err == fs.ErrorDirNotFound { 270 return nil, fs.ErrorObjectNotFound 271 } 272 return nil, err 273 } 274 275 folderID, err := strconv.Atoi(directoryID) 276 if err != nil { 277 return nil, err 278 } 279 files, err := f.listFiles(ctx, folderID) 280 if err != nil { 281 return nil, err 282 } 283 284 for _, file := range files.Items { 285 if file.Filename == leaf { 286 path, ok := f.dirCache.GetInv(directoryID) 287 288 if !ok { 289 return nil, errors.New("Cannot find dir in dircache") 290 } 291 292 return f.newObjectFromFile(ctx, path, file), nil 293 } 294 } 295 296 return nil, fs.ErrorObjectNotFound 297 } 298 299 // Put in to the remote path with the modTime given of the given size 300 // 301 // When called from outside an Fs by rclone, src.Size() will always be >= 0. 302 // But for unknown-sized objects (indicated by src.Size() == -1), Put should either 303 // return an error or upload it properly (rather than e.g. calling panic). 304 // 305 // May create the object even if it returns an error - if so 306 // will return the object and the error, otherwise will return 307 // nil and the error 308 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { 309 exisitingObj, err := f.NewObject(ctx, src.Remote()) 310 switch err { 311 case nil: 312 return exisitingObj, exisitingObj.Update(ctx, in, src, options...) 313 case fs.ErrorObjectNotFound: 314 // Not found so create it 315 return f.PutUnchecked(ctx, in, src, options...) 316 default: 317 return nil, err 318 } 319 } 320 321 // putUnchecked uploads the object with the given name and size 322 // 323 // This will create a duplicate if we upload a new file without 324 // checking to see if there is one already - use Put() for that. 325 func (f *Fs) putUnchecked(ctx context.Context, in io.Reader, remote string, size int64, options ...fs.OpenOption) (fs.Object, error) { 326 if size > int64(100e9) { 327 return nil, errors.New("File too big, cant upload") 328 } else if size == 0 { 329 return nil, fs.ErrorCantUploadEmptyFiles 330 } 331 332 nodeResponse, err := f.getUploadNode(ctx) 333 if err != nil { 334 return nil, err 335 } 336 337 leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, true) 338 if err != nil { 339 return nil, err 340 } 341 342 _, err = f.uploadFile(ctx, in, size, leaf, directoryID, nodeResponse.ID, nodeResponse.URL, options...) 343 if err != nil { 344 return nil, err 345 } 346 347 fileUploadResponse, err := f.endUpload(ctx, nodeResponse.ID, nodeResponse.URL) 348 if err != nil { 349 return nil, err 350 } 351 352 if len(fileUploadResponse.Links) != 1 { 353 return nil, errors.New("unexpected amount of files") 354 } 355 356 link := fileUploadResponse.Links[0] 357 fileSize, err := strconv.ParseInt(link.Size, 10, 64) 358 359 if err != nil { 360 return nil, err 361 } 362 363 return &Object{ 364 fs: f, 365 remote: remote, 366 file: File{ 367 ACL: 0, 368 CDN: 0, 369 Checksum: link.Whirlpool, 370 ContentType: "", 371 Date: time.Now().Format("2006-01-02 15:04:05"), 372 Filename: link.Filename, 373 Pass: 0, 374 Size: fileSize, 375 URL: link.Download, 376 }, 377 }, nil 378 } 379 380 // PutUnchecked uploads the object 381 // 382 // This will create a duplicate if we upload a new file without 383 // checking to see if there is one already - use Put() for that. 384 func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { 385 return f.putUnchecked(ctx, in, src.Remote(), src.Size(), options...) 386 } 387 388 // Mkdir makes the directory (container, bucket) 389 // 390 // Shouldn't return an error if it already exists 391 func (f *Fs) Mkdir(ctx context.Context, dir string) error { 392 err := f.dirCache.FindRoot(ctx, true) 393 if err != nil { 394 return err 395 } 396 if dir != "" { 397 _, err = f.dirCache.FindDir(ctx, dir, true) 398 } 399 return err 400 } 401 402 // Rmdir removes the directory (container, bucket) if empty 403 // 404 // Return an error if it doesn't exist or isn't empty 405 func (f *Fs) Rmdir(ctx context.Context, dir string) error { 406 err := f.dirCache.FindRoot(ctx, false) 407 if err != nil { 408 return err 409 } 410 411 directoryID, err := f.dirCache.FindDir(ctx, dir, false) 412 if err != nil { 413 return err 414 } 415 416 folderID, err := strconv.Atoi(directoryID) 417 if err != nil { 418 return err 419 } 420 421 _, err = f.removeFolder(ctx, dir, folderID) 422 if err != nil { 423 return err 424 } 425 426 f.dirCache.FlushDir(dir) 427 428 return nil 429 } 430 431 // Check the interfaces are satisfied 432 var ( 433 _ fs.Fs = (*Fs)(nil) 434 _ fs.PutUncheckeder = (*Fs)(nil) 435 _ dircache.DirCacher = (*Fs)(nil) 436 )