github.com/rclone/rclone@v1.66.1-0.20240517100346-7b89735ae726/fs/operations/rc.go (about)

     1  package operations
     2  
     3  import (
     4  	"context"
     5  	"errors"
     6  	"fmt"
     7  	"io"
     8  	"mime"
     9  	"mime/multipart"
    10  	"net/http"
    11  	"path"
    12  	"strings"
    13  	"time"
    14  
    15  	"github.com/rclone/rclone/fs"
    16  	"github.com/rclone/rclone/fs/config"
    17  	"github.com/rclone/rclone/fs/hash"
    18  	"github.com/rclone/rclone/fs/rc"
    19  	"github.com/rclone/rclone/lib/diskusage"
    20  )
    21  
    22  func init() {
    23  	rc.Add(rc.Call{
    24  		Path:         "operations/list",
    25  		AuthRequired: true,
    26  		Fn:           rcList,
    27  		Title:        "List the given remote and path in JSON format",
    28  		Help: `This takes the following parameters:
    29  
    30  - fs - a remote name string e.g. "drive:"
    31  - remote - a path within that remote e.g. "dir"
    32  - opt - a dictionary of options to control the listing (optional)
    33      - recurse - If set recurse directories
    34      - noModTime - If set return modification time
    35      - showEncrypted -  If set show decrypted names
    36      - showOrigIDs - If set show the IDs for each item if known
    37      - showHash - If set return a dictionary of hashes
    38      - noMimeType - If set don't show mime types
    39      - dirsOnly - If set only show directories
    40      - filesOnly - If set only show files
    41      - metadata - If set return metadata of objects also
    42      - hashTypes - array of strings of hash types to show if showHash set
    43  
    44  Returns:
    45  
    46  - list
    47      - This is an array of objects as described in the lsjson command
    48  
    49  See the [lsjson](/commands/rclone_lsjson/) command for more information on the above and examples.
    50  `,
    51  	})
    52  }
    53  
    54  // List the directory
    55  func rcList(ctx context.Context, in rc.Params) (out rc.Params, err error) {
    56  	f, remote, err := rc.GetFsAndRemote(ctx, in)
    57  	if err != nil {
    58  		return nil, err
    59  	}
    60  	var opt ListJSONOpt
    61  	err = in.GetStruct("opt", &opt)
    62  	if rc.NotErrParamNotFound(err) {
    63  		return nil, err
    64  	}
    65  	var list = []*ListJSONItem{}
    66  	err = ListJSON(ctx, f, remote, &opt, func(item *ListJSONItem) error {
    67  		list = append(list, item)
    68  		return nil
    69  	})
    70  	if err != nil {
    71  		return nil, err
    72  	}
    73  	out = make(rc.Params)
    74  	out["list"] = list
    75  	return out, nil
    76  }
    77  
    78  func init() {
    79  	rc.Add(rc.Call{
    80  		Path:         "operations/stat",
    81  		AuthRequired: true,
    82  		Fn:           rcStat,
    83  		Title:        "Give information about the supplied file or directory",
    84  		Help: `This takes the following parameters
    85  
    86  - fs - a remote name string eg "drive:"
    87  - remote - a path within that remote eg "dir"
    88  - opt - a dictionary of options to control the listing (optional)
    89      - see operations/list for the options
    90  
    91  The result is
    92  
    93  - item - an object as described in the lsjson command. Will be null if not found.
    94  
    95  Note that if you are only interested in files then it is much more
    96  efficient to set the filesOnly flag in the options.
    97  
    98  See the [lsjson](/commands/rclone_lsjson/) command for more information on the above and examples.
    99  `,
   100  	})
   101  }
   102  
   103  // List the directory
   104  func rcStat(ctx context.Context, in rc.Params) (out rc.Params, err error) {
   105  	f, remote, err := rc.GetFsAndRemote(ctx, in)
   106  	if err != nil {
   107  		return nil, err
   108  	}
   109  	var opt ListJSONOpt
   110  	err = in.GetStruct("opt", &opt)
   111  	if rc.NotErrParamNotFound(err) {
   112  		return nil, err
   113  	}
   114  	item, err := StatJSON(ctx, f, remote, &opt)
   115  	if err != nil {
   116  		return nil, err
   117  	}
   118  	out = make(rc.Params)
   119  	out["item"] = item
   120  	return out, nil
   121  }
   122  
   123  func init() {
   124  	rc.Add(rc.Call{
   125  		Path:         "operations/about",
   126  		AuthRequired: true,
   127  		Fn:           rcAbout,
   128  		Title:        "Return the space used on the remote",
   129  		Help: `This takes the following parameters:
   130  
   131  - fs - a remote name string e.g. "drive:"
   132  
   133  The result is as returned from rclone about --json
   134  
   135  See the [about](/commands/rclone_about/) command for more information on the above.
   136  `,
   137  	})
   138  }
   139  
   140  // About the remote
   141  func rcAbout(ctx context.Context, in rc.Params) (out rc.Params, err error) {
   142  	f, err := rc.GetFs(ctx, in)
   143  	if err != nil {
   144  		return nil, err
   145  	}
   146  	doAbout := f.Features().About
   147  	if doAbout == nil {
   148  		return nil, fmt.Errorf("%v doesn't support about", f)
   149  	}
   150  	u, err := doAbout(ctx)
   151  	if err != nil {
   152  		return nil, fmt.Errorf("about call failed: %w", err)
   153  	}
   154  	err = rc.Reshape(&out, u)
   155  	if err != nil {
   156  		return nil, fmt.Errorf("about Reshape failed: %w", err)
   157  	}
   158  	return out, nil
   159  }
   160  
   161  func init() {
   162  	for _, copy := range []bool{false, true} {
   163  		copy := copy
   164  		name := "Move"
   165  		if copy {
   166  			name = "Copy"
   167  		}
   168  		rc.Add(rc.Call{
   169  			Path:         "operations/" + strings.ToLower(name) + "file",
   170  			AuthRequired: true,
   171  			Fn: func(ctx context.Context, in rc.Params) (rc.Params, error) {
   172  				return rcMoveOrCopyFile(ctx, in, copy)
   173  			},
   174  			Title: name + " a file from source remote to destination remote",
   175  			Help: `This takes the following parameters:
   176  
   177  - srcFs - a remote name string e.g. "drive:" for the source, "/" for local filesystem
   178  - srcRemote - a path within that remote e.g. "file.txt" for the source
   179  - dstFs - a remote name string e.g. "drive2:" for the destination, "/" for local filesystem
   180  - dstRemote - a path within that remote e.g. "file2.txt" for the destination
   181  `,
   182  		})
   183  	}
   184  }
   185  
   186  // Copy a file
   187  func rcMoveOrCopyFile(ctx context.Context, in rc.Params, cp bool) (out rc.Params, err error) {
   188  	srcFs, srcRemote, err := rc.GetFsAndRemoteNamed(ctx, in, "srcFs", "srcRemote")
   189  	if err != nil {
   190  		return nil, err
   191  	}
   192  	dstFs, dstRemote, err := rc.GetFsAndRemoteNamed(ctx, in, "dstFs", "dstRemote")
   193  	if err != nil {
   194  		return nil, err
   195  	}
   196  	return nil, moveOrCopyFile(ctx, dstFs, srcFs, dstRemote, srcRemote, cp)
   197  }
   198  
   199  func init() {
   200  	for _, op := range []struct {
   201  		name         string
   202  		title        string
   203  		help         string
   204  		noRemote     bool
   205  		needsRequest bool
   206  	}{
   207  		{name: "mkdir", title: "Make a destination directory or container"},
   208  		{name: "rmdir", title: "Remove an empty directory or container"},
   209  		{name: "purge", title: "Remove a directory or container and all of its contents"},
   210  		{name: "rmdirs", title: "Remove all the empty directories in the path", help: "- leaveRoot - boolean, set to true not to delete the root\n"},
   211  		{name: "delete", title: "Remove files in the path", noRemote: true},
   212  		{name: "deletefile", title: "Remove the single file pointed to"},
   213  		{name: "copyurl", title: "Copy the URL to the object", help: "- url - string, URL to read from\n - autoFilename - boolean, set to true to retrieve destination file name from url\n"},
   214  		{name: "uploadfile", title: "Upload file using multiform/form-data", help: "- each part in body represents a file to be uploaded\n", needsRequest: true},
   215  		{name: "cleanup", title: "Remove trashed files in the remote or path", noRemote: true},
   216  		{name: "settier", title: "Changes storage tier or class on all files in the path", noRemote: true},
   217  		{name: "settierfile", title: "Changes storage tier or class on the single file pointed to"},
   218  	} {
   219  		op := op
   220  		remote := "- remote - a path within that remote e.g. \"dir\"\n"
   221  		if op.noRemote {
   222  			remote = ""
   223  		}
   224  		rc.Add(rc.Call{
   225  			Path:         "operations/" + op.name,
   226  			AuthRequired: true,
   227  			NeedsRequest: op.needsRequest,
   228  			Fn: func(ctx context.Context, in rc.Params) (rc.Params, error) {
   229  				return rcSingleCommand(ctx, in, op.name, op.noRemote)
   230  			},
   231  			Title: op.title,
   232  			Help: `This takes the following parameters:
   233  
   234  - fs - a remote name string e.g. "drive:"
   235  ` + remote + op.help + `
   236  See the [` + op.name + `](/commands/rclone_` + op.name + `/) command for more information on the above.
   237  `,
   238  		})
   239  	}
   240  }
   241  
   242  // Run a single command, e.g. Mkdir
   243  func rcSingleCommand(ctx context.Context, in rc.Params, name string, noRemote bool) (out rc.Params, err error) {
   244  	var (
   245  		f      fs.Fs
   246  		remote string
   247  	)
   248  	if noRemote {
   249  		f, err = rc.GetFs(ctx, in)
   250  	} else {
   251  		f, remote, err = rc.GetFsAndRemote(ctx, in)
   252  	}
   253  	if err != nil {
   254  		return nil, err
   255  	}
   256  	switch name {
   257  	case "mkdir":
   258  		return nil, Mkdir(ctx, f, remote)
   259  	case "rmdir":
   260  		return nil, Rmdir(ctx, f, remote)
   261  	case "purge":
   262  		return nil, Purge(ctx, f, remote)
   263  	case "rmdirs":
   264  		leaveRoot, err := in.GetBool("leaveRoot")
   265  		if rc.NotErrParamNotFound(err) {
   266  			return nil, err
   267  		}
   268  		return nil, Rmdirs(ctx, f, remote, leaveRoot)
   269  	case "delete":
   270  		return nil, Delete(ctx, f)
   271  	case "deletefile":
   272  		o, err := f.NewObject(ctx, remote)
   273  		if err != nil {
   274  			return nil, err
   275  		}
   276  		return nil, DeleteFile(ctx, o)
   277  	case "copyurl":
   278  		url, err := in.GetString("url")
   279  		if err != nil {
   280  			return nil, err
   281  		}
   282  		autoFilename, _ := in.GetBool("autoFilename")
   283  		noClobber, _ := in.GetBool("noClobber")
   284  		headerFilename, _ := in.GetBool("headerFilename")
   285  
   286  		_, err = CopyURL(ctx, f, remote, url, autoFilename, headerFilename, noClobber)
   287  		return nil, err
   288  	case "uploadfile":
   289  
   290  		var request *http.Request
   291  		request, err := in.GetHTTPRequest()
   292  
   293  		if err != nil {
   294  			return nil, err
   295  		}
   296  
   297  		contentType := request.Header.Get("Content-Type")
   298  		mediaType, params, err := mime.ParseMediaType(contentType)
   299  		if err != nil {
   300  			return nil, err
   301  		}
   302  
   303  		if strings.HasPrefix(mediaType, "multipart/") {
   304  			mr := multipart.NewReader(request.Body, params["boundary"])
   305  			for {
   306  				p, err := mr.NextPart()
   307  				if err == io.EOF {
   308  					return nil, nil
   309  				}
   310  				if err != nil {
   311  					return nil, err
   312  				}
   313  				if p.FileName() != "" {
   314  					obj, err := Rcat(ctx, f, path.Join(remote, p.FileName()), p, time.Now(), nil)
   315  					if err != nil {
   316  						return nil, err
   317  					}
   318  					fs.Debugf(obj, "Upload Succeeded")
   319  				}
   320  			}
   321  		}
   322  		return nil, nil
   323  	case "cleanup":
   324  		return nil, CleanUp(ctx, f)
   325  	case "settier":
   326  		if !f.Features().SetTier {
   327  			return nil, fmt.Errorf("remote %s does not support settier", f.Name())
   328  		}
   329  		tier, err := in.GetString("tier")
   330  		if err != nil {
   331  			return nil, err
   332  		}
   333  		return nil, SetTier(ctx, f, tier)
   334  	case "settierfile":
   335  		if !f.Features().SetTier {
   336  			return nil, fmt.Errorf("remote %s does not support settier", f.Name())
   337  		}
   338  		tier, err := in.GetString("tier")
   339  		if err != nil {
   340  			return nil, err
   341  		}
   342  		o, err := f.NewObject(ctx, remote)
   343  		if err != nil {
   344  			return nil, err
   345  		}
   346  		return nil, SetTierFile(ctx, o, tier)
   347  	}
   348  	panic("unknown rcSingleCommand type")
   349  }
   350  
   351  func init() {
   352  	rc.Add(rc.Call{
   353  		Path:         "operations/size",
   354  		AuthRequired: true,
   355  		Fn:           rcSize,
   356  		Title:        "Count the number of bytes and files in remote",
   357  		Help: `This takes the following parameters:
   358  
   359  - fs - a remote name string e.g. "drive:path/to/dir"
   360  
   361  Returns:
   362  
   363  - count - number of files
   364  - bytes - number of bytes in those files
   365  
   366  See the [size](/commands/rclone_size/) command for more information on the above.
   367  `,
   368  	})
   369  }
   370  
   371  // Size a directory
   372  func rcSize(ctx context.Context, in rc.Params) (out rc.Params, err error) {
   373  	f, err := rc.GetFs(ctx, in)
   374  	if err != nil {
   375  		return nil, err
   376  	}
   377  	count, bytes, sizeless, err := Count(ctx, f)
   378  	if err != nil {
   379  		return nil, err
   380  	}
   381  	out = make(rc.Params)
   382  	out["count"] = count
   383  	out["bytes"] = bytes
   384  	out["sizeless"] = sizeless
   385  	return out, nil
   386  }
   387  
   388  func init() {
   389  	rc.Add(rc.Call{
   390  		Path:         "operations/publiclink",
   391  		AuthRequired: true,
   392  		Fn:           rcPublicLink,
   393  		Title:        "Create or retrieve a public link to the given file or folder.",
   394  		Help: `This takes the following parameters:
   395  
   396  - fs - a remote name string e.g. "drive:"
   397  - remote - a path within that remote e.g. "dir"
   398  - unlink - boolean - if set removes the link rather than adding it (optional)
   399  - expire - string - the expiry time of the link e.g. "1d" (optional)
   400  
   401  Returns:
   402  
   403  - url - URL of the resource
   404  
   405  See the [link](/commands/rclone_link/) command for more information on the above.
   406  `,
   407  	})
   408  }
   409  
   410  // Make a public link
   411  func rcPublicLink(ctx context.Context, in rc.Params) (out rc.Params, err error) {
   412  	f, remote, err := rc.GetFsAndRemote(ctx, in)
   413  	if err != nil {
   414  		return nil, err
   415  	}
   416  	unlink, _ := in.GetBool("unlink")
   417  	expire, err := in.GetDuration("expire")
   418  	if rc.IsErrParamNotFound(err) {
   419  		expire = time.Duration(fs.DurationOff)
   420  	} else if err != nil {
   421  		return nil, err
   422  	}
   423  	url, err := PublicLink(ctx, f, remote, fs.Duration(expire), unlink)
   424  	if err != nil {
   425  		return nil, err
   426  	}
   427  	out = make(rc.Params)
   428  	out["url"] = url
   429  	return out, nil
   430  }
   431  
   432  func init() {
   433  	rc.Add(rc.Call{
   434  		Path:  "operations/fsinfo",
   435  		Fn:    rcFsInfo,
   436  		Title: "Return information about the remote",
   437  		Help: `This takes the following parameters:
   438  
   439  - fs - a remote name string e.g. "drive:"
   440  
   441  This returns info about the remote passed in;
   442  
   443  ` + "```" + `
   444  {
   445          // optional features and whether they are available or not
   446          "Features": {
   447                  "About": true,
   448                  "BucketBased": false,
   449                  "BucketBasedRootOK": false,
   450                  "CanHaveEmptyDirectories": true,
   451                  "CaseInsensitive": false,
   452                  "ChangeNotify": false,
   453                  "CleanUp": false,
   454                  "Command": true,
   455                  "Copy": false,
   456                  "DirCacheFlush": false,
   457                  "DirMove": true,
   458                  "Disconnect": false,
   459                  "DuplicateFiles": false,
   460                  "GetTier": false,
   461                  "IsLocal": true,
   462                  "ListR": false,
   463                  "MergeDirs": false,
   464                  "MetadataInfo": true,
   465                  "Move": true,
   466                  "OpenWriterAt": true,
   467                  "PublicLink": false,
   468                  "Purge": true,
   469                  "PutStream": true,
   470                  "PutUnchecked": false,
   471                  "ReadMetadata": true,
   472                  "ReadMimeType": false,
   473                  "ServerSideAcrossConfigs": false,
   474                  "SetTier": false,
   475                  "SetWrapper": false,
   476                  "Shutdown": false,
   477                  "SlowHash": true,
   478                  "SlowModTime": false,
   479                  "UnWrap": false,
   480                  "UserInfo": false,
   481                  "UserMetadata": true,
   482                  "WrapFs": false,
   483                  "WriteMetadata": true,
   484                  "WriteMimeType": false
   485          },
   486          // Names of hashes available
   487          "Hashes": [
   488                  "md5",
   489                  "sha1",
   490                  "whirlpool",
   491                  "crc32",
   492                  "sha256",
   493                  "dropbox",
   494                  "mailru",
   495                  "quickxor"
   496          ],
   497          "Name": "local",        // Name as created
   498          "Precision": 1,         // Precision of timestamps in ns
   499          "Root": "/",            // Path as created
   500          "String": "Local file system at /", // how the remote will appear in logs
   501          // Information about the system metadata for this backend
   502          "MetadataInfo": {
   503                  "System": {
   504                          "atime": {
   505                                  "Help": "Time of last access",
   506                                  "Type": "RFC 3339",
   507                                  "Example": "2006-01-02T15:04:05.999999999Z07:00"
   508                          },
   509                          "btime": {
   510                                  "Help": "Time of file birth (creation)",
   511                                  "Type": "RFC 3339",
   512                                  "Example": "2006-01-02T15:04:05.999999999Z07:00"
   513                          },
   514                          "gid": {
   515                                  "Help": "Group ID of owner",
   516                                  "Type": "decimal number",
   517                                  "Example": "500"
   518                          },
   519                          "mode": {
   520                                  "Help": "File type and mode",
   521                                  "Type": "octal, unix style",
   522                                  "Example": "0100664"
   523                          },
   524                          "mtime": {
   525                                  "Help": "Time of last modification",
   526                                  "Type": "RFC 3339",
   527                                  "Example": "2006-01-02T15:04:05.999999999Z07:00"
   528                          },
   529                          "rdev": {
   530                                  "Help": "Device ID (if special file)",
   531                                  "Type": "hexadecimal",
   532                                  "Example": "1abc"
   533                          },
   534                          "uid": {
   535                                  "Help": "User ID of owner",
   536                                  "Type": "decimal number",
   537                                  "Example": "500"
   538                          }
   539                  },
   540                  "Help": "Textual help string\n"
   541          }
   542  }
   543  ` + "```" + `
   544  
   545  This command does not have a command line equivalent so use this instead:
   546  
   547      rclone rc --loopback operations/fsinfo fs=remote:
   548  
   549  `,
   550  	})
   551  }
   552  
   553  // Fsinfo the remote
   554  func rcFsInfo(ctx context.Context, in rc.Params) (out rc.Params, err error) {
   555  	f, err := rc.GetFs(ctx, in)
   556  	if err != nil {
   557  		return nil, err
   558  	}
   559  	info := GetFsInfo(f)
   560  	err = rc.Reshape(&out, info)
   561  	if err != nil {
   562  		return nil, fmt.Errorf("fsinfo Reshape failed: %w", err)
   563  	}
   564  	return out, nil
   565  }
   566  
   567  func init() {
   568  	rc.Add(rc.Call{
   569  		Path:         "backend/command",
   570  		AuthRequired: true,
   571  		Fn:           rcBackend,
   572  		Title:        "Runs a backend command.",
   573  		Help: `This takes the following parameters:
   574  
   575  - command - a string with the command name
   576  - fs - a remote name string e.g. "drive:"
   577  - arg - a list of arguments for the backend command
   578  - opt - a map of string to string of options
   579  
   580  Returns:
   581  
   582  - result - result from the backend command
   583  
   584  Example:
   585  
   586      rclone rc backend/command command=noop fs=. -o echo=yes -o blue -a path1 -a path2
   587  
   588  Returns
   589  
   590  ` + "```" + `
   591  {
   592  	"result": {
   593  		"arg": [
   594  			"path1",
   595  			"path2"
   596  		],
   597  		"name": "noop",
   598  		"opt": {
   599  			"blue": "",
   600  			"echo": "yes"
   601  		}
   602  	}
   603  }
   604  ` + "```" + `
   605  
   606  Note that this is the direct equivalent of using this "backend"
   607  command:
   608  
   609      rclone backend noop . -o echo=yes -o blue path1 path2
   610  
   611  Note that arguments must be preceded by the "-a" flag
   612  
   613  See the [backend](/commands/rclone_backend/) command for more information.
   614  `,
   615  	})
   616  }
   617  
   618  // Make a public link
   619  func rcBackend(ctx context.Context, in rc.Params) (out rc.Params, err error) {
   620  	f, err := rc.GetFs(ctx, in)
   621  	if err != nil {
   622  		return nil, err
   623  	}
   624  	doCommand := f.Features().Command
   625  	if doCommand == nil {
   626  		return nil, fmt.Errorf("%v: doesn't support backend commands", f)
   627  	}
   628  	command, err := in.GetString("command")
   629  	if err != nil {
   630  		return nil, err
   631  	}
   632  	var opt = map[string]string{}
   633  	err = in.GetStructMissingOK("opt", &opt)
   634  	if err != nil {
   635  		return nil, err
   636  	}
   637  	var arg = []string{}
   638  	err = in.GetStructMissingOK("arg", &arg)
   639  	if err != nil {
   640  		return nil, err
   641  	}
   642  	result, err := doCommand(ctx, command, arg, opt)
   643  	if err != nil {
   644  		return nil, fmt.Errorf("command %q failed: %w", command, err)
   645  
   646  	}
   647  	out = make(rc.Params)
   648  	out["result"] = result
   649  	return out, nil
   650  }
   651  
   652  // This should really be in fs/rc/internal.go but can't go there due
   653  // to a circular dependency on config.
   654  func init() {
   655  	rc.Add(rc.Call{
   656  		Path:  "core/du",
   657  		Fn:    rcDu,
   658  		Title: "Returns disk usage of a locally attached disk.",
   659  		Help: `
   660  This returns the disk usage for the local directory passed in as dir.
   661  
   662  If the directory is not passed in, it defaults to the directory
   663  pointed to by --cache-dir.
   664  
   665  - dir - string (optional)
   666  
   667  Returns:
   668  
   669  ` + "```" + `
   670  {
   671  	"dir": "/",
   672  	"info": {
   673  		"Available": 361769115648,
   674  		"Free": 361785892864,
   675  		"Total": 982141468672
   676  	}
   677  }
   678  ` + "```" + `
   679  `,
   680  	})
   681  }
   682  
   683  // Terminates app
   684  func rcDu(ctx context.Context, in rc.Params) (out rc.Params, err error) {
   685  	dir, err := in.GetString("dir")
   686  	if rc.IsErrParamNotFound(err) {
   687  		dir = config.GetCacheDir()
   688  
   689  	} else if err != nil {
   690  		return nil, err
   691  	}
   692  	info, err := diskusage.New(dir)
   693  	if err != nil {
   694  		return nil, err
   695  	}
   696  	out = rc.Params{
   697  		"dir":  dir,
   698  		"info": info,
   699  	}
   700  	return out, nil
   701  }
   702  
   703  func init() {
   704  	rc.Add(rc.Call{
   705  		Path:         "operations/check",
   706  		AuthRequired: true,
   707  		Fn:           rcCheck,
   708  		Title:        "check the source and destination are the same",
   709  		Help: `Checks the files in the source and destination match.  It compares
   710  sizes and hashes and logs a report of files that don't
   711  match.  It doesn't alter the source or destination.
   712  
   713  This takes the following parameters:
   714  
   715  - srcFs - a remote name string e.g. "drive:" for the source, "/" for local filesystem
   716  - dstFs - a remote name string e.g. "drive2:" for the destination, "/" for local filesystem
   717  - download - check by downloading rather than with hash
   718  - checkFileHash - treat checkFileFs:checkFileRemote as a SUM file with hashes of given type
   719  - checkFileFs - treat checkFileFs:checkFileRemote as a SUM file with hashes of given type
   720  - checkFileRemote - treat checkFileFs:checkFileRemote as a SUM file with hashes of given type
   721  - oneWay -  check one way only, source files must exist on remote
   722  - combined - make a combined report of changes (default false)
   723  - missingOnSrc - report all files missing from the source (default true)
   724  - missingOnDst - report all files missing from the destination (default true)
   725  - match - report all matching files (default false)
   726  - differ - report all non-matching files (default true)
   727  - error - report all files with errors (hashing or reading) (default true)
   728  
   729  If you supply the download flag, it will download the data from
   730  both remotes and check them against each other on the fly.  This can
   731  be useful for remotes that don't support hashes or if you really want
   732  to check all the data.
   733  
   734  If you supply the size-only global flag, it will only compare the sizes not
   735  the hashes as well.  Use this for a quick check.
   736  
   737  If you supply the checkFileHash option with a valid hash name, the
   738  checkFileFs:checkFileRemote must point to a text file in the SUM
   739  format. This treats the checksum file as the source and dstFs as the
   740  destination. Note that srcFs is not used and should not be supplied in
   741  this case.
   742  
   743  Returns:
   744  
   745  - success - true if no error, false otherwise
   746  - status - textual summary of check, OK or text string
   747  - hashType - hash used in check, may be missing
   748  - combined - array of strings of combined report of changes
   749  - missingOnSrc - array of strings of all files missing from the source
   750  - missingOnDst - array of strings of all files missing from the destination
   751  - match - array of strings of all matching files
   752  - differ - array of strings of all non-matching files
   753  - error - array of strings of all files with errors (hashing or reading)
   754  
   755  `,
   756  	})
   757  }
   758  
   759  // Writer which writes into the slice provided
   760  type stringWriter struct {
   761  	out *[]string
   762  }
   763  
   764  // Write writes len(p) bytes from p to the underlying data stream. It returns
   765  // the number of bytes written from p (0 <= n <= len(p)) and any error
   766  // encountered that caused the write to stop early. Write must return a non-nil
   767  // error if it returns n < len(p). Write must not modify the slice data,
   768  // even temporarily.
   769  //
   770  // Implementations must not retain p.
   771  func (s stringWriter) Write(p []byte) (n int, err error) {
   772  	result := string(p)
   773  	result = strings.TrimSuffix(result, "\n")
   774  	*s.out = append(*s.out, result)
   775  	return len(p), nil
   776  }
   777  
   778  // Check two directories
   779  func rcCheck(ctx context.Context, in rc.Params) (out rc.Params, err error) {
   780  	srcFs, err := rc.GetFsNamed(ctx, in, "srcFs")
   781  	if err != nil && !rc.IsErrParamNotFound(err) {
   782  		return nil, err
   783  	}
   784  
   785  	dstFs, err := rc.GetFsNamed(ctx, in, "dstFs")
   786  	if err != nil {
   787  		return nil, err
   788  	}
   789  
   790  	checkFileFs, checkFileRemote, err := rc.GetFsAndRemoteNamed(ctx, in, "checkFileFs", "checkFileRemote")
   791  	if err != nil && !rc.IsErrParamNotFound(err) {
   792  		return nil, err
   793  	}
   794  
   795  	checkFileHash, err := in.GetString("checkFileHash")
   796  	if err != nil && !rc.IsErrParamNotFound(err) {
   797  		return nil, err
   798  	}
   799  
   800  	checkFileSet := 0
   801  	if checkFileHash != "" {
   802  		checkFileSet++
   803  	}
   804  	if checkFileFs != nil {
   805  		checkFileSet++
   806  	}
   807  	if checkFileRemote != "" {
   808  		checkFileSet++
   809  	}
   810  	if checkFileSet > 0 && checkFileSet < 3 {
   811  		return nil, fmt.Errorf("need all of checkFileFs, checkFileRemote, checkFileHash to be set together")
   812  	}
   813  
   814  	var checkFileHashType hash.Type
   815  	if checkFileHash != "" {
   816  		if err := checkFileHashType.Set(checkFileHash); err != nil {
   817  			return nil, err
   818  		}
   819  		if srcFs != nil {
   820  			return nil, rc.NewErrParamInvalid(errors.New("only supply dstFs when using checkFileHash"))
   821  		}
   822  	} else {
   823  		if srcFs == nil {
   824  			return nil, rc.NewErrParamInvalid(errors.New("need srcFs parameter when not using checkFileHash"))
   825  		}
   826  	}
   827  
   828  	oneway, _ := in.GetBool("oneway")
   829  	download, _ := in.GetBool("download")
   830  
   831  	opt := &CheckOpt{
   832  		Fsrc:   srcFs,
   833  		Fdst:   dstFs,
   834  		OneWay: oneway,
   835  	}
   836  
   837  	out = rc.Params{}
   838  
   839  	getOutput := func(name string, Default bool) io.Writer {
   840  		active, err := in.GetBool(name)
   841  		if err != nil {
   842  			active = Default
   843  		}
   844  		if !active {
   845  			return nil
   846  		}
   847  		result := []string{}
   848  		out[name] = &result
   849  		return stringWriter{&result}
   850  	}
   851  
   852  	opt.Combined = getOutput("combined", false)
   853  	opt.MissingOnSrc = getOutput("missingOnSrc", true)
   854  	opt.MissingOnDst = getOutput("missingOnDst", true)
   855  	opt.Match = getOutput("match", false)
   856  	opt.Differ = getOutput("differ", true)
   857  	opt.Error = getOutput("error", true)
   858  
   859  	if checkFileHash != "" {
   860  		out["hashType"] = checkFileHashType.String()
   861  		err = CheckSum(ctx, dstFs, checkFileFs, checkFileRemote, checkFileHashType, opt, download)
   862  	} else {
   863  		if download {
   864  			err = CheckDownload(ctx, opt)
   865  		} else {
   866  			out["hashType"] = srcFs.Hashes().Overlap(dstFs.Hashes()).GetOne().String()
   867  			err = Check(ctx, opt)
   868  		}
   869  	}
   870  	if err != nil {
   871  		out["status"] = err.Error()
   872  		out["success"] = false
   873  	} else {
   874  		out["status"] = "OK"
   875  		out["success"] = true
   876  	}
   877  	return out, nil
   878  }
   879  
   880  func init() {
   881  	rc.Add(rc.Call{
   882  		Path:         "operations/hashsum",
   883  		AuthRequired: true,
   884  		Fn:           rcHashsum,
   885  		Title:        "Produces a hashsum file for all the objects in the path.",
   886  		Help: `Produces a hash file for all the objects in the path using the hash
   887  named.  The output is in the same format as the standard
   888  md5sum/sha1sum tool.
   889  
   890  This takes the following parameters:
   891  
   892  - fs - a remote name string e.g. "drive:" for the source, "/" for local filesystem
   893      - this can point to a file and just that file will be returned in the listing.
   894  - hashType - type of hash to be used
   895  - download - check by downloading rather than with hash (boolean)
   896  - base64 - output the hashes in base64 rather than hex (boolean)
   897  
   898  If you supply the download flag, it will download the data from the
   899  remote and create the hash on the fly. This can be useful for remotes
   900  that don't support the given hash or if you really want to check all
   901  the data.
   902  
   903  Note that if you wish to supply a checkfile to check hashes against
   904  the current files then you should use operations/check instead of
   905  operations/hashsum.
   906  
   907  Returns:
   908  
   909  - hashsum - array of strings of the hashes
   910  - hashType - type of hash used
   911  
   912  Example:
   913  
   914      $ rclone rc --loopback operations/hashsum fs=bin hashType=MD5 download=true base64=true
   915      {
   916          "hashType": "md5",
   917          "hashsum": [
   918              "WTSVLpuiXyJO_kGzJerRLg==  backend-versions.sh",
   919              "v1b_OlWCJO9LtNq3EIKkNQ==  bisect-go-rclone.sh",
   920              "VHbmHzHh4taXzgag8BAIKQ==  bisect-rclone.sh",
   921          ]
   922      }
   923  
   924  See the [hashsum](/commands/rclone_hashsum/) command for more information on the above.
   925  `,
   926  	})
   927  }
   928  
   929  // Hashsum a directory
   930  func rcHashsum(ctx context.Context, in rc.Params) (out rc.Params, err error) {
   931  	ctx, f, err := rc.GetFsNamedFileOK(ctx, in, "fs")
   932  	if err != nil {
   933  		return nil, err
   934  	}
   935  
   936  	download, _ := in.GetBool("download")
   937  	base64, _ := in.GetBool("base64")
   938  	hashType, err := in.GetString("hashType")
   939  	if err != nil {
   940  		return nil, fmt.Errorf("%s\n%w", hash.HelpString(0), err)
   941  	}
   942  	var ht hash.Type
   943  	err = ht.Set(hashType)
   944  	if err != nil {
   945  		return nil, fmt.Errorf("%s\n%w", hash.HelpString(0), err)
   946  	}
   947  
   948  	hashes := []string{}
   949  	err = HashLister(ctx, ht, base64, download, f, stringWriter{&hashes})
   950  	out = rc.Params{
   951  		"hashType": ht.String(),
   952  		"hashsum":  hashes,
   953  	}
   954  	return out, err
   955  }