github.com/rclone/rclone@v1.66.1-0.20240517100346-7b89735ae726/backend/webdav/webdav.go (about)

     1  // Package webdav provides an interface to the Webdav
     2  // object storage system.
     3  package webdav
     4  
     5  // SetModTime might be possible
     6  // https://stackoverflow.com/questions/3579608/webdav-can-a-client-modify-the-mtime-of-a-file
     7  // ...support for a PROPSET to lastmodified (mind the missing get) which does the utime() call might be an option.
     8  // For example the ownCloud WebDAV server does it that way.
     9  
    10  import (
    11  	"bytes"
    12  	"context"
    13  	"crypto/tls"
    14  	"encoding/xml"
    15  	"errors"
    16  	"fmt"
    17  	"io"
    18  	"net/http"
    19  	"net/url"
    20  	"os/exec"
    21  	"path"
    22  	"regexp"
    23  	"strconv"
    24  	"strings"
    25  	"sync"
    26  	"time"
    27  
    28  	"github.com/rclone/rclone/backend/webdav/api"
    29  	"github.com/rclone/rclone/backend/webdav/odrvcookie"
    30  	"github.com/rclone/rclone/fs"
    31  	"github.com/rclone/rclone/fs/config"
    32  	"github.com/rclone/rclone/fs/config/configmap"
    33  	"github.com/rclone/rclone/fs/config/configstruct"
    34  	"github.com/rclone/rclone/fs/config/obscure"
    35  	"github.com/rclone/rclone/fs/fserrors"
    36  	"github.com/rclone/rclone/fs/fshttp"
    37  	"github.com/rclone/rclone/fs/hash"
    38  	"github.com/rclone/rclone/lib/encoder"
    39  	"github.com/rclone/rclone/lib/pacer"
    40  	"github.com/rclone/rclone/lib/rest"
    41  
    42  	ntlmssp "github.com/Azure/go-ntlmssp"
    43  )
    44  
    45  const (
    46  	minSleep      = fs.Duration(10 * time.Millisecond)
    47  	maxSleep      = 2 * time.Second
    48  	decayConstant = 2   // bigger for slower decay, exponential
    49  	defaultDepth  = "1" // depth for PROPFIND
    50  )
    51  
    52  const defaultEncodingSharepointNTLM = (encoder.EncodeWin |
    53  	encoder.EncodeHashPercent | // required by IIS/8.5 in contrast with onedrive which doesn't need it
    54  	(encoder.Display &^ encoder.EncodeDot) | // test with IIS/8.5 shows that EncodeDot is not needed
    55  	encoder.EncodeBackSlash |
    56  	encoder.EncodeLeftSpace |
    57  	encoder.EncodeLeftTilde |
    58  	encoder.EncodeRightPeriod |
    59  	encoder.EncodeRightSpace |
    60  	encoder.EncodeInvalidUtf8)
    61  
    62  // Register with Fs
    63  func init() {
    64  	configEncodingHelp := fmt.Sprintf(
    65  		"%s\n\nDefault encoding is %s for sharepoint-ntlm or identity otherwise.",
    66  		config.ConfigEncodingHelp, defaultEncodingSharepointNTLM)
    67  
    68  	fs.Register(&fs.RegInfo{
    69  		Name:        "webdav",
    70  		Description: "WebDAV",
    71  		NewFs:       NewFs,
    72  		Options: []fs.Option{{
    73  			Name:     "url",
    74  			Help:     "URL of http host to connect to.\n\nE.g. https://example.com.",
    75  			Required: true,
    76  		}, {
    77  			Name: "vendor",
    78  			Help: "Name of the WebDAV site/service/software you are using.",
    79  			Examples: []fs.OptionExample{{
    80  				Value: "fastmail",
    81  				Help:  "Fastmail Files",
    82  			}, {
    83  				Value: "nextcloud",
    84  				Help:  "Nextcloud",
    85  			}, {
    86  				Value: "owncloud",
    87  				Help:  "Owncloud",
    88  			}, {
    89  				Value: "sharepoint",
    90  				Help:  "Sharepoint Online, authenticated by Microsoft account",
    91  			}, {
    92  				Value: "sharepoint-ntlm",
    93  				Help:  "Sharepoint with NTLM authentication, usually self-hosted or on-premises",
    94  			}, {
    95  				Value: "rclone",
    96  				Help:  "rclone WebDAV server to serve a remote over HTTP via the WebDAV protocol",
    97  			}, {
    98  				Value: "other",
    99  				Help:  "Other site/service or software",
   100  			}},
   101  		}, {
   102  			Name:      "user",
   103  			Help:      "User name.\n\nIn case NTLM authentication is used, the username should be in the format 'Domain\\User'.",
   104  			Sensitive: true,
   105  		}, {
   106  			Name:       "pass",
   107  			Help:       "Password.",
   108  			IsPassword: true,
   109  		}, {
   110  			Name:      "bearer_token",
   111  			Help:      "Bearer token instead of user/pass (e.g. a Macaroon).",
   112  			Sensitive: true,
   113  		}, {
   114  			Name:     "bearer_token_command",
   115  			Help:     "Command to run to get a bearer token.",
   116  			Advanced: true,
   117  		}, {
   118  			Name:     config.ConfigEncoding,
   119  			Help:     configEncodingHelp,
   120  			Advanced: true,
   121  		}, {
   122  			Name: "headers",
   123  			Help: `Set HTTP headers for all transactions.
   124  
   125  Use this to set additional HTTP headers for all transactions
   126  
   127  The input format is comma separated list of key,value pairs.  Standard
   128  [CSV encoding](https://godoc.org/encoding/csv) may be used.
   129  
   130  For example, to set a Cookie use 'Cookie,name=value', or '"Cookie","name=value"'.
   131  
   132  You can set multiple headers, e.g. '"Cookie","name=value","Authorization","xxx"'.
   133  `,
   134  			Default:  fs.CommaSepList{},
   135  			Advanced: true,
   136  		}, {
   137  			Name:     "pacer_min_sleep",
   138  			Help:     "Minimum time to sleep between API calls.",
   139  			Default:  minSleep,
   140  			Advanced: true,
   141  		}, {
   142  			Name: "nextcloud_chunk_size",
   143  			Help: `Nextcloud upload chunk size.
   144  
   145  We recommend configuring your NextCloud instance to increase the max chunk size to 1 GB for better upload performances.
   146  See https://docs.nextcloud.com/server/latest/admin_manual/configuration_files/big_file_upload_configuration.html#adjust-chunk-size-on-nextcloud-side
   147  
   148  Set to 0 to disable chunked uploading.
   149  `,
   150  			Advanced: true,
   151  			Default:  10 * fs.Mebi, // Default NextCloud `max_chunk_size` is `10 MiB`. See https://github.com/nextcloud/server/blob/0447b53bda9fe95ea0cbed765aa332584605d652/apps/files/lib/App.php#L57
   152  		}, {
   153  			Name:     "owncloud_exclude_shares",
   154  			Help:     "Exclude ownCloud shares",
   155  			Advanced: true,
   156  			Default:  false,
   157  		}, {
   158  			Name:     "owncloud_exclude_mounts",
   159  			Help:     "Exclude ownCloud mounted storages",
   160  			Advanced: true,
   161  			Default:  false,
   162  		}},
   163  	})
   164  }
   165  
   166  // Options defines the configuration for this backend
   167  type Options struct {
   168  	URL                string               `config:"url"`
   169  	Vendor             string               `config:"vendor"`
   170  	User               string               `config:"user"`
   171  	Pass               string               `config:"pass"`
   172  	BearerToken        string               `config:"bearer_token"`
   173  	BearerTokenCommand string               `config:"bearer_token_command"`
   174  	Enc                encoder.MultiEncoder `config:"encoding"`
   175  	Headers            fs.CommaSepList      `config:"headers"`
   176  	PacerMinSleep      fs.Duration          `config:"pacer_min_sleep"`
   177  	ChunkSize          fs.SizeSuffix        `config:"nextcloud_chunk_size"`
   178  	ExcludeShares      bool                 `config:"owncloud_exclude_shares"`
   179  	ExcludeMounts      bool                 `config:"owncloud_exclude_mounts"`
   180  }
   181  
   182  // Fs represents a remote webdav
   183  type Fs struct {
   184  	name               string        // name of this remote
   185  	root               string        // the path we are working on
   186  	opt                Options       // parsed options
   187  	features           *fs.Features  // optional features
   188  	endpoint           *url.URL      // URL of the host
   189  	endpointURL        string        // endpoint as a string
   190  	srv                *rest.Client  // the connection to the server
   191  	pacer              *fs.Pacer     // pacer for API calls
   192  	precision          time.Duration // mod time precision
   193  	canStream          bool          // set if can stream
   194  	useOCMtime         bool          // set if can use X-OC-Mtime
   195  	propsetMtime       bool          // set if can use propset
   196  	retryWithZeroDepth bool          // some vendors (sharepoint) won't list files when Depth is 1 (our default)
   197  	checkBeforePurge   bool          // enables extra check that directory to purge really exists
   198  	hasOCMD5           bool          // set if can use owncloud style checksums for MD5
   199  	hasOCSHA1          bool          // set if can use owncloud style checksums for SHA1
   200  	hasMESHA1          bool          // set if can use fastmail style checksums for SHA1
   201  	ntlmAuthMu         sync.Mutex    // mutex to serialize NTLM auth roundtrips
   202  	chunksUploadURL    string        // upload URL for nextcloud chunked
   203  	canChunk           bool          // set if nextcloud and nextcloud_chunk_size is set
   204  }
   205  
   206  // Object describes a webdav object
   207  //
   208  // Will definitely have info but maybe not meta
   209  type Object struct {
   210  	fs          *Fs       // what this object is part of
   211  	remote      string    // The remote path
   212  	hasMetaData bool      // whether info below has been set
   213  	size        int64     // size of the object
   214  	modTime     time.Time // modification time of the object
   215  	sha1        string    // SHA-1 of the object content if known
   216  	md5         string    // MD5 of the object content if known
   217  }
   218  
   219  // ------------------------------------------------------------
   220  
   221  // Name of the remote (as passed into NewFs)
   222  func (f *Fs) Name() string {
   223  	return f.name
   224  }
   225  
   226  // Root of the remote (as passed into NewFs)
   227  func (f *Fs) Root() string {
   228  	return f.root
   229  }
   230  
   231  // String converts this Fs to a string
   232  func (f *Fs) String() string {
   233  	return fmt.Sprintf("webdav root '%s'", f.root)
   234  }
   235  
   236  // Features returns the optional features of this Fs
   237  func (f *Fs) Features() *fs.Features {
   238  	return f.features
   239  }
   240  
   241  // retryErrorCodes is a slice of error codes that we will retry
   242  var retryErrorCodes = []int{
   243  	423, // Locked
   244  	429, // Too Many Requests.
   245  	500, // Internal Server Error
   246  	502, // Bad Gateway
   247  	503, // Service Unavailable
   248  	504, // Gateway Timeout
   249  	509, // Bandwidth Limit Exceeded
   250  }
   251  
   252  // shouldRetry returns a boolean as to whether this resp and err
   253  // deserve to be retried.  It returns the err as a convenience
   254  func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
   255  	if fserrors.ContextError(ctx, &err) {
   256  		return false, err
   257  	}
   258  	// If we have a bearer token command and it has expired then refresh it
   259  	if f.opt.BearerTokenCommand != "" && resp != nil && resp.StatusCode == 401 {
   260  		fs.Debugf(f, "Bearer token expired: %v", err)
   261  		authErr := f.fetchAndSetBearerToken()
   262  		if authErr != nil {
   263  			err = authErr
   264  		}
   265  		return true, err
   266  	}
   267  	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
   268  }
   269  
   270  // safeRoundTripper is a wrapper for http.RoundTripper that serializes
   271  // http roundtrips. NTLM authentication sequence can involve up to four
   272  // rounds of negotiations and might fail due to concurrency.
   273  // This wrapper allows to use ntlmssp.Negotiator safely with goroutines.
   274  type safeRoundTripper struct {
   275  	fs *Fs
   276  	rt http.RoundTripper
   277  }
   278  
   279  // RoundTrip guards wrapped RoundTripper by a mutex.
   280  func (srt *safeRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
   281  	srt.fs.ntlmAuthMu.Lock()
   282  	defer srt.fs.ntlmAuthMu.Unlock()
   283  	return srt.rt.RoundTrip(req)
   284  }
   285  
   286  // itemIsDir returns true if the item is a directory
   287  //
   288  // When a client sees a resourcetype it doesn't recognize it should
   289  // assume it is a regular non-collection resource.  [WebDav book by
   290  // Lisa Dusseault ch 7.5.8 p170]
   291  func itemIsDir(item *api.Response) bool {
   292  	if t := item.Props.Type; t != nil {
   293  		if t.Space == "DAV:" && t.Local == "collection" {
   294  			return true
   295  		}
   296  		fs.Debugf(nil, "Unknown resource type %q/%q on %q", t.Space, t.Local, item.Props.Name)
   297  	}
   298  	// the iscollection prop is a Microsoft extension, but if present it is a reliable indicator
   299  	// if the above check failed - see #2716. This can be an integer or a boolean - see #2964
   300  	if t := item.Props.IsCollection; t != nil {
   301  		switch x := strings.ToLower(*t); x {
   302  		case "0", "false":
   303  			return false
   304  		case "1", "true":
   305  			return true
   306  		default:
   307  			fs.Debugf(nil, "Unknown value %q for IsCollection", x)
   308  		}
   309  	}
   310  	return false
   311  }
   312  
   313  // readMetaDataForPath reads the metadata from the path
   314  func (f *Fs) readMetaDataForPath(ctx context.Context, path string, depth string) (info *api.Prop, err error) {
   315  	// FIXME how do we read back additional properties?
   316  	opts := rest.Opts{
   317  		Method: "PROPFIND",
   318  		Path:   f.filePath(path),
   319  		ExtraHeaders: map[string]string{
   320  			"Depth": depth,
   321  		},
   322  		NoRedirect: true,
   323  	}
   324  	if f.hasOCMD5 || f.hasOCSHA1 {
   325  		opts.Body = bytes.NewBuffer(owncloudProps)
   326  	}
   327  	var result api.Multistatus
   328  	var resp *http.Response
   329  	err = f.pacer.Call(func() (bool, error) {
   330  		resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
   331  		return f.shouldRetry(ctx, resp, err)
   332  	})
   333  	if apiErr, ok := err.(*api.Error); ok {
   334  		// does not exist
   335  		switch apiErr.StatusCode {
   336  		case http.StatusNotFound:
   337  			if f.retryWithZeroDepth && depth != "0" {
   338  				return f.readMetaDataForPath(ctx, path, "0")
   339  			}
   340  			return nil, fs.ErrorObjectNotFound
   341  		case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther:
   342  			// Some sort of redirect - go doesn't deal with these properly (it resets
   343  			// the method to GET).  However we can assume that if it was redirected the
   344  			// object was not found.
   345  			return nil, fs.ErrorObjectNotFound
   346  		}
   347  	}
   348  	if err != nil {
   349  		return nil, fmt.Errorf("read metadata failed: %w", err)
   350  	}
   351  	if len(result.Responses) < 1 {
   352  		return nil, fs.ErrorObjectNotFound
   353  	}
   354  	item := result.Responses[0]
   355  	if !item.Props.StatusOK() {
   356  		return nil, fs.ErrorObjectNotFound
   357  	}
   358  	if itemIsDir(&item) {
   359  		return nil, fs.ErrorIsDir
   360  	}
   361  	return &item.Props, nil
   362  }
   363  
   364  // errorHandler parses a non 2xx error response into an error
   365  func errorHandler(resp *http.Response) error {
   366  	body, err := rest.ReadBody(resp)
   367  	if err != nil {
   368  		return fmt.Errorf("error when trying to read error from body: %w", err)
   369  	}
   370  	// Decode error response
   371  	errResponse := new(api.Error)
   372  	err = xml.Unmarshal(body, &errResponse)
   373  	if err != nil {
   374  		// set the Message to be the body if can't parse the XML
   375  		errResponse.Message = strings.TrimSpace(string(body))
   376  	}
   377  	errResponse.Status = resp.Status
   378  	errResponse.StatusCode = resp.StatusCode
   379  	return errResponse
   380  }
   381  
   382  // addSlash makes sure s is terminated with a / if non empty
   383  func addSlash(s string) string {
   384  	if s != "" && !strings.HasSuffix(s, "/") {
   385  		s += "/"
   386  	}
   387  	return s
   388  }
   389  
   390  // filePath returns a file path (f.root, file)
   391  func (f *Fs) filePath(file string) string {
   392  	subPath := path.Join(f.root, file)
   393  	if f.opt.Enc != encoder.EncodeZero {
   394  		subPath = f.opt.Enc.FromStandardPath(subPath)
   395  	}
   396  	return rest.URLPathEscape(subPath)
   397  }
   398  
   399  // dirPath returns a directory path (f.root, dir)
   400  func (f *Fs) dirPath(dir string) string {
   401  	return addSlash(f.filePath(dir))
   402  }
   403  
   404  // filePath returns a file path (f.root, remote)
   405  func (o *Object) filePath() string {
   406  	return o.fs.filePath(o.remote)
   407  }
   408  
   409  // NewFs constructs an Fs from the path, container:path
   410  func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
   411  	// Parse config into Options struct
   412  	opt := new(Options)
   413  	err := configstruct.Set(m, opt)
   414  	if err != nil {
   415  		return nil, err
   416  	}
   417  
   418  	if len(opt.Headers)%2 != 0 {
   419  		return nil, errors.New("odd number of headers supplied")
   420  	}
   421  	fs.Debugf(nil, "found headers: %v", opt.Headers)
   422  
   423  	rootIsDir := strings.HasSuffix(root, "/")
   424  	root = strings.Trim(root, "/")
   425  
   426  	if !strings.HasSuffix(opt.URL, "/") {
   427  		opt.URL += "/"
   428  	}
   429  	if opt.Pass != "" {
   430  		var err error
   431  		opt.Pass, err = obscure.Reveal(opt.Pass)
   432  		if err != nil {
   433  			return nil, fmt.Errorf("couldn't decrypt password: %w", err)
   434  		}
   435  	}
   436  	if opt.Vendor == "" {
   437  		opt.Vendor = "other"
   438  	}
   439  	root = strings.Trim(root, "/")
   440  
   441  	if opt.Enc == encoder.EncodeZero && opt.Vendor == "sharepoint-ntlm" {
   442  		opt.Enc = defaultEncodingSharepointNTLM
   443  	}
   444  
   445  	// Parse the endpoint
   446  	u, err := url.Parse(opt.URL)
   447  	if err != nil {
   448  		return nil, err
   449  	}
   450  
   451  	f := &Fs{
   452  		name:        name,
   453  		root:        root,
   454  		opt:         *opt,
   455  		endpoint:    u,
   456  		endpointURL: u.String(),
   457  		pacer:       fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(opt.PacerMinSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
   458  		precision:   fs.ModTimeNotSupported,
   459  	}
   460  
   461  	client := fshttp.NewClient(ctx)
   462  	if opt.Vendor == "sharepoint-ntlm" {
   463  		// Disable transparent HTTP/2 support as per https://golang.org/pkg/net/http/ ,
   464  		// otherwise any connection to IIS 10.0 fails with 'stream error: stream ID 39; HTTP_1_1_REQUIRED'
   465  		// https://docs.microsoft.com/en-us/iis/get-started/whats-new-in-iis-10/http2-on-iis says:
   466  		// 'Windows authentication (NTLM/Kerberos/Negotiate) is not supported with HTTP/2.'
   467  		t := fshttp.NewTransportCustom(ctx, func(t *http.Transport) {
   468  			t.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{}
   469  		})
   470  
   471  		// Add NTLM layer
   472  		client.Transport = &safeRoundTripper{
   473  			fs: f,
   474  			rt: ntlmssp.Negotiator{RoundTripper: t},
   475  		}
   476  	}
   477  	f.srv = rest.NewClient(client).SetRoot(u.String())
   478  
   479  	f.features = (&fs.Features{
   480  		CanHaveEmptyDirectories: true,
   481  	}).Fill(ctx, f)
   482  	if opt.User != "" || opt.Pass != "" {
   483  		f.srv.SetUserPass(opt.User, opt.Pass)
   484  	} else if opt.BearerToken != "" {
   485  		f.setBearerToken(opt.BearerToken)
   486  	} else if f.opt.BearerTokenCommand != "" {
   487  		err = f.fetchAndSetBearerToken()
   488  		if err != nil {
   489  			return nil, err
   490  		}
   491  	}
   492  	if opt.Headers != nil {
   493  		f.addHeaders(opt.Headers)
   494  	}
   495  	f.srv.SetErrorHandler(errorHandler)
   496  	err = f.setQuirks(ctx, opt.Vendor)
   497  	if err != nil {
   498  		return nil, err
   499  	}
   500  	if !f.findHeader(opt.Headers, "Referer") {
   501  		f.srv.SetHeader("Referer", u.String())
   502  	}
   503  
   504  	if root != "" && !rootIsDir {
   505  		// Check to see if the root actually an existing file
   506  		remote := path.Base(root)
   507  		f.root = path.Dir(root)
   508  		if f.root == "." {
   509  			f.root = ""
   510  		}
   511  		_, err := f.NewObject(ctx, remote)
   512  		if err != nil {
   513  			if errors.Is(err, fs.ErrorObjectNotFound) || errors.Is(err, fs.ErrorIsDir) {
   514  				// File doesn't exist so return old f
   515  				f.root = root
   516  				return f, nil
   517  			}
   518  			return nil, err
   519  		}
   520  		// return an error with an fs which points to the parent
   521  		return f, fs.ErrorIsFile
   522  	}
   523  	return f, nil
   524  }
   525  
   526  // sets the BearerToken up
   527  func (f *Fs) setBearerToken(token string) {
   528  	f.opt.BearerToken = token
   529  	f.srv.SetHeader("Authorization", "Bearer "+token)
   530  }
   531  
   532  // fetch the bearer token using the command
   533  func (f *Fs) fetchBearerToken(cmd string) (string, error) {
   534  	var (
   535  		args   = strings.Split(cmd, " ")
   536  		stdout bytes.Buffer
   537  		stderr bytes.Buffer
   538  		c      = exec.Command(args[0], args[1:]...)
   539  	)
   540  	c.Stdout = &stdout
   541  	c.Stderr = &stderr
   542  	var (
   543  		err          = c.Run()
   544  		stdoutString = strings.TrimSpace(stdout.String())
   545  		stderrString = strings.TrimSpace(stderr.String())
   546  	)
   547  	if err != nil {
   548  		if stderrString == "" {
   549  			stderrString = stdoutString
   550  		}
   551  		return "", fmt.Errorf("failed to get bearer token using %q: %s: %w", f.opt.BearerTokenCommand, stderrString, err)
   552  	}
   553  	return stdoutString, nil
   554  }
   555  
   556  // Adds the configured headers to the request if any
   557  func (f *Fs) addHeaders(headers fs.CommaSepList) {
   558  	for i := 0; i < len(headers); i += 2 {
   559  		key := f.opt.Headers[i]
   560  		value := f.opt.Headers[i+1]
   561  		f.srv.SetHeader(key, value)
   562  	}
   563  }
   564  
   565  // Returns true if the header was configured
   566  func (f *Fs) findHeader(headers fs.CommaSepList, find string) bool {
   567  	for i := 0; i < len(headers); i += 2 {
   568  		key := f.opt.Headers[i]
   569  		if strings.EqualFold(key, find) {
   570  			return true
   571  		}
   572  	}
   573  	return false
   574  }
   575  
   576  // fetch the bearer token and set it if successful
   577  func (f *Fs) fetchAndSetBearerToken() error {
   578  	if f.opt.BearerTokenCommand == "" {
   579  		return nil
   580  	}
   581  	token, err := f.fetchBearerToken(f.opt.BearerTokenCommand)
   582  	if err != nil {
   583  		return err
   584  	}
   585  	f.setBearerToken(token)
   586  	return nil
   587  }
   588  
   589  // The WebDAV url can optionally be suffixed with a path. This suffix needs to be ignored for determining the temporary upload directory of chunks.
   590  var nextCloudURLRegex = regexp.MustCompile(`^(.*)/dav/files/([^/]+)`)
   591  
   592  // setQuirks adjusts the Fs for the vendor passed in
   593  func (f *Fs) setQuirks(ctx context.Context, vendor string) error {
   594  	switch vendor {
   595  	case "fastmail":
   596  		f.canStream = true
   597  		f.precision = time.Second
   598  		f.useOCMtime = true
   599  		f.hasMESHA1 = true
   600  	case "owncloud":
   601  		f.canStream = true
   602  		f.precision = time.Second
   603  		f.useOCMtime = true
   604  		f.propsetMtime = true
   605  		f.hasOCMD5 = true
   606  		f.hasOCSHA1 = true
   607  	case "nextcloud":
   608  		f.precision = time.Second
   609  		f.useOCMtime = true
   610  		f.propsetMtime = true
   611  		f.hasOCSHA1 = true
   612  		f.canChunk = true
   613  
   614  		if f.opt.ChunkSize == 0 {
   615  			fs.Logf(nil, "Chunked uploads are disabled because nextcloud_chunk_size is set to 0")
   616  		} else {
   617  			chunksUploadURL, err := f.getChunksUploadURL()
   618  			if err != nil {
   619  				return err
   620  			}
   621  
   622  			f.chunksUploadURL = chunksUploadURL
   623  			fs.Debugf(nil, "Chunks temporary upload directory: %s", f.chunksUploadURL)
   624  		}
   625  	case "sharepoint":
   626  		// To mount sharepoint, two Cookies are required
   627  		// They have to be set instead of BasicAuth
   628  		f.srv.RemoveHeader("Authorization") // We don't need this Header if using cookies
   629  		spCk := odrvcookie.New(f.opt.User, f.opt.Pass, f.endpointURL)
   630  		spCookies, err := spCk.Cookies(ctx)
   631  		if err != nil {
   632  			return err
   633  		}
   634  
   635  		odrvcookie.NewRenew(12*time.Hour, func() {
   636  			spCookies, err := spCk.Cookies(ctx)
   637  			if err != nil {
   638  				fs.Errorf("could not renew cookies: %s", err.Error())
   639  				return
   640  			}
   641  			f.srv.SetCookie(&spCookies.FedAuth, &spCookies.RtFa)
   642  			fs.Debugf(spCookies, "successfully renewed sharepoint cookies")
   643  		})
   644  
   645  		f.srv.SetCookie(&spCookies.FedAuth, &spCookies.RtFa)
   646  
   647  		// sharepoint, unlike the other vendors, only lists files if the depth header is set to 0
   648  		// however, rclone defaults to 1 since it provides recursive directory listing
   649  		// to determine if we may have found a file, the request has to be resent
   650  		// with the depth set to 0
   651  		f.retryWithZeroDepth = true
   652  	case "sharepoint-ntlm":
   653  		// Sharepoint with NTLM authentication
   654  		// See comment above
   655  		f.retryWithZeroDepth = true
   656  
   657  		// Sharepoint 2016 returns status 204 to the purge request
   658  		// even if the directory to purge does not really exist
   659  		// so we must perform an extra check to detect this
   660  		// condition and return a proper error code.
   661  		f.checkBeforePurge = true
   662  	case "rclone":
   663  		f.canStream = true
   664  		f.precision = time.Second
   665  		f.useOCMtime = true
   666  	case "other":
   667  	default:
   668  		fs.Debugf(f, "Unknown vendor %q", vendor)
   669  	}
   670  
   671  	// Remove PutStream from optional features
   672  	if !f.canStream {
   673  		f.features.PutStream = nil
   674  	}
   675  	return nil
   676  }
   677  
   678  // Return an Object from a path
   679  //
   680  // If it can't be found it returns the error fs.ErrorObjectNotFound.
   681  func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *api.Prop) (fs.Object, error) {
   682  	o := &Object{
   683  		fs:     f,
   684  		remote: remote,
   685  	}
   686  	var err error
   687  	if info != nil {
   688  		// Set info
   689  		err = o.setMetaData(info)
   690  	} else {
   691  		err = o.readMetaData(ctx) // reads info and meta, returning an error
   692  	}
   693  	if err != nil {
   694  		return nil, err
   695  	}
   696  	return o, nil
   697  }
   698  
   699  // NewObject finds the Object at remote.  If it can't be found
   700  // it returns the error fs.ErrorObjectNotFound.
   701  func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
   702  	return f.newObjectWithInfo(ctx, remote, nil)
   703  }
   704  
   705  // Read the normal props, plus the checksums
   706  //
   707  // <oc:checksums><oc:checksum>SHA1:f572d396fae9206628714fb2ce00f72e94f2258f MD5:b1946ac92492d2347c6235b4d2611184 ADLER32:084b021f</oc:checksum></oc:checksums>
   708  var owncloudProps = []byte(`<?xml version="1.0"?>
   709  <d:propfind  xmlns:d="DAV:" xmlns:oc="http://owncloud.org/ns" xmlns:nc="http://nextcloud.org/ns">
   710   <d:prop>
   711    <d:displayname />
   712    <d:getlastmodified />
   713    <d:getcontentlength />
   714    <d:resourcetype />
   715    <d:getcontenttype />
   716    <oc:checksums />
   717    <oc:permissions />
   718   </d:prop>
   719  </d:propfind>
   720  `)
   721  
   722  // list the objects into the function supplied
   723  //
   724  // If directories is set it only sends directories
   725  // User function to process a File item from listAll
   726  //
   727  // Should return true to finish processing
   728  type listAllFn func(string, bool, *api.Prop) bool
   729  
   730  // Lists the directory required calling the user function on each item found
   731  //
   732  // If the user fn ever returns true then it early exits with found = true
   733  func (f *Fs) listAll(ctx context.Context, dir string, directoriesOnly bool, filesOnly bool, depth string, fn listAllFn) (found bool, err error) {
   734  	opts := rest.Opts{
   735  		Method: "PROPFIND",
   736  		Path:   f.dirPath(dir), // FIXME Should not start with /
   737  		ExtraHeaders: map[string]string{
   738  			"Depth": depth,
   739  		},
   740  	}
   741  	if f.hasOCMD5 || f.hasOCSHA1 {
   742  		opts.Body = bytes.NewBuffer(owncloudProps)
   743  	}
   744  	var result api.Multistatus
   745  	var resp *http.Response
   746  	err = f.pacer.Call(func() (bool, error) {
   747  		resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
   748  		return f.shouldRetry(ctx, resp, err)
   749  	})
   750  	if err != nil {
   751  		if apiErr, ok := err.(*api.Error); ok {
   752  			// does not exist
   753  			if apiErr.StatusCode == http.StatusNotFound {
   754  				if f.retryWithZeroDepth && depth != "0" {
   755  					return f.listAll(ctx, dir, directoriesOnly, filesOnly, "0", fn)
   756  				}
   757  				return found, fs.ErrorDirNotFound
   758  			}
   759  		}
   760  		return found, fmt.Errorf("couldn't list files: %w", err)
   761  	}
   762  	// fmt.Printf("result = %#v", &result)
   763  	baseURL, err := rest.URLJoin(f.endpoint, opts.Path)
   764  	if err != nil {
   765  		return false, fmt.Errorf("couldn't join URL: %w", err)
   766  	}
   767  	for i := range result.Responses {
   768  		item := &result.Responses[i]
   769  		isDir := itemIsDir(item)
   770  
   771  		// Find name
   772  		u, err := rest.URLJoin(baseURL, item.Href)
   773  		if err != nil {
   774  			fs.Errorf(nil, "URL Join failed for %q and %q: %v", baseURL, item.Href, err)
   775  			continue
   776  		}
   777  		// Make sure directories end with a /
   778  		if isDir {
   779  			u.Path = addSlash(u.Path)
   780  		}
   781  		if !strings.HasPrefix(u.Path, baseURL.Path) {
   782  			fs.Debugf(nil, "Item with unknown path received: %q, %q", u.Path, baseURL.Path)
   783  			continue
   784  		}
   785  		subPath := u.Path[len(baseURL.Path):]
   786  		subPath = strings.TrimPrefix(subPath, "/") // ignore leading / here for davrods
   787  		if f.opt.Enc != encoder.EncodeZero {
   788  			subPath = f.opt.Enc.ToStandardPath(subPath)
   789  		}
   790  		remote := path.Join(dir, subPath)
   791  		remote = strings.TrimSuffix(remote, "/")
   792  
   793  		// the listing contains info about itself which we ignore
   794  		if remote == dir {
   795  			continue
   796  		}
   797  
   798  		// Check OK
   799  		if !item.Props.StatusOK() {
   800  			fs.Debugf(remote, "Ignoring item with bad status %q", item.Props.Status)
   801  			continue
   802  		}
   803  
   804  		if isDir {
   805  			if filesOnly {
   806  				continue
   807  			}
   808  		} else {
   809  			if directoriesOnly {
   810  				continue
   811  			}
   812  		}
   813  		if f.opt.ExcludeShares {
   814  			// https: //owncloud.dev/apis/http/webdav/#supported-webdav-properties
   815  			if strings.Contains(item.Props.Permissions, "S") {
   816  				continue
   817  			}
   818  		}
   819  		if f.opt.ExcludeMounts {
   820  			// https: //owncloud.dev/apis/http/webdav/#supported-webdav-properties
   821  			if strings.Contains(item.Props.Permissions, "M") {
   822  				continue
   823  			}
   824  		}
   825  		// 	item.Name = restoreReservedChars(item.Name)
   826  		if fn(remote, isDir, &item.Props) {
   827  			found = true
   828  			break
   829  		}
   830  	}
   831  	return
   832  }
   833  
   834  // List the objects and directories in dir into entries.  The
   835  // entries can be returned in any order but should be for a
   836  // complete directory.
   837  //
   838  // dir should be "" to list the root, and should not have
   839  // trailing slashes.
   840  //
   841  // This should return ErrDirNotFound if the directory isn't
   842  // found.
   843  func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
   844  	var iErr error
   845  	_, err = f.listAll(ctx, dir, false, false, defaultDepth, func(remote string, isDir bool, info *api.Prop) bool {
   846  		if isDir {
   847  			d := fs.NewDir(remote, time.Time(info.Modified))
   848  			// .SetID(info.ID)
   849  			// FIXME more info from dir? can set size, items?
   850  			entries = append(entries, d)
   851  		} else {
   852  			o, err := f.newObjectWithInfo(ctx, remote, info)
   853  			if err != nil {
   854  				iErr = err
   855  				return true
   856  			}
   857  			entries = append(entries, o)
   858  		}
   859  		return false
   860  	})
   861  	if err != nil {
   862  		return nil, err
   863  	}
   864  	if iErr != nil {
   865  		return nil, iErr
   866  	}
   867  	return entries, nil
   868  }
   869  
   870  // Creates from the parameters passed in a half finished Object which
   871  // must have setMetaData called on it
   872  //
   873  // Used to create new objects
   874  func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Object) {
   875  	// Temporary Object under construction
   876  	o = &Object{
   877  		fs:      f,
   878  		remote:  remote,
   879  		size:    size,
   880  		modTime: modTime,
   881  	}
   882  	return o
   883  }
   884  
   885  // Put the object
   886  //
   887  // Copy the reader in to the new object which is returned.
   888  //
   889  // The new object may have been created if an error is returned
   890  func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
   891  	o := f.createObject(src.Remote(), src.ModTime(ctx), src.Size())
   892  	return o, o.Update(ctx, in, src, options...)
   893  }
   894  
   895  // PutStream uploads to the remote path with the modTime given of indeterminate size
   896  func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
   897  	return f.Put(ctx, in, src, options...)
   898  }
   899  
   900  // mkParentDir makes the parent of the native path dirPath if
   901  // necessary and any directories above that
   902  func (f *Fs) mkParentDir(ctx context.Context, dirPath string) (err error) {
   903  	// defer log.Trace(dirPath, "")("err=%v", &err)
   904  	// chop off trailing / if it exists
   905  	parent := path.Dir(strings.TrimSuffix(dirPath, "/"))
   906  	if parent == "." {
   907  		parent = ""
   908  	}
   909  	return f.mkdir(ctx, parent)
   910  }
   911  
   912  // _dirExists - list dirPath to see if it exists
   913  //
   914  // dirPath should be a native path ending in a /
   915  func (f *Fs) _dirExists(ctx context.Context, dirPath string) (exists bool) {
   916  	opts := rest.Opts{
   917  		Method: "PROPFIND",
   918  		Path:   dirPath,
   919  		ExtraHeaders: map[string]string{
   920  			"Depth": "0",
   921  		},
   922  	}
   923  	var result api.Multistatus
   924  	var resp *http.Response
   925  	var err error
   926  	err = f.pacer.Call(func() (bool, error) {
   927  		resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
   928  		return f.shouldRetry(ctx, resp, err)
   929  	})
   930  	return err == nil
   931  }
   932  
   933  // low level mkdir, only makes the directory, doesn't attempt to create parents
   934  func (f *Fs) _mkdir(ctx context.Context, dirPath string) error {
   935  	// We assume the root is already created
   936  	if dirPath == "" {
   937  		return nil
   938  	}
   939  	// Collections must end with /
   940  	if !strings.HasSuffix(dirPath, "/") {
   941  		dirPath += "/"
   942  	}
   943  	opts := rest.Opts{
   944  		Method:     "MKCOL",
   945  		Path:       dirPath,
   946  		NoResponse: true,
   947  	}
   948  	err := f.pacer.Call(func() (bool, error) {
   949  		resp, err := f.srv.Call(ctx, &opts)
   950  		return f.shouldRetry(ctx, resp, err)
   951  	})
   952  	if apiErr, ok := err.(*api.Error); ok {
   953  		// Check if it already exists. The response code for this isn't
   954  		// defined in the RFC so the implementations vary wildly.
   955  		//
   956  		// owncloud returns 423/StatusLocked if the create is already in progress
   957  		if apiErr.StatusCode == http.StatusMethodNotAllowed || apiErr.StatusCode == http.StatusNotAcceptable || apiErr.StatusCode == http.StatusLocked {
   958  			return nil
   959  		}
   960  		// 4shared returns a 409/StatusConflict here which clashes
   961  		// horribly with the intermediate paths don't exist meaning. So
   962  		// check to see if actually exists. This will correct other
   963  		// error codes too.
   964  		if f._dirExists(ctx, dirPath) {
   965  			return nil
   966  		}
   967  
   968  	}
   969  	return err
   970  }
   971  
   972  // mkdir makes the directory and parents using native paths
   973  func (f *Fs) mkdir(ctx context.Context, dirPath string) (err error) {
   974  	// defer log.Trace(dirPath, "")("err=%v", &err)
   975  	err = f._mkdir(ctx, dirPath)
   976  	if apiErr, ok := err.(*api.Error); ok {
   977  		// parent does not exist so create it first then try again
   978  		if apiErr.StatusCode == http.StatusConflict {
   979  			err = f.mkParentDir(ctx, dirPath)
   980  			if err == nil {
   981  				err = f._mkdir(ctx, dirPath)
   982  			}
   983  		}
   984  	}
   985  	return err
   986  }
   987  
   988  // Mkdir creates the directory if it doesn't exist
   989  func (f *Fs) Mkdir(ctx context.Context, dir string) error {
   990  	dirPath := f.dirPath(dir)
   991  	return f.mkdir(ctx, dirPath)
   992  }
   993  
   994  // dirNotEmpty returns true if the directory exists and is not Empty
   995  //
   996  // if the directory does not exist then err will be ErrorDirNotFound
   997  func (f *Fs) dirNotEmpty(ctx context.Context, dir string) (found bool, err error) {
   998  	return f.listAll(ctx, dir, false, false, defaultDepth, func(remote string, isDir bool, info *api.Prop) bool {
   999  		return true
  1000  	})
  1001  }
  1002  
  1003  // purgeCheck removes the root directory, if check is set then it
  1004  // refuses to do so if it has anything in
  1005  func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
  1006  	if check {
  1007  		notEmpty, err := f.dirNotEmpty(ctx, dir)
  1008  		if err != nil {
  1009  			return err
  1010  		}
  1011  		if notEmpty {
  1012  			return fs.ErrorDirectoryNotEmpty
  1013  		}
  1014  	} else if f.checkBeforePurge {
  1015  		// We are doing purge as the `check` argument is unset.
  1016  		// The quirk says that we are working with Sharepoint 2016.
  1017  		// This provider returns status 204 even if the purged directory
  1018  		// does not really exist so we perform an extra check here.
  1019  		// Only the existence is checked, all other errors must be
  1020  		// ignored here to make the rclone test suite pass.
  1021  		depth := defaultDepth
  1022  		if f.retryWithZeroDepth {
  1023  			depth = "0"
  1024  		}
  1025  		_, err := f.readMetaDataForPath(ctx, dir, depth)
  1026  		if err == fs.ErrorObjectNotFound {
  1027  			return fs.ErrorDirNotFound
  1028  		}
  1029  	}
  1030  	opts := rest.Opts{
  1031  		Method:     "DELETE",
  1032  		Path:       f.dirPath(dir),
  1033  		NoResponse: true,
  1034  	}
  1035  	var resp *http.Response
  1036  	var err error
  1037  	err = f.pacer.Call(func() (bool, error) {
  1038  		resp, err = f.srv.CallXML(ctx, &opts, nil, nil)
  1039  		return f.shouldRetry(ctx, resp, err)
  1040  	})
  1041  	if err != nil {
  1042  		return fmt.Errorf("rmdir failed: %w", err)
  1043  	}
  1044  	// FIXME parse Multistatus response
  1045  	return nil
  1046  }
  1047  
  1048  // Rmdir deletes the root folder
  1049  //
  1050  // Returns an error if it isn't empty
  1051  func (f *Fs) Rmdir(ctx context.Context, dir string) error {
  1052  	return f.purgeCheck(ctx, dir, true)
  1053  }
  1054  
  1055  // Precision return the precision of this Fs
  1056  func (f *Fs) Precision() time.Duration {
  1057  	return f.precision
  1058  }
  1059  
  1060  // Copy or Move src to this remote using server-side copy operations.
  1061  //
  1062  // This is stored with the remote path given.
  1063  //
  1064  // It returns the destination Object and a possible error.
  1065  //
  1066  // Will only be called if src.Fs().Name() == f.Name()
  1067  //
  1068  // If it isn't possible then return fs.ErrorCantCopy/fs.ErrorCantMove
  1069  func (f *Fs) copyOrMove(ctx context.Context, src fs.Object, remote string, method string) (fs.Object, error) {
  1070  	srcObj, ok := src.(*Object)
  1071  	if !ok {
  1072  		fs.Debugf(src, "Can't copy - not same remote type")
  1073  		if method == "COPY" {
  1074  			return nil, fs.ErrorCantCopy
  1075  		}
  1076  		return nil, fs.ErrorCantMove
  1077  	}
  1078  	srcFs := srcObj.fs
  1079  	dstPath := f.filePath(remote)
  1080  	err := f.mkParentDir(ctx, dstPath)
  1081  	if err != nil {
  1082  		return nil, fmt.Errorf("copy mkParentDir failed: %w", err)
  1083  	}
  1084  	destinationURL, err := rest.URLJoin(f.endpoint, dstPath)
  1085  	if err != nil {
  1086  		return nil, fmt.Errorf("copyOrMove couldn't join URL: %w", err)
  1087  	}
  1088  	var resp *http.Response
  1089  	opts := rest.Opts{
  1090  		Method:     method,
  1091  		Path:       srcObj.filePath(),
  1092  		NoResponse: true,
  1093  		ExtraHeaders: map[string]string{
  1094  			"Destination": destinationURL.String(),
  1095  			"Overwrite":   "T",
  1096  		},
  1097  	}
  1098  	if f.useOCMtime {
  1099  		opts.ExtraHeaders["X-OC-Mtime"] = fmt.Sprintf("%d", src.ModTime(ctx).Unix())
  1100  	}
  1101  	// Direct the MOVE/COPY to the source server
  1102  	err = srcFs.pacer.Call(func() (bool, error) {
  1103  		resp, err = srcFs.srv.Call(ctx, &opts)
  1104  		return srcFs.shouldRetry(ctx, resp, err)
  1105  	})
  1106  	if err != nil {
  1107  		return nil, fmt.Errorf("copy call failed: %w", err)
  1108  	}
  1109  	dstObj, err := f.NewObject(ctx, remote)
  1110  	if err != nil {
  1111  		return nil, fmt.Errorf("copy NewObject failed: %w", err)
  1112  	}
  1113  	if f.useOCMtime && resp.Header.Get("X-OC-Mtime") != "accepted" && f.propsetMtime && !dstObj.ModTime(ctx).Equal(src.ModTime(ctx)) {
  1114  		fs.Debugf(dstObj, "Setting modtime after copy to %v", src.ModTime(ctx))
  1115  		err = dstObj.SetModTime(ctx, src.ModTime(ctx))
  1116  		if err != nil {
  1117  			return nil, fmt.Errorf("failed to set modtime: %w", err)
  1118  		}
  1119  	}
  1120  	return dstObj, nil
  1121  }
  1122  
  1123  // Copy src to this remote using server-side copy operations.
  1124  //
  1125  // This is stored with the remote path given.
  1126  //
  1127  // It returns the destination Object and a possible error.
  1128  //
  1129  // Will only be called if src.Fs().Name() == f.Name()
  1130  //
  1131  // If it isn't possible then return fs.ErrorCantCopy
  1132  func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
  1133  	return f.copyOrMove(ctx, src, remote, "COPY")
  1134  }
  1135  
  1136  // Purge deletes all the files in the directory
  1137  //
  1138  // Optional interface: Only implement this if you have a way of
  1139  // deleting all the files quicker than just running Remove() on the
  1140  // result of List()
  1141  func (f *Fs) Purge(ctx context.Context, dir string) error {
  1142  	return f.purgeCheck(ctx, dir, false)
  1143  }
  1144  
  1145  // Move src to this remote using server-side move operations.
  1146  //
  1147  // This is stored with the remote path given.
  1148  //
  1149  // It returns the destination Object and a possible error.
  1150  //
  1151  // Will only be called if src.Fs().Name() == f.Name()
  1152  //
  1153  // If it isn't possible then return fs.ErrorCantMove
  1154  func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
  1155  	return f.copyOrMove(ctx, src, remote, "MOVE")
  1156  }
  1157  
  1158  // DirMove moves src, srcRemote to this remote at dstRemote
  1159  // using server-side move operations.
  1160  //
  1161  // Will only be called if src.Fs().Name() == f.Name()
  1162  //
  1163  // If it isn't possible then return fs.ErrorCantDirMove
  1164  //
  1165  // If destination exists then return fs.ErrorDirExists
  1166  func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
  1167  	srcFs, ok := src.(*Fs)
  1168  	if !ok {
  1169  		fs.Debugf(srcFs, "Can't move directory - not same remote type")
  1170  		return fs.ErrorCantDirMove
  1171  	}
  1172  	srcPath := srcFs.filePath(srcRemote)
  1173  	dstPath := f.filePath(dstRemote)
  1174  
  1175  	// Check if destination exists
  1176  	_, err := f.dirNotEmpty(ctx, dstRemote)
  1177  	if err == nil {
  1178  		return fs.ErrorDirExists
  1179  	}
  1180  	if err != fs.ErrorDirNotFound {
  1181  		return fmt.Errorf("DirMove dirExists dst failed: %w", err)
  1182  	}
  1183  
  1184  	// Make sure the parent directory exists
  1185  	err = f.mkParentDir(ctx, dstPath)
  1186  	if err != nil {
  1187  		return fmt.Errorf("DirMove mkParentDir dst failed: %w", err)
  1188  	}
  1189  
  1190  	destinationURL, err := rest.URLJoin(f.endpoint, dstPath)
  1191  	if err != nil {
  1192  		return fmt.Errorf("DirMove couldn't join URL: %w", err)
  1193  	}
  1194  
  1195  	var resp *http.Response
  1196  	opts := rest.Opts{
  1197  		Method:     "MOVE",
  1198  		Path:       addSlash(srcPath),
  1199  		NoResponse: true,
  1200  		ExtraHeaders: map[string]string{
  1201  			"Destination": addSlash(destinationURL.String()),
  1202  			"Overwrite":   "T",
  1203  		},
  1204  	}
  1205  	// Direct the MOVE/COPY to the source server
  1206  	err = srcFs.pacer.Call(func() (bool, error) {
  1207  		resp, err = srcFs.srv.Call(ctx, &opts)
  1208  		return srcFs.shouldRetry(ctx, resp, err)
  1209  	})
  1210  	if err != nil {
  1211  		return fmt.Errorf("DirMove MOVE call failed: %w", err)
  1212  	}
  1213  	return nil
  1214  }
  1215  
  1216  // Hashes returns the supported hash sets.
  1217  func (f *Fs) Hashes() hash.Set {
  1218  	hashes := hash.Set(hash.None)
  1219  	if f.hasOCMD5 {
  1220  		hashes.Add(hash.MD5)
  1221  	}
  1222  	if f.hasOCSHA1 || f.hasMESHA1 {
  1223  		hashes.Add(hash.SHA1)
  1224  	}
  1225  	return hashes
  1226  }
  1227  
  1228  // About gets quota information
  1229  func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
  1230  	opts := rest.Opts{
  1231  		Method: "PROPFIND",
  1232  		Path:   "",
  1233  		ExtraHeaders: map[string]string{
  1234  			"Depth": "0",
  1235  		},
  1236  	}
  1237  	opts.Body = bytes.NewBuffer([]byte(`<?xml version="1.0" ?>
  1238  <D:propfind xmlns:D="DAV:">
  1239   <D:prop>
  1240    <D:quota-available-bytes/>
  1241    <D:quota-used-bytes/>
  1242   </D:prop>
  1243  </D:propfind>
  1244  `))
  1245  	var q api.Quota
  1246  	var resp *http.Response
  1247  	var err error
  1248  	err = f.pacer.Call(func() (bool, error) {
  1249  		resp, err = f.srv.CallXML(ctx, &opts, nil, &q)
  1250  		return f.shouldRetry(ctx, resp, err)
  1251  	})
  1252  	if err != nil {
  1253  		return nil, err
  1254  	}
  1255  	usage := &fs.Usage{}
  1256  	if i, err := strconv.ParseInt(q.Used, 10, 64); err == nil && i >= 0 {
  1257  		usage.Used = fs.NewUsageValue(i)
  1258  	}
  1259  	if i, err := strconv.ParseInt(q.Available, 10, 64); err == nil && i >= 0 {
  1260  		usage.Free = fs.NewUsageValue(i)
  1261  	}
  1262  	if usage.Used != nil && usage.Free != nil {
  1263  		usage.Total = fs.NewUsageValue(*usage.Used + *usage.Free)
  1264  	}
  1265  	return usage, nil
  1266  }
  1267  
  1268  // ------------------------------------------------------------
  1269  
  1270  // Fs returns the parent Fs
  1271  func (o *Object) Fs() fs.Info {
  1272  	return o.fs
  1273  }
  1274  
  1275  // Return a string version
  1276  func (o *Object) String() string {
  1277  	if o == nil {
  1278  		return "<nil>"
  1279  	}
  1280  	return o.remote
  1281  }
  1282  
  1283  // Remote returns the remote path
  1284  func (o *Object) Remote() string {
  1285  	return o.remote
  1286  }
  1287  
  1288  // Hash returns the SHA1 or MD5 of an object returning a lowercase hex string
  1289  func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
  1290  	if t == hash.MD5 && o.fs.hasOCMD5 {
  1291  		return o.md5, nil
  1292  	}
  1293  	if t == hash.SHA1 && (o.fs.hasOCSHA1 || o.fs.hasMESHA1) {
  1294  		return o.sha1, nil
  1295  	}
  1296  	return "", hash.ErrUnsupported
  1297  }
  1298  
  1299  // Size returns the size of an object in bytes
  1300  func (o *Object) Size() int64 {
  1301  	ctx := context.TODO()
  1302  	err := o.readMetaData(ctx)
  1303  	if err != nil {
  1304  		fs.Logf(o, "Failed to read metadata: %v", err)
  1305  		return 0
  1306  	}
  1307  	return o.size
  1308  }
  1309  
  1310  // setMetaData sets the metadata from info
  1311  func (o *Object) setMetaData(info *api.Prop) (err error) {
  1312  	o.hasMetaData = true
  1313  	o.size = info.Size
  1314  	o.modTime = time.Time(info.Modified)
  1315  	if o.fs.hasOCMD5 || o.fs.hasOCSHA1 || o.fs.hasMESHA1 {
  1316  		hashes := info.Hashes()
  1317  		if o.fs.hasOCSHA1 || o.fs.hasMESHA1 {
  1318  			o.sha1 = hashes[hash.SHA1]
  1319  		}
  1320  		if o.fs.hasOCMD5 {
  1321  			o.md5 = hashes[hash.MD5]
  1322  		}
  1323  	}
  1324  	return nil
  1325  }
  1326  
  1327  // readMetaData gets the metadata if it hasn't already been fetched
  1328  //
  1329  // it also sets the info
  1330  func (o *Object) readMetaData(ctx context.Context) (err error) {
  1331  	if o.hasMetaData {
  1332  		return nil
  1333  	}
  1334  	info, err := o.fs.readMetaDataForPath(ctx, o.remote, defaultDepth)
  1335  	if err != nil {
  1336  		return err
  1337  	}
  1338  	return o.setMetaData(info)
  1339  }
  1340  
  1341  // ModTime returns the modification time of the object
  1342  //
  1343  // It attempts to read the objects mtime and if that isn't present the
  1344  // LastModified returned in the http headers
  1345  func (o *Object) ModTime(ctx context.Context) time.Time {
  1346  	err := o.readMetaData(ctx)
  1347  	if err != nil {
  1348  		fs.Logf(o, "Failed to read metadata: %v", err)
  1349  		return time.Now()
  1350  	}
  1351  	return o.modTime
  1352  }
  1353  
  1354  // Set modified time using propset
  1355  //
  1356  // <d:multistatus xmlns:d="DAV:" xmlns:s="http://sabredav.org/ns"><d:response><d:href>/ocm/remote.php/webdav/office/wir.jpg</d:href><d:propstat><d:prop><d:lastmodified/></d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response></d:multistatus>
  1357  var owncloudPropset = `<?xml version="1.0" encoding="utf-8" ?>
  1358  <D:propertyupdate xmlns:D="DAV:">
  1359   <D:set>
  1360    <D:prop>
  1361     <lastmodified xmlns="DAV:">%d</lastmodified>
  1362    </D:prop>
  1363   </D:set>
  1364  </D:propertyupdate>
  1365  `
  1366  
  1367  var owncloudPropsetWithChecksum = `<?xml version="1.0" encoding="utf-8" ?>
  1368  <D:propertyupdate xmlns:D="DAV:" xmlns:oc="http://owncloud.org/ns">
  1369   <D:set>
  1370    <D:prop>
  1371     <lastmodified xmlns="DAV:">%d</lastmodified>
  1372     <oc:checksums>
  1373  	<oc:checksum>%s</oc:checksum>
  1374     </oc:checksums>
  1375    </D:prop>
  1376   </D:set>
  1377  </D:propertyupdate>
  1378  `
  1379  
  1380  // SetModTime sets the modification time of the local fs object
  1381  func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
  1382  	if o.fs.propsetMtime {
  1383  		checksums := ""
  1384  		if o.fs.hasOCSHA1 && o.sha1 != "" {
  1385  			checksums = "SHA1:" + o.sha1
  1386  		} else if o.fs.hasOCMD5 && o.md5 != "" {
  1387  			checksums = "MD5:" + o.md5
  1388  		}
  1389  
  1390  		opts := rest.Opts{
  1391  			Method:     "PROPPATCH",
  1392  			Path:       o.filePath(),
  1393  			NoRedirect: true,
  1394  			Body:       strings.NewReader(fmt.Sprintf(owncloudPropset, modTime.Unix())),
  1395  		}
  1396  		if checksums != "" {
  1397  			opts.Body = strings.NewReader(fmt.Sprintf(owncloudPropsetWithChecksum, modTime.Unix(), checksums))
  1398  		}
  1399  		var result api.Multistatus
  1400  		var resp *http.Response
  1401  		var err error
  1402  		err = o.fs.pacer.Call(func() (bool, error) {
  1403  			resp, err = o.fs.srv.CallXML(ctx, &opts, nil, &result)
  1404  			return o.fs.shouldRetry(ctx, resp, err)
  1405  		})
  1406  		if err != nil {
  1407  			if apiErr, ok := err.(*api.Error); ok {
  1408  				// does not exist
  1409  				if apiErr.StatusCode == http.StatusNotFound {
  1410  					return fs.ErrorObjectNotFound
  1411  				}
  1412  			}
  1413  			return fmt.Errorf("couldn't set modified time: %w", err)
  1414  		}
  1415  		// FIXME check if response is valid
  1416  		if len(result.Responses) == 1 && result.Responses[0].Props.StatusOK() {
  1417  			// update cached modtime
  1418  			o.modTime = modTime
  1419  			return nil
  1420  		}
  1421  		// got an error, but it's possible it actually worked, so double-check
  1422  		newO, err := o.fs.NewObject(ctx, o.remote)
  1423  		if err != nil {
  1424  			return err
  1425  		}
  1426  		if newO.ModTime(ctx).Equal(modTime) {
  1427  			return nil
  1428  		}
  1429  		// fallback
  1430  		return fs.ErrorCantSetModTime
  1431  	}
  1432  	return fs.ErrorCantSetModTime
  1433  }
  1434  
  1435  // Storable returns a boolean showing whether this object storable
  1436  func (o *Object) Storable() bool {
  1437  	return true
  1438  }
  1439  
  1440  // Open an object for read
  1441  func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
  1442  	var resp *http.Response
  1443  	fs.FixRangeOption(options, o.size)
  1444  	opts := rest.Opts{
  1445  		Method:  "GET",
  1446  		Path:    o.filePath(),
  1447  		Options: options,
  1448  		ExtraHeaders: map[string]string{
  1449  			"Depth": "0",
  1450  		},
  1451  	}
  1452  	err = o.fs.pacer.Call(func() (bool, error) {
  1453  		resp, err = o.fs.srv.Call(ctx, &opts)
  1454  		return o.fs.shouldRetry(ctx, resp, err)
  1455  	})
  1456  	if err != nil {
  1457  		return nil, err
  1458  	}
  1459  	return resp.Body, err
  1460  }
  1461  
  1462  // Update the object with the contents of the io.Reader, modTime and size
  1463  //
  1464  // If existing is set then it updates the object rather than creating a new one.
  1465  //
  1466  // The new object may have been created if an error is returned
  1467  func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
  1468  	err = o.fs.mkParentDir(ctx, o.filePath())
  1469  	if err != nil {
  1470  		return fmt.Errorf("Update mkParentDir failed: %w", err)
  1471  	}
  1472  
  1473  	if o.shouldUseChunkedUpload(src) {
  1474  		fs.Debugf(src, "Update will use the chunked upload strategy")
  1475  		err = o.updateChunked(ctx, in, src, options...)
  1476  		if err != nil {
  1477  			return err
  1478  		}
  1479  	} else {
  1480  		fs.Debugf(src, "Update will use the normal upload strategy (no chunks)")
  1481  		contentType := fs.MimeType(ctx, src)
  1482  		filePath := o.filePath()
  1483  		extraHeaders := o.extraHeaders(ctx, src)
  1484  		// TODO: define getBody() to enable low-level HTTP/2 retries
  1485  		err = o.updateSimple(ctx, in, nil, filePath, src.Size(), contentType, extraHeaders, o.fs.endpointURL, options...)
  1486  		if err != nil {
  1487  			return err
  1488  		}
  1489  	}
  1490  
  1491  	// read metadata from remote
  1492  	o.hasMetaData = false
  1493  	return o.readMetaData(ctx)
  1494  }
  1495  
  1496  func (o *Object) extraHeaders(ctx context.Context, src fs.ObjectInfo) map[string]string {
  1497  	extraHeaders := map[string]string{}
  1498  	if o.fs.useOCMtime || o.fs.hasOCMD5 || o.fs.hasOCSHA1 {
  1499  		if o.fs.useOCMtime {
  1500  			extraHeaders["X-OC-Mtime"] = fmt.Sprintf("%d", src.ModTime(ctx).Unix())
  1501  		}
  1502  		// Set one upload checksum
  1503  		// Owncloud uses one checksum only to check the upload and stores its own SHA1 and MD5
  1504  		// Nextcloud stores the checksum you supply (SHA1 or MD5) but only stores one
  1505  		if o.fs.hasOCSHA1 {
  1506  			if sha1, _ := src.Hash(ctx, hash.SHA1); sha1 != "" {
  1507  				extraHeaders["OC-Checksum"] = "SHA1:" + sha1
  1508  			}
  1509  		}
  1510  		if o.fs.hasOCMD5 && extraHeaders["OC-Checksum"] == "" {
  1511  			if md5, _ := src.Hash(ctx, hash.MD5); md5 != "" {
  1512  				extraHeaders["OC-Checksum"] = "MD5:" + md5
  1513  			}
  1514  		}
  1515  	}
  1516  	return extraHeaders
  1517  }
  1518  
  1519  // Standard update in one request (no chunks)
  1520  func (o *Object) updateSimple(ctx context.Context, body io.Reader, getBody func() (io.ReadCloser, error), filePath string, size int64, contentType string, extraHeaders map[string]string, rootURL string, options ...fs.OpenOption) (err error) {
  1521  	var resp *http.Response
  1522  
  1523  	if extraHeaders == nil {
  1524  		extraHeaders = map[string]string{}
  1525  	}
  1526  
  1527  	opts := rest.Opts{
  1528  		Method:        "PUT",
  1529  		Path:          filePath,
  1530  		GetBody:       getBody,
  1531  		Body:          body,
  1532  		NoResponse:    true,
  1533  		ContentLength: &size, // FIXME this isn't necessary with owncloud - See https://github.com/nextcloud/nextcloud-snap/issues/365
  1534  		ContentType:   contentType,
  1535  		Options:       options,
  1536  		ExtraHeaders:  extraHeaders,
  1537  		RootURL:       rootURL,
  1538  	}
  1539  	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
  1540  		resp, err = o.fs.srv.Call(ctx, &opts)
  1541  		return o.fs.shouldRetry(ctx, resp, err)
  1542  	})
  1543  	if err != nil {
  1544  		// Give the WebDAV server a chance to get its internal state in order after the
  1545  		// error.  The error may have been local in which case we closed the connection.
  1546  		// The server may still be dealing with it for a moment. A sleep isn't ideal but I
  1547  		// haven't been able to think of a better method to find out if the server has
  1548  		// finished - ncw
  1549  		time.Sleep(1 * time.Second)
  1550  		// Remove failed upload
  1551  		_ = o.Remove(ctx)
  1552  		return err
  1553  	}
  1554  	return nil
  1555  }
  1556  
  1557  // Remove an object
  1558  func (o *Object) Remove(ctx context.Context) error {
  1559  	opts := rest.Opts{
  1560  		Method:     "DELETE",
  1561  		Path:       o.filePath(),
  1562  		NoResponse: true,
  1563  	}
  1564  	return o.fs.pacer.Call(func() (bool, error) {
  1565  		resp, err := o.fs.srv.Call(ctx, &opts)
  1566  		return o.fs.shouldRetry(ctx, resp, err)
  1567  	})
  1568  }
  1569  
  1570  // Check the interfaces are satisfied
  1571  var (
  1572  	_ fs.Fs          = (*Fs)(nil)
  1573  	_ fs.Purger      = (*Fs)(nil)
  1574  	_ fs.PutStreamer = (*Fs)(nil)
  1575  	_ fs.Copier      = (*Fs)(nil)
  1576  	_ fs.Mover       = (*Fs)(nil)
  1577  	_ fs.DirMover    = (*Fs)(nil)
  1578  	_ fs.Abouter     = (*Fs)(nil)
  1579  	_ fs.Object      = (*Object)(nil)
  1580  )