github.com/rclone/rclone@v1.66.1-0.20240517100346-7b89735ae726/backend/webdav/webdav.go (about) 1 // Package webdav provides an interface to the Webdav 2 // object storage system. 3 package webdav 4 5 // SetModTime might be possible 6 // https://stackoverflow.com/questions/3579608/webdav-can-a-client-modify-the-mtime-of-a-file 7 // ...support for a PROPSET to lastmodified (mind the missing get) which does the utime() call might be an option. 8 // For example the ownCloud WebDAV server does it that way. 9 10 import ( 11 "bytes" 12 "context" 13 "crypto/tls" 14 "encoding/xml" 15 "errors" 16 "fmt" 17 "io" 18 "net/http" 19 "net/url" 20 "os/exec" 21 "path" 22 "regexp" 23 "strconv" 24 "strings" 25 "sync" 26 "time" 27 28 "github.com/rclone/rclone/backend/webdav/api" 29 "github.com/rclone/rclone/backend/webdav/odrvcookie" 30 "github.com/rclone/rclone/fs" 31 "github.com/rclone/rclone/fs/config" 32 "github.com/rclone/rclone/fs/config/configmap" 33 "github.com/rclone/rclone/fs/config/configstruct" 34 "github.com/rclone/rclone/fs/config/obscure" 35 "github.com/rclone/rclone/fs/fserrors" 36 "github.com/rclone/rclone/fs/fshttp" 37 "github.com/rclone/rclone/fs/hash" 38 "github.com/rclone/rclone/lib/encoder" 39 "github.com/rclone/rclone/lib/pacer" 40 "github.com/rclone/rclone/lib/rest" 41 42 ntlmssp "github.com/Azure/go-ntlmssp" 43 ) 44 45 const ( 46 minSleep = fs.Duration(10 * time.Millisecond) 47 maxSleep = 2 * time.Second 48 decayConstant = 2 // bigger for slower decay, exponential 49 defaultDepth = "1" // depth for PROPFIND 50 ) 51 52 const defaultEncodingSharepointNTLM = (encoder.EncodeWin | 53 encoder.EncodeHashPercent | // required by IIS/8.5 in contrast with onedrive which doesn't need it 54 (encoder.Display &^ encoder.EncodeDot) | // test with IIS/8.5 shows that EncodeDot is not needed 55 encoder.EncodeBackSlash | 56 encoder.EncodeLeftSpace | 57 encoder.EncodeLeftTilde | 58 encoder.EncodeRightPeriod | 59 encoder.EncodeRightSpace | 60 encoder.EncodeInvalidUtf8) 61 62 // Register with Fs 63 func init() { 64 configEncodingHelp := fmt.Sprintf( 65 "%s\n\nDefault encoding is %s for sharepoint-ntlm or identity otherwise.", 66 config.ConfigEncodingHelp, defaultEncodingSharepointNTLM) 67 68 fs.Register(&fs.RegInfo{ 69 Name: "webdav", 70 Description: "WebDAV", 71 NewFs: NewFs, 72 Options: []fs.Option{{ 73 Name: "url", 74 Help: "URL of http host to connect to.\n\nE.g. https://example.com.", 75 Required: true, 76 }, { 77 Name: "vendor", 78 Help: "Name of the WebDAV site/service/software you are using.", 79 Examples: []fs.OptionExample{{ 80 Value: "fastmail", 81 Help: "Fastmail Files", 82 }, { 83 Value: "nextcloud", 84 Help: "Nextcloud", 85 }, { 86 Value: "owncloud", 87 Help: "Owncloud", 88 }, { 89 Value: "sharepoint", 90 Help: "Sharepoint Online, authenticated by Microsoft account", 91 }, { 92 Value: "sharepoint-ntlm", 93 Help: "Sharepoint with NTLM authentication, usually self-hosted or on-premises", 94 }, { 95 Value: "rclone", 96 Help: "rclone WebDAV server to serve a remote over HTTP via the WebDAV protocol", 97 }, { 98 Value: "other", 99 Help: "Other site/service or software", 100 }}, 101 }, { 102 Name: "user", 103 Help: "User name.\n\nIn case NTLM authentication is used, the username should be in the format 'Domain\\User'.", 104 Sensitive: true, 105 }, { 106 Name: "pass", 107 Help: "Password.", 108 IsPassword: true, 109 }, { 110 Name: "bearer_token", 111 Help: "Bearer token instead of user/pass (e.g. a Macaroon).", 112 Sensitive: true, 113 }, { 114 Name: "bearer_token_command", 115 Help: "Command to run to get a bearer token.", 116 Advanced: true, 117 }, { 118 Name: config.ConfigEncoding, 119 Help: configEncodingHelp, 120 Advanced: true, 121 }, { 122 Name: "headers", 123 Help: `Set HTTP headers for all transactions. 124 125 Use this to set additional HTTP headers for all transactions 126 127 The input format is comma separated list of key,value pairs. Standard 128 [CSV encoding](https://godoc.org/encoding/csv) may be used. 129 130 For example, to set a Cookie use 'Cookie,name=value', or '"Cookie","name=value"'. 131 132 You can set multiple headers, e.g. '"Cookie","name=value","Authorization","xxx"'. 133 `, 134 Default: fs.CommaSepList{}, 135 Advanced: true, 136 }, { 137 Name: "pacer_min_sleep", 138 Help: "Minimum time to sleep between API calls.", 139 Default: minSleep, 140 Advanced: true, 141 }, { 142 Name: "nextcloud_chunk_size", 143 Help: `Nextcloud upload chunk size. 144 145 We recommend configuring your NextCloud instance to increase the max chunk size to 1 GB for better upload performances. 146 See https://docs.nextcloud.com/server/latest/admin_manual/configuration_files/big_file_upload_configuration.html#adjust-chunk-size-on-nextcloud-side 147 148 Set to 0 to disable chunked uploading. 149 `, 150 Advanced: true, 151 Default: 10 * fs.Mebi, // Default NextCloud `max_chunk_size` is `10 MiB`. See https://github.com/nextcloud/server/blob/0447b53bda9fe95ea0cbed765aa332584605d652/apps/files/lib/App.php#L57 152 }, { 153 Name: "owncloud_exclude_shares", 154 Help: "Exclude ownCloud shares", 155 Advanced: true, 156 Default: false, 157 }, { 158 Name: "owncloud_exclude_mounts", 159 Help: "Exclude ownCloud mounted storages", 160 Advanced: true, 161 Default: false, 162 }}, 163 }) 164 } 165 166 // Options defines the configuration for this backend 167 type Options struct { 168 URL string `config:"url"` 169 Vendor string `config:"vendor"` 170 User string `config:"user"` 171 Pass string `config:"pass"` 172 BearerToken string `config:"bearer_token"` 173 BearerTokenCommand string `config:"bearer_token_command"` 174 Enc encoder.MultiEncoder `config:"encoding"` 175 Headers fs.CommaSepList `config:"headers"` 176 PacerMinSleep fs.Duration `config:"pacer_min_sleep"` 177 ChunkSize fs.SizeSuffix `config:"nextcloud_chunk_size"` 178 ExcludeShares bool `config:"owncloud_exclude_shares"` 179 ExcludeMounts bool `config:"owncloud_exclude_mounts"` 180 } 181 182 // Fs represents a remote webdav 183 type Fs struct { 184 name string // name of this remote 185 root string // the path we are working on 186 opt Options // parsed options 187 features *fs.Features // optional features 188 endpoint *url.URL // URL of the host 189 endpointURL string // endpoint as a string 190 srv *rest.Client // the connection to the server 191 pacer *fs.Pacer // pacer for API calls 192 precision time.Duration // mod time precision 193 canStream bool // set if can stream 194 useOCMtime bool // set if can use X-OC-Mtime 195 propsetMtime bool // set if can use propset 196 retryWithZeroDepth bool // some vendors (sharepoint) won't list files when Depth is 1 (our default) 197 checkBeforePurge bool // enables extra check that directory to purge really exists 198 hasOCMD5 bool // set if can use owncloud style checksums for MD5 199 hasOCSHA1 bool // set if can use owncloud style checksums for SHA1 200 hasMESHA1 bool // set if can use fastmail style checksums for SHA1 201 ntlmAuthMu sync.Mutex // mutex to serialize NTLM auth roundtrips 202 chunksUploadURL string // upload URL for nextcloud chunked 203 canChunk bool // set if nextcloud and nextcloud_chunk_size is set 204 } 205 206 // Object describes a webdav object 207 // 208 // Will definitely have info but maybe not meta 209 type Object struct { 210 fs *Fs // what this object is part of 211 remote string // The remote path 212 hasMetaData bool // whether info below has been set 213 size int64 // size of the object 214 modTime time.Time // modification time of the object 215 sha1 string // SHA-1 of the object content if known 216 md5 string // MD5 of the object content if known 217 } 218 219 // ------------------------------------------------------------ 220 221 // Name of the remote (as passed into NewFs) 222 func (f *Fs) Name() string { 223 return f.name 224 } 225 226 // Root of the remote (as passed into NewFs) 227 func (f *Fs) Root() string { 228 return f.root 229 } 230 231 // String converts this Fs to a string 232 func (f *Fs) String() string { 233 return fmt.Sprintf("webdav root '%s'", f.root) 234 } 235 236 // Features returns the optional features of this Fs 237 func (f *Fs) Features() *fs.Features { 238 return f.features 239 } 240 241 // retryErrorCodes is a slice of error codes that we will retry 242 var retryErrorCodes = []int{ 243 423, // Locked 244 429, // Too Many Requests. 245 500, // Internal Server Error 246 502, // Bad Gateway 247 503, // Service Unavailable 248 504, // Gateway Timeout 249 509, // Bandwidth Limit Exceeded 250 } 251 252 // shouldRetry returns a boolean as to whether this resp and err 253 // deserve to be retried. It returns the err as a convenience 254 func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) { 255 if fserrors.ContextError(ctx, &err) { 256 return false, err 257 } 258 // If we have a bearer token command and it has expired then refresh it 259 if f.opt.BearerTokenCommand != "" && resp != nil && resp.StatusCode == 401 { 260 fs.Debugf(f, "Bearer token expired: %v", err) 261 authErr := f.fetchAndSetBearerToken() 262 if authErr != nil { 263 err = authErr 264 } 265 return true, err 266 } 267 return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err 268 } 269 270 // safeRoundTripper is a wrapper for http.RoundTripper that serializes 271 // http roundtrips. NTLM authentication sequence can involve up to four 272 // rounds of negotiations and might fail due to concurrency. 273 // This wrapper allows to use ntlmssp.Negotiator safely with goroutines. 274 type safeRoundTripper struct { 275 fs *Fs 276 rt http.RoundTripper 277 } 278 279 // RoundTrip guards wrapped RoundTripper by a mutex. 280 func (srt *safeRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { 281 srt.fs.ntlmAuthMu.Lock() 282 defer srt.fs.ntlmAuthMu.Unlock() 283 return srt.rt.RoundTrip(req) 284 } 285 286 // itemIsDir returns true if the item is a directory 287 // 288 // When a client sees a resourcetype it doesn't recognize it should 289 // assume it is a regular non-collection resource. [WebDav book by 290 // Lisa Dusseault ch 7.5.8 p170] 291 func itemIsDir(item *api.Response) bool { 292 if t := item.Props.Type; t != nil { 293 if t.Space == "DAV:" && t.Local == "collection" { 294 return true 295 } 296 fs.Debugf(nil, "Unknown resource type %q/%q on %q", t.Space, t.Local, item.Props.Name) 297 } 298 // the iscollection prop is a Microsoft extension, but if present it is a reliable indicator 299 // if the above check failed - see #2716. This can be an integer or a boolean - see #2964 300 if t := item.Props.IsCollection; t != nil { 301 switch x := strings.ToLower(*t); x { 302 case "0", "false": 303 return false 304 case "1", "true": 305 return true 306 default: 307 fs.Debugf(nil, "Unknown value %q for IsCollection", x) 308 } 309 } 310 return false 311 } 312 313 // readMetaDataForPath reads the metadata from the path 314 func (f *Fs) readMetaDataForPath(ctx context.Context, path string, depth string) (info *api.Prop, err error) { 315 // FIXME how do we read back additional properties? 316 opts := rest.Opts{ 317 Method: "PROPFIND", 318 Path: f.filePath(path), 319 ExtraHeaders: map[string]string{ 320 "Depth": depth, 321 }, 322 NoRedirect: true, 323 } 324 if f.hasOCMD5 || f.hasOCSHA1 { 325 opts.Body = bytes.NewBuffer(owncloudProps) 326 } 327 var result api.Multistatus 328 var resp *http.Response 329 err = f.pacer.Call(func() (bool, error) { 330 resp, err = f.srv.CallXML(ctx, &opts, nil, &result) 331 return f.shouldRetry(ctx, resp, err) 332 }) 333 if apiErr, ok := err.(*api.Error); ok { 334 // does not exist 335 switch apiErr.StatusCode { 336 case http.StatusNotFound: 337 if f.retryWithZeroDepth && depth != "0" { 338 return f.readMetaDataForPath(ctx, path, "0") 339 } 340 return nil, fs.ErrorObjectNotFound 341 case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther: 342 // Some sort of redirect - go doesn't deal with these properly (it resets 343 // the method to GET). However we can assume that if it was redirected the 344 // object was not found. 345 return nil, fs.ErrorObjectNotFound 346 } 347 } 348 if err != nil { 349 return nil, fmt.Errorf("read metadata failed: %w", err) 350 } 351 if len(result.Responses) < 1 { 352 return nil, fs.ErrorObjectNotFound 353 } 354 item := result.Responses[0] 355 if !item.Props.StatusOK() { 356 return nil, fs.ErrorObjectNotFound 357 } 358 if itemIsDir(&item) { 359 return nil, fs.ErrorIsDir 360 } 361 return &item.Props, nil 362 } 363 364 // errorHandler parses a non 2xx error response into an error 365 func errorHandler(resp *http.Response) error { 366 body, err := rest.ReadBody(resp) 367 if err != nil { 368 return fmt.Errorf("error when trying to read error from body: %w", err) 369 } 370 // Decode error response 371 errResponse := new(api.Error) 372 err = xml.Unmarshal(body, &errResponse) 373 if err != nil { 374 // set the Message to be the body if can't parse the XML 375 errResponse.Message = strings.TrimSpace(string(body)) 376 } 377 errResponse.Status = resp.Status 378 errResponse.StatusCode = resp.StatusCode 379 return errResponse 380 } 381 382 // addSlash makes sure s is terminated with a / if non empty 383 func addSlash(s string) string { 384 if s != "" && !strings.HasSuffix(s, "/") { 385 s += "/" 386 } 387 return s 388 } 389 390 // filePath returns a file path (f.root, file) 391 func (f *Fs) filePath(file string) string { 392 subPath := path.Join(f.root, file) 393 if f.opt.Enc != encoder.EncodeZero { 394 subPath = f.opt.Enc.FromStandardPath(subPath) 395 } 396 return rest.URLPathEscape(subPath) 397 } 398 399 // dirPath returns a directory path (f.root, dir) 400 func (f *Fs) dirPath(dir string) string { 401 return addSlash(f.filePath(dir)) 402 } 403 404 // filePath returns a file path (f.root, remote) 405 func (o *Object) filePath() string { 406 return o.fs.filePath(o.remote) 407 } 408 409 // NewFs constructs an Fs from the path, container:path 410 func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { 411 // Parse config into Options struct 412 opt := new(Options) 413 err := configstruct.Set(m, opt) 414 if err != nil { 415 return nil, err 416 } 417 418 if len(opt.Headers)%2 != 0 { 419 return nil, errors.New("odd number of headers supplied") 420 } 421 fs.Debugf(nil, "found headers: %v", opt.Headers) 422 423 rootIsDir := strings.HasSuffix(root, "/") 424 root = strings.Trim(root, "/") 425 426 if !strings.HasSuffix(opt.URL, "/") { 427 opt.URL += "/" 428 } 429 if opt.Pass != "" { 430 var err error 431 opt.Pass, err = obscure.Reveal(opt.Pass) 432 if err != nil { 433 return nil, fmt.Errorf("couldn't decrypt password: %w", err) 434 } 435 } 436 if opt.Vendor == "" { 437 opt.Vendor = "other" 438 } 439 root = strings.Trim(root, "/") 440 441 if opt.Enc == encoder.EncodeZero && opt.Vendor == "sharepoint-ntlm" { 442 opt.Enc = defaultEncodingSharepointNTLM 443 } 444 445 // Parse the endpoint 446 u, err := url.Parse(opt.URL) 447 if err != nil { 448 return nil, err 449 } 450 451 f := &Fs{ 452 name: name, 453 root: root, 454 opt: *opt, 455 endpoint: u, 456 endpointURL: u.String(), 457 pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(opt.PacerMinSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))), 458 precision: fs.ModTimeNotSupported, 459 } 460 461 client := fshttp.NewClient(ctx) 462 if opt.Vendor == "sharepoint-ntlm" { 463 // Disable transparent HTTP/2 support as per https://golang.org/pkg/net/http/ , 464 // otherwise any connection to IIS 10.0 fails with 'stream error: stream ID 39; HTTP_1_1_REQUIRED' 465 // https://docs.microsoft.com/en-us/iis/get-started/whats-new-in-iis-10/http2-on-iis says: 466 // 'Windows authentication (NTLM/Kerberos/Negotiate) is not supported with HTTP/2.' 467 t := fshttp.NewTransportCustom(ctx, func(t *http.Transport) { 468 t.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{} 469 }) 470 471 // Add NTLM layer 472 client.Transport = &safeRoundTripper{ 473 fs: f, 474 rt: ntlmssp.Negotiator{RoundTripper: t}, 475 } 476 } 477 f.srv = rest.NewClient(client).SetRoot(u.String()) 478 479 f.features = (&fs.Features{ 480 CanHaveEmptyDirectories: true, 481 }).Fill(ctx, f) 482 if opt.User != "" || opt.Pass != "" { 483 f.srv.SetUserPass(opt.User, opt.Pass) 484 } else if opt.BearerToken != "" { 485 f.setBearerToken(opt.BearerToken) 486 } else if f.opt.BearerTokenCommand != "" { 487 err = f.fetchAndSetBearerToken() 488 if err != nil { 489 return nil, err 490 } 491 } 492 if opt.Headers != nil { 493 f.addHeaders(opt.Headers) 494 } 495 f.srv.SetErrorHandler(errorHandler) 496 err = f.setQuirks(ctx, opt.Vendor) 497 if err != nil { 498 return nil, err 499 } 500 if !f.findHeader(opt.Headers, "Referer") { 501 f.srv.SetHeader("Referer", u.String()) 502 } 503 504 if root != "" && !rootIsDir { 505 // Check to see if the root actually an existing file 506 remote := path.Base(root) 507 f.root = path.Dir(root) 508 if f.root == "." { 509 f.root = "" 510 } 511 _, err := f.NewObject(ctx, remote) 512 if err != nil { 513 if errors.Is(err, fs.ErrorObjectNotFound) || errors.Is(err, fs.ErrorIsDir) { 514 // File doesn't exist so return old f 515 f.root = root 516 return f, nil 517 } 518 return nil, err 519 } 520 // return an error with an fs which points to the parent 521 return f, fs.ErrorIsFile 522 } 523 return f, nil 524 } 525 526 // sets the BearerToken up 527 func (f *Fs) setBearerToken(token string) { 528 f.opt.BearerToken = token 529 f.srv.SetHeader("Authorization", "Bearer "+token) 530 } 531 532 // fetch the bearer token using the command 533 func (f *Fs) fetchBearerToken(cmd string) (string, error) { 534 var ( 535 args = strings.Split(cmd, " ") 536 stdout bytes.Buffer 537 stderr bytes.Buffer 538 c = exec.Command(args[0], args[1:]...) 539 ) 540 c.Stdout = &stdout 541 c.Stderr = &stderr 542 var ( 543 err = c.Run() 544 stdoutString = strings.TrimSpace(stdout.String()) 545 stderrString = strings.TrimSpace(stderr.String()) 546 ) 547 if err != nil { 548 if stderrString == "" { 549 stderrString = stdoutString 550 } 551 return "", fmt.Errorf("failed to get bearer token using %q: %s: %w", f.opt.BearerTokenCommand, stderrString, err) 552 } 553 return stdoutString, nil 554 } 555 556 // Adds the configured headers to the request if any 557 func (f *Fs) addHeaders(headers fs.CommaSepList) { 558 for i := 0; i < len(headers); i += 2 { 559 key := f.opt.Headers[i] 560 value := f.opt.Headers[i+1] 561 f.srv.SetHeader(key, value) 562 } 563 } 564 565 // Returns true if the header was configured 566 func (f *Fs) findHeader(headers fs.CommaSepList, find string) bool { 567 for i := 0; i < len(headers); i += 2 { 568 key := f.opt.Headers[i] 569 if strings.EqualFold(key, find) { 570 return true 571 } 572 } 573 return false 574 } 575 576 // fetch the bearer token and set it if successful 577 func (f *Fs) fetchAndSetBearerToken() error { 578 if f.opt.BearerTokenCommand == "" { 579 return nil 580 } 581 token, err := f.fetchBearerToken(f.opt.BearerTokenCommand) 582 if err != nil { 583 return err 584 } 585 f.setBearerToken(token) 586 return nil 587 } 588 589 // The WebDAV url can optionally be suffixed with a path. This suffix needs to be ignored for determining the temporary upload directory of chunks. 590 var nextCloudURLRegex = regexp.MustCompile(`^(.*)/dav/files/([^/]+)`) 591 592 // setQuirks adjusts the Fs for the vendor passed in 593 func (f *Fs) setQuirks(ctx context.Context, vendor string) error { 594 switch vendor { 595 case "fastmail": 596 f.canStream = true 597 f.precision = time.Second 598 f.useOCMtime = true 599 f.hasMESHA1 = true 600 case "owncloud": 601 f.canStream = true 602 f.precision = time.Second 603 f.useOCMtime = true 604 f.propsetMtime = true 605 f.hasOCMD5 = true 606 f.hasOCSHA1 = true 607 case "nextcloud": 608 f.precision = time.Second 609 f.useOCMtime = true 610 f.propsetMtime = true 611 f.hasOCSHA1 = true 612 f.canChunk = true 613 614 if f.opt.ChunkSize == 0 { 615 fs.Logf(nil, "Chunked uploads are disabled because nextcloud_chunk_size is set to 0") 616 } else { 617 chunksUploadURL, err := f.getChunksUploadURL() 618 if err != nil { 619 return err 620 } 621 622 f.chunksUploadURL = chunksUploadURL 623 fs.Debugf(nil, "Chunks temporary upload directory: %s", f.chunksUploadURL) 624 } 625 case "sharepoint": 626 // To mount sharepoint, two Cookies are required 627 // They have to be set instead of BasicAuth 628 f.srv.RemoveHeader("Authorization") // We don't need this Header if using cookies 629 spCk := odrvcookie.New(f.opt.User, f.opt.Pass, f.endpointURL) 630 spCookies, err := spCk.Cookies(ctx) 631 if err != nil { 632 return err 633 } 634 635 odrvcookie.NewRenew(12*time.Hour, func() { 636 spCookies, err := spCk.Cookies(ctx) 637 if err != nil { 638 fs.Errorf("could not renew cookies: %s", err.Error()) 639 return 640 } 641 f.srv.SetCookie(&spCookies.FedAuth, &spCookies.RtFa) 642 fs.Debugf(spCookies, "successfully renewed sharepoint cookies") 643 }) 644 645 f.srv.SetCookie(&spCookies.FedAuth, &spCookies.RtFa) 646 647 // sharepoint, unlike the other vendors, only lists files if the depth header is set to 0 648 // however, rclone defaults to 1 since it provides recursive directory listing 649 // to determine if we may have found a file, the request has to be resent 650 // with the depth set to 0 651 f.retryWithZeroDepth = true 652 case "sharepoint-ntlm": 653 // Sharepoint with NTLM authentication 654 // See comment above 655 f.retryWithZeroDepth = true 656 657 // Sharepoint 2016 returns status 204 to the purge request 658 // even if the directory to purge does not really exist 659 // so we must perform an extra check to detect this 660 // condition and return a proper error code. 661 f.checkBeforePurge = true 662 case "rclone": 663 f.canStream = true 664 f.precision = time.Second 665 f.useOCMtime = true 666 case "other": 667 default: 668 fs.Debugf(f, "Unknown vendor %q", vendor) 669 } 670 671 // Remove PutStream from optional features 672 if !f.canStream { 673 f.features.PutStream = nil 674 } 675 return nil 676 } 677 678 // Return an Object from a path 679 // 680 // If it can't be found it returns the error fs.ErrorObjectNotFound. 681 func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *api.Prop) (fs.Object, error) { 682 o := &Object{ 683 fs: f, 684 remote: remote, 685 } 686 var err error 687 if info != nil { 688 // Set info 689 err = o.setMetaData(info) 690 } else { 691 err = o.readMetaData(ctx) // reads info and meta, returning an error 692 } 693 if err != nil { 694 return nil, err 695 } 696 return o, nil 697 } 698 699 // NewObject finds the Object at remote. If it can't be found 700 // it returns the error fs.ErrorObjectNotFound. 701 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { 702 return f.newObjectWithInfo(ctx, remote, nil) 703 } 704 705 // Read the normal props, plus the checksums 706 // 707 // <oc:checksums><oc:checksum>SHA1:f572d396fae9206628714fb2ce00f72e94f2258f MD5:b1946ac92492d2347c6235b4d2611184 ADLER32:084b021f</oc:checksum></oc:checksums> 708 var owncloudProps = []byte(`<?xml version="1.0"?> 709 <d:propfind xmlns:d="DAV:" xmlns:oc="http://owncloud.org/ns" xmlns:nc="http://nextcloud.org/ns"> 710 <d:prop> 711 <d:displayname /> 712 <d:getlastmodified /> 713 <d:getcontentlength /> 714 <d:resourcetype /> 715 <d:getcontenttype /> 716 <oc:checksums /> 717 <oc:permissions /> 718 </d:prop> 719 </d:propfind> 720 `) 721 722 // list the objects into the function supplied 723 // 724 // If directories is set it only sends directories 725 // User function to process a File item from listAll 726 // 727 // Should return true to finish processing 728 type listAllFn func(string, bool, *api.Prop) bool 729 730 // Lists the directory required calling the user function on each item found 731 // 732 // If the user fn ever returns true then it early exits with found = true 733 func (f *Fs) listAll(ctx context.Context, dir string, directoriesOnly bool, filesOnly bool, depth string, fn listAllFn) (found bool, err error) { 734 opts := rest.Opts{ 735 Method: "PROPFIND", 736 Path: f.dirPath(dir), // FIXME Should not start with / 737 ExtraHeaders: map[string]string{ 738 "Depth": depth, 739 }, 740 } 741 if f.hasOCMD5 || f.hasOCSHA1 { 742 opts.Body = bytes.NewBuffer(owncloudProps) 743 } 744 var result api.Multistatus 745 var resp *http.Response 746 err = f.pacer.Call(func() (bool, error) { 747 resp, err = f.srv.CallXML(ctx, &opts, nil, &result) 748 return f.shouldRetry(ctx, resp, err) 749 }) 750 if err != nil { 751 if apiErr, ok := err.(*api.Error); ok { 752 // does not exist 753 if apiErr.StatusCode == http.StatusNotFound { 754 if f.retryWithZeroDepth && depth != "0" { 755 return f.listAll(ctx, dir, directoriesOnly, filesOnly, "0", fn) 756 } 757 return found, fs.ErrorDirNotFound 758 } 759 } 760 return found, fmt.Errorf("couldn't list files: %w", err) 761 } 762 // fmt.Printf("result = %#v", &result) 763 baseURL, err := rest.URLJoin(f.endpoint, opts.Path) 764 if err != nil { 765 return false, fmt.Errorf("couldn't join URL: %w", err) 766 } 767 for i := range result.Responses { 768 item := &result.Responses[i] 769 isDir := itemIsDir(item) 770 771 // Find name 772 u, err := rest.URLJoin(baseURL, item.Href) 773 if err != nil { 774 fs.Errorf(nil, "URL Join failed for %q and %q: %v", baseURL, item.Href, err) 775 continue 776 } 777 // Make sure directories end with a / 778 if isDir { 779 u.Path = addSlash(u.Path) 780 } 781 if !strings.HasPrefix(u.Path, baseURL.Path) { 782 fs.Debugf(nil, "Item with unknown path received: %q, %q", u.Path, baseURL.Path) 783 continue 784 } 785 subPath := u.Path[len(baseURL.Path):] 786 subPath = strings.TrimPrefix(subPath, "/") // ignore leading / here for davrods 787 if f.opt.Enc != encoder.EncodeZero { 788 subPath = f.opt.Enc.ToStandardPath(subPath) 789 } 790 remote := path.Join(dir, subPath) 791 remote = strings.TrimSuffix(remote, "/") 792 793 // the listing contains info about itself which we ignore 794 if remote == dir { 795 continue 796 } 797 798 // Check OK 799 if !item.Props.StatusOK() { 800 fs.Debugf(remote, "Ignoring item with bad status %q", item.Props.Status) 801 continue 802 } 803 804 if isDir { 805 if filesOnly { 806 continue 807 } 808 } else { 809 if directoriesOnly { 810 continue 811 } 812 } 813 if f.opt.ExcludeShares { 814 // https: //owncloud.dev/apis/http/webdav/#supported-webdav-properties 815 if strings.Contains(item.Props.Permissions, "S") { 816 continue 817 } 818 } 819 if f.opt.ExcludeMounts { 820 // https: //owncloud.dev/apis/http/webdav/#supported-webdav-properties 821 if strings.Contains(item.Props.Permissions, "M") { 822 continue 823 } 824 } 825 // item.Name = restoreReservedChars(item.Name) 826 if fn(remote, isDir, &item.Props) { 827 found = true 828 break 829 } 830 } 831 return 832 } 833 834 // List the objects and directories in dir into entries. The 835 // entries can be returned in any order but should be for a 836 // complete directory. 837 // 838 // dir should be "" to list the root, and should not have 839 // trailing slashes. 840 // 841 // This should return ErrDirNotFound if the directory isn't 842 // found. 843 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { 844 var iErr error 845 _, err = f.listAll(ctx, dir, false, false, defaultDepth, func(remote string, isDir bool, info *api.Prop) bool { 846 if isDir { 847 d := fs.NewDir(remote, time.Time(info.Modified)) 848 // .SetID(info.ID) 849 // FIXME more info from dir? can set size, items? 850 entries = append(entries, d) 851 } else { 852 o, err := f.newObjectWithInfo(ctx, remote, info) 853 if err != nil { 854 iErr = err 855 return true 856 } 857 entries = append(entries, o) 858 } 859 return false 860 }) 861 if err != nil { 862 return nil, err 863 } 864 if iErr != nil { 865 return nil, iErr 866 } 867 return entries, nil 868 } 869 870 // Creates from the parameters passed in a half finished Object which 871 // must have setMetaData called on it 872 // 873 // Used to create new objects 874 func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Object) { 875 // Temporary Object under construction 876 o = &Object{ 877 fs: f, 878 remote: remote, 879 size: size, 880 modTime: modTime, 881 } 882 return o 883 } 884 885 // Put the object 886 // 887 // Copy the reader in to the new object which is returned. 888 // 889 // The new object may have been created if an error is returned 890 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { 891 o := f.createObject(src.Remote(), src.ModTime(ctx), src.Size()) 892 return o, o.Update(ctx, in, src, options...) 893 } 894 895 // PutStream uploads to the remote path with the modTime given of indeterminate size 896 func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { 897 return f.Put(ctx, in, src, options...) 898 } 899 900 // mkParentDir makes the parent of the native path dirPath if 901 // necessary and any directories above that 902 func (f *Fs) mkParentDir(ctx context.Context, dirPath string) (err error) { 903 // defer log.Trace(dirPath, "")("err=%v", &err) 904 // chop off trailing / if it exists 905 parent := path.Dir(strings.TrimSuffix(dirPath, "/")) 906 if parent == "." { 907 parent = "" 908 } 909 return f.mkdir(ctx, parent) 910 } 911 912 // _dirExists - list dirPath to see if it exists 913 // 914 // dirPath should be a native path ending in a / 915 func (f *Fs) _dirExists(ctx context.Context, dirPath string) (exists bool) { 916 opts := rest.Opts{ 917 Method: "PROPFIND", 918 Path: dirPath, 919 ExtraHeaders: map[string]string{ 920 "Depth": "0", 921 }, 922 } 923 var result api.Multistatus 924 var resp *http.Response 925 var err error 926 err = f.pacer.Call(func() (bool, error) { 927 resp, err = f.srv.CallXML(ctx, &opts, nil, &result) 928 return f.shouldRetry(ctx, resp, err) 929 }) 930 return err == nil 931 } 932 933 // low level mkdir, only makes the directory, doesn't attempt to create parents 934 func (f *Fs) _mkdir(ctx context.Context, dirPath string) error { 935 // We assume the root is already created 936 if dirPath == "" { 937 return nil 938 } 939 // Collections must end with / 940 if !strings.HasSuffix(dirPath, "/") { 941 dirPath += "/" 942 } 943 opts := rest.Opts{ 944 Method: "MKCOL", 945 Path: dirPath, 946 NoResponse: true, 947 } 948 err := f.pacer.Call(func() (bool, error) { 949 resp, err := f.srv.Call(ctx, &opts) 950 return f.shouldRetry(ctx, resp, err) 951 }) 952 if apiErr, ok := err.(*api.Error); ok { 953 // Check if it already exists. The response code for this isn't 954 // defined in the RFC so the implementations vary wildly. 955 // 956 // owncloud returns 423/StatusLocked if the create is already in progress 957 if apiErr.StatusCode == http.StatusMethodNotAllowed || apiErr.StatusCode == http.StatusNotAcceptable || apiErr.StatusCode == http.StatusLocked { 958 return nil 959 } 960 // 4shared returns a 409/StatusConflict here which clashes 961 // horribly with the intermediate paths don't exist meaning. So 962 // check to see if actually exists. This will correct other 963 // error codes too. 964 if f._dirExists(ctx, dirPath) { 965 return nil 966 } 967 968 } 969 return err 970 } 971 972 // mkdir makes the directory and parents using native paths 973 func (f *Fs) mkdir(ctx context.Context, dirPath string) (err error) { 974 // defer log.Trace(dirPath, "")("err=%v", &err) 975 err = f._mkdir(ctx, dirPath) 976 if apiErr, ok := err.(*api.Error); ok { 977 // parent does not exist so create it first then try again 978 if apiErr.StatusCode == http.StatusConflict { 979 err = f.mkParentDir(ctx, dirPath) 980 if err == nil { 981 err = f._mkdir(ctx, dirPath) 982 } 983 } 984 } 985 return err 986 } 987 988 // Mkdir creates the directory if it doesn't exist 989 func (f *Fs) Mkdir(ctx context.Context, dir string) error { 990 dirPath := f.dirPath(dir) 991 return f.mkdir(ctx, dirPath) 992 } 993 994 // dirNotEmpty returns true if the directory exists and is not Empty 995 // 996 // if the directory does not exist then err will be ErrorDirNotFound 997 func (f *Fs) dirNotEmpty(ctx context.Context, dir string) (found bool, err error) { 998 return f.listAll(ctx, dir, false, false, defaultDepth, func(remote string, isDir bool, info *api.Prop) bool { 999 return true 1000 }) 1001 } 1002 1003 // purgeCheck removes the root directory, if check is set then it 1004 // refuses to do so if it has anything in 1005 func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error { 1006 if check { 1007 notEmpty, err := f.dirNotEmpty(ctx, dir) 1008 if err != nil { 1009 return err 1010 } 1011 if notEmpty { 1012 return fs.ErrorDirectoryNotEmpty 1013 } 1014 } else if f.checkBeforePurge { 1015 // We are doing purge as the `check` argument is unset. 1016 // The quirk says that we are working with Sharepoint 2016. 1017 // This provider returns status 204 even if the purged directory 1018 // does not really exist so we perform an extra check here. 1019 // Only the existence is checked, all other errors must be 1020 // ignored here to make the rclone test suite pass. 1021 depth := defaultDepth 1022 if f.retryWithZeroDepth { 1023 depth = "0" 1024 } 1025 _, err := f.readMetaDataForPath(ctx, dir, depth) 1026 if err == fs.ErrorObjectNotFound { 1027 return fs.ErrorDirNotFound 1028 } 1029 } 1030 opts := rest.Opts{ 1031 Method: "DELETE", 1032 Path: f.dirPath(dir), 1033 NoResponse: true, 1034 } 1035 var resp *http.Response 1036 var err error 1037 err = f.pacer.Call(func() (bool, error) { 1038 resp, err = f.srv.CallXML(ctx, &opts, nil, nil) 1039 return f.shouldRetry(ctx, resp, err) 1040 }) 1041 if err != nil { 1042 return fmt.Errorf("rmdir failed: %w", err) 1043 } 1044 // FIXME parse Multistatus response 1045 return nil 1046 } 1047 1048 // Rmdir deletes the root folder 1049 // 1050 // Returns an error if it isn't empty 1051 func (f *Fs) Rmdir(ctx context.Context, dir string) error { 1052 return f.purgeCheck(ctx, dir, true) 1053 } 1054 1055 // Precision return the precision of this Fs 1056 func (f *Fs) Precision() time.Duration { 1057 return f.precision 1058 } 1059 1060 // Copy or Move src to this remote using server-side copy operations. 1061 // 1062 // This is stored with the remote path given. 1063 // 1064 // It returns the destination Object and a possible error. 1065 // 1066 // Will only be called if src.Fs().Name() == f.Name() 1067 // 1068 // If it isn't possible then return fs.ErrorCantCopy/fs.ErrorCantMove 1069 func (f *Fs) copyOrMove(ctx context.Context, src fs.Object, remote string, method string) (fs.Object, error) { 1070 srcObj, ok := src.(*Object) 1071 if !ok { 1072 fs.Debugf(src, "Can't copy - not same remote type") 1073 if method == "COPY" { 1074 return nil, fs.ErrorCantCopy 1075 } 1076 return nil, fs.ErrorCantMove 1077 } 1078 srcFs := srcObj.fs 1079 dstPath := f.filePath(remote) 1080 err := f.mkParentDir(ctx, dstPath) 1081 if err != nil { 1082 return nil, fmt.Errorf("copy mkParentDir failed: %w", err) 1083 } 1084 destinationURL, err := rest.URLJoin(f.endpoint, dstPath) 1085 if err != nil { 1086 return nil, fmt.Errorf("copyOrMove couldn't join URL: %w", err) 1087 } 1088 var resp *http.Response 1089 opts := rest.Opts{ 1090 Method: method, 1091 Path: srcObj.filePath(), 1092 NoResponse: true, 1093 ExtraHeaders: map[string]string{ 1094 "Destination": destinationURL.String(), 1095 "Overwrite": "T", 1096 }, 1097 } 1098 if f.useOCMtime { 1099 opts.ExtraHeaders["X-OC-Mtime"] = fmt.Sprintf("%d", src.ModTime(ctx).Unix()) 1100 } 1101 // Direct the MOVE/COPY to the source server 1102 err = srcFs.pacer.Call(func() (bool, error) { 1103 resp, err = srcFs.srv.Call(ctx, &opts) 1104 return srcFs.shouldRetry(ctx, resp, err) 1105 }) 1106 if err != nil { 1107 return nil, fmt.Errorf("copy call failed: %w", err) 1108 } 1109 dstObj, err := f.NewObject(ctx, remote) 1110 if err != nil { 1111 return nil, fmt.Errorf("copy NewObject failed: %w", err) 1112 } 1113 if f.useOCMtime && resp.Header.Get("X-OC-Mtime") != "accepted" && f.propsetMtime && !dstObj.ModTime(ctx).Equal(src.ModTime(ctx)) { 1114 fs.Debugf(dstObj, "Setting modtime after copy to %v", src.ModTime(ctx)) 1115 err = dstObj.SetModTime(ctx, src.ModTime(ctx)) 1116 if err != nil { 1117 return nil, fmt.Errorf("failed to set modtime: %w", err) 1118 } 1119 } 1120 return dstObj, nil 1121 } 1122 1123 // Copy src to this remote using server-side copy operations. 1124 // 1125 // This is stored with the remote path given. 1126 // 1127 // It returns the destination Object and a possible error. 1128 // 1129 // Will only be called if src.Fs().Name() == f.Name() 1130 // 1131 // If it isn't possible then return fs.ErrorCantCopy 1132 func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { 1133 return f.copyOrMove(ctx, src, remote, "COPY") 1134 } 1135 1136 // Purge deletes all the files in the directory 1137 // 1138 // Optional interface: Only implement this if you have a way of 1139 // deleting all the files quicker than just running Remove() on the 1140 // result of List() 1141 func (f *Fs) Purge(ctx context.Context, dir string) error { 1142 return f.purgeCheck(ctx, dir, false) 1143 } 1144 1145 // Move src to this remote using server-side move operations. 1146 // 1147 // This is stored with the remote path given. 1148 // 1149 // It returns the destination Object and a possible error. 1150 // 1151 // Will only be called if src.Fs().Name() == f.Name() 1152 // 1153 // If it isn't possible then return fs.ErrorCantMove 1154 func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { 1155 return f.copyOrMove(ctx, src, remote, "MOVE") 1156 } 1157 1158 // DirMove moves src, srcRemote to this remote at dstRemote 1159 // using server-side move operations. 1160 // 1161 // Will only be called if src.Fs().Name() == f.Name() 1162 // 1163 // If it isn't possible then return fs.ErrorCantDirMove 1164 // 1165 // If destination exists then return fs.ErrorDirExists 1166 func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error { 1167 srcFs, ok := src.(*Fs) 1168 if !ok { 1169 fs.Debugf(srcFs, "Can't move directory - not same remote type") 1170 return fs.ErrorCantDirMove 1171 } 1172 srcPath := srcFs.filePath(srcRemote) 1173 dstPath := f.filePath(dstRemote) 1174 1175 // Check if destination exists 1176 _, err := f.dirNotEmpty(ctx, dstRemote) 1177 if err == nil { 1178 return fs.ErrorDirExists 1179 } 1180 if err != fs.ErrorDirNotFound { 1181 return fmt.Errorf("DirMove dirExists dst failed: %w", err) 1182 } 1183 1184 // Make sure the parent directory exists 1185 err = f.mkParentDir(ctx, dstPath) 1186 if err != nil { 1187 return fmt.Errorf("DirMove mkParentDir dst failed: %w", err) 1188 } 1189 1190 destinationURL, err := rest.URLJoin(f.endpoint, dstPath) 1191 if err != nil { 1192 return fmt.Errorf("DirMove couldn't join URL: %w", err) 1193 } 1194 1195 var resp *http.Response 1196 opts := rest.Opts{ 1197 Method: "MOVE", 1198 Path: addSlash(srcPath), 1199 NoResponse: true, 1200 ExtraHeaders: map[string]string{ 1201 "Destination": addSlash(destinationURL.String()), 1202 "Overwrite": "T", 1203 }, 1204 } 1205 // Direct the MOVE/COPY to the source server 1206 err = srcFs.pacer.Call(func() (bool, error) { 1207 resp, err = srcFs.srv.Call(ctx, &opts) 1208 return srcFs.shouldRetry(ctx, resp, err) 1209 }) 1210 if err != nil { 1211 return fmt.Errorf("DirMove MOVE call failed: %w", err) 1212 } 1213 return nil 1214 } 1215 1216 // Hashes returns the supported hash sets. 1217 func (f *Fs) Hashes() hash.Set { 1218 hashes := hash.Set(hash.None) 1219 if f.hasOCMD5 { 1220 hashes.Add(hash.MD5) 1221 } 1222 if f.hasOCSHA1 || f.hasMESHA1 { 1223 hashes.Add(hash.SHA1) 1224 } 1225 return hashes 1226 } 1227 1228 // About gets quota information 1229 func (f *Fs) About(ctx context.Context) (*fs.Usage, error) { 1230 opts := rest.Opts{ 1231 Method: "PROPFIND", 1232 Path: "", 1233 ExtraHeaders: map[string]string{ 1234 "Depth": "0", 1235 }, 1236 } 1237 opts.Body = bytes.NewBuffer([]byte(`<?xml version="1.0" ?> 1238 <D:propfind xmlns:D="DAV:"> 1239 <D:prop> 1240 <D:quota-available-bytes/> 1241 <D:quota-used-bytes/> 1242 </D:prop> 1243 </D:propfind> 1244 `)) 1245 var q api.Quota 1246 var resp *http.Response 1247 var err error 1248 err = f.pacer.Call(func() (bool, error) { 1249 resp, err = f.srv.CallXML(ctx, &opts, nil, &q) 1250 return f.shouldRetry(ctx, resp, err) 1251 }) 1252 if err != nil { 1253 return nil, err 1254 } 1255 usage := &fs.Usage{} 1256 if i, err := strconv.ParseInt(q.Used, 10, 64); err == nil && i >= 0 { 1257 usage.Used = fs.NewUsageValue(i) 1258 } 1259 if i, err := strconv.ParseInt(q.Available, 10, 64); err == nil && i >= 0 { 1260 usage.Free = fs.NewUsageValue(i) 1261 } 1262 if usage.Used != nil && usage.Free != nil { 1263 usage.Total = fs.NewUsageValue(*usage.Used + *usage.Free) 1264 } 1265 return usage, nil 1266 } 1267 1268 // ------------------------------------------------------------ 1269 1270 // Fs returns the parent Fs 1271 func (o *Object) Fs() fs.Info { 1272 return o.fs 1273 } 1274 1275 // Return a string version 1276 func (o *Object) String() string { 1277 if o == nil { 1278 return "<nil>" 1279 } 1280 return o.remote 1281 } 1282 1283 // Remote returns the remote path 1284 func (o *Object) Remote() string { 1285 return o.remote 1286 } 1287 1288 // Hash returns the SHA1 or MD5 of an object returning a lowercase hex string 1289 func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) { 1290 if t == hash.MD5 && o.fs.hasOCMD5 { 1291 return o.md5, nil 1292 } 1293 if t == hash.SHA1 && (o.fs.hasOCSHA1 || o.fs.hasMESHA1) { 1294 return o.sha1, nil 1295 } 1296 return "", hash.ErrUnsupported 1297 } 1298 1299 // Size returns the size of an object in bytes 1300 func (o *Object) Size() int64 { 1301 ctx := context.TODO() 1302 err := o.readMetaData(ctx) 1303 if err != nil { 1304 fs.Logf(o, "Failed to read metadata: %v", err) 1305 return 0 1306 } 1307 return o.size 1308 } 1309 1310 // setMetaData sets the metadata from info 1311 func (o *Object) setMetaData(info *api.Prop) (err error) { 1312 o.hasMetaData = true 1313 o.size = info.Size 1314 o.modTime = time.Time(info.Modified) 1315 if o.fs.hasOCMD5 || o.fs.hasOCSHA1 || o.fs.hasMESHA1 { 1316 hashes := info.Hashes() 1317 if o.fs.hasOCSHA1 || o.fs.hasMESHA1 { 1318 o.sha1 = hashes[hash.SHA1] 1319 } 1320 if o.fs.hasOCMD5 { 1321 o.md5 = hashes[hash.MD5] 1322 } 1323 } 1324 return nil 1325 } 1326 1327 // readMetaData gets the metadata if it hasn't already been fetched 1328 // 1329 // it also sets the info 1330 func (o *Object) readMetaData(ctx context.Context) (err error) { 1331 if o.hasMetaData { 1332 return nil 1333 } 1334 info, err := o.fs.readMetaDataForPath(ctx, o.remote, defaultDepth) 1335 if err != nil { 1336 return err 1337 } 1338 return o.setMetaData(info) 1339 } 1340 1341 // ModTime returns the modification time of the object 1342 // 1343 // It attempts to read the objects mtime and if that isn't present the 1344 // LastModified returned in the http headers 1345 func (o *Object) ModTime(ctx context.Context) time.Time { 1346 err := o.readMetaData(ctx) 1347 if err != nil { 1348 fs.Logf(o, "Failed to read metadata: %v", err) 1349 return time.Now() 1350 } 1351 return o.modTime 1352 } 1353 1354 // Set modified time using propset 1355 // 1356 // <d:multistatus xmlns:d="DAV:" xmlns:s="http://sabredav.org/ns"><d:response><d:href>/ocm/remote.php/webdav/office/wir.jpg</d:href><d:propstat><d:prop><d:lastmodified/></d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response></d:multistatus> 1357 var owncloudPropset = `<?xml version="1.0" encoding="utf-8" ?> 1358 <D:propertyupdate xmlns:D="DAV:"> 1359 <D:set> 1360 <D:prop> 1361 <lastmodified xmlns="DAV:">%d</lastmodified> 1362 </D:prop> 1363 </D:set> 1364 </D:propertyupdate> 1365 ` 1366 1367 var owncloudPropsetWithChecksum = `<?xml version="1.0" encoding="utf-8" ?> 1368 <D:propertyupdate xmlns:D="DAV:" xmlns:oc="http://owncloud.org/ns"> 1369 <D:set> 1370 <D:prop> 1371 <lastmodified xmlns="DAV:">%d</lastmodified> 1372 <oc:checksums> 1373 <oc:checksum>%s</oc:checksum> 1374 </oc:checksums> 1375 </D:prop> 1376 </D:set> 1377 </D:propertyupdate> 1378 ` 1379 1380 // SetModTime sets the modification time of the local fs object 1381 func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error { 1382 if o.fs.propsetMtime { 1383 checksums := "" 1384 if o.fs.hasOCSHA1 && o.sha1 != "" { 1385 checksums = "SHA1:" + o.sha1 1386 } else if o.fs.hasOCMD5 && o.md5 != "" { 1387 checksums = "MD5:" + o.md5 1388 } 1389 1390 opts := rest.Opts{ 1391 Method: "PROPPATCH", 1392 Path: o.filePath(), 1393 NoRedirect: true, 1394 Body: strings.NewReader(fmt.Sprintf(owncloudPropset, modTime.Unix())), 1395 } 1396 if checksums != "" { 1397 opts.Body = strings.NewReader(fmt.Sprintf(owncloudPropsetWithChecksum, modTime.Unix(), checksums)) 1398 } 1399 var result api.Multistatus 1400 var resp *http.Response 1401 var err error 1402 err = o.fs.pacer.Call(func() (bool, error) { 1403 resp, err = o.fs.srv.CallXML(ctx, &opts, nil, &result) 1404 return o.fs.shouldRetry(ctx, resp, err) 1405 }) 1406 if err != nil { 1407 if apiErr, ok := err.(*api.Error); ok { 1408 // does not exist 1409 if apiErr.StatusCode == http.StatusNotFound { 1410 return fs.ErrorObjectNotFound 1411 } 1412 } 1413 return fmt.Errorf("couldn't set modified time: %w", err) 1414 } 1415 // FIXME check if response is valid 1416 if len(result.Responses) == 1 && result.Responses[0].Props.StatusOK() { 1417 // update cached modtime 1418 o.modTime = modTime 1419 return nil 1420 } 1421 // got an error, but it's possible it actually worked, so double-check 1422 newO, err := o.fs.NewObject(ctx, o.remote) 1423 if err != nil { 1424 return err 1425 } 1426 if newO.ModTime(ctx).Equal(modTime) { 1427 return nil 1428 } 1429 // fallback 1430 return fs.ErrorCantSetModTime 1431 } 1432 return fs.ErrorCantSetModTime 1433 } 1434 1435 // Storable returns a boolean showing whether this object storable 1436 func (o *Object) Storable() bool { 1437 return true 1438 } 1439 1440 // Open an object for read 1441 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) { 1442 var resp *http.Response 1443 fs.FixRangeOption(options, o.size) 1444 opts := rest.Opts{ 1445 Method: "GET", 1446 Path: o.filePath(), 1447 Options: options, 1448 ExtraHeaders: map[string]string{ 1449 "Depth": "0", 1450 }, 1451 } 1452 err = o.fs.pacer.Call(func() (bool, error) { 1453 resp, err = o.fs.srv.Call(ctx, &opts) 1454 return o.fs.shouldRetry(ctx, resp, err) 1455 }) 1456 if err != nil { 1457 return nil, err 1458 } 1459 return resp.Body, err 1460 } 1461 1462 // Update the object with the contents of the io.Reader, modTime and size 1463 // 1464 // If existing is set then it updates the object rather than creating a new one. 1465 // 1466 // The new object may have been created if an error is returned 1467 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { 1468 err = o.fs.mkParentDir(ctx, o.filePath()) 1469 if err != nil { 1470 return fmt.Errorf("Update mkParentDir failed: %w", err) 1471 } 1472 1473 if o.shouldUseChunkedUpload(src) { 1474 fs.Debugf(src, "Update will use the chunked upload strategy") 1475 err = o.updateChunked(ctx, in, src, options...) 1476 if err != nil { 1477 return err 1478 } 1479 } else { 1480 fs.Debugf(src, "Update will use the normal upload strategy (no chunks)") 1481 contentType := fs.MimeType(ctx, src) 1482 filePath := o.filePath() 1483 extraHeaders := o.extraHeaders(ctx, src) 1484 // TODO: define getBody() to enable low-level HTTP/2 retries 1485 err = o.updateSimple(ctx, in, nil, filePath, src.Size(), contentType, extraHeaders, o.fs.endpointURL, options...) 1486 if err != nil { 1487 return err 1488 } 1489 } 1490 1491 // read metadata from remote 1492 o.hasMetaData = false 1493 return o.readMetaData(ctx) 1494 } 1495 1496 func (o *Object) extraHeaders(ctx context.Context, src fs.ObjectInfo) map[string]string { 1497 extraHeaders := map[string]string{} 1498 if o.fs.useOCMtime || o.fs.hasOCMD5 || o.fs.hasOCSHA1 { 1499 if o.fs.useOCMtime { 1500 extraHeaders["X-OC-Mtime"] = fmt.Sprintf("%d", src.ModTime(ctx).Unix()) 1501 } 1502 // Set one upload checksum 1503 // Owncloud uses one checksum only to check the upload and stores its own SHA1 and MD5 1504 // Nextcloud stores the checksum you supply (SHA1 or MD5) but only stores one 1505 if o.fs.hasOCSHA1 { 1506 if sha1, _ := src.Hash(ctx, hash.SHA1); sha1 != "" { 1507 extraHeaders["OC-Checksum"] = "SHA1:" + sha1 1508 } 1509 } 1510 if o.fs.hasOCMD5 && extraHeaders["OC-Checksum"] == "" { 1511 if md5, _ := src.Hash(ctx, hash.MD5); md5 != "" { 1512 extraHeaders["OC-Checksum"] = "MD5:" + md5 1513 } 1514 } 1515 } 1516 return extraHeaders 1517 } 1518 1519 // Standard update in one request (no chunks) 1520 func (o *Object) updateSimple(ctx context.Context, body io.Reader, getBody func() (io.ReadCloser, error), filePath string, size int64, contentType string, extraHeaders map[string]string, rootURL string, options ...fs.OpenOption) (err error) { 1521 var resp *http.Response 1522 1523 if extraHeaders == nil { 1524 extraHeaders = map[string]string{} 1525 } 1526 1527 opts := rest.Opts{ 1528 Method: "PUT", 1529 Path: filePath, 1530 GetBody: getBody, 1531 Body: body, 1532 NoResponse: true, 1533 ContentLength: &size, // FIXME this isn't necessary with owncloud - See https://github.com/nextcloud/nextcloud-snap/issues/365 1534 ContentType: contentType, 1535 Options: options, 1536 ExtraHeaders: extraHeaders, 1537 RootURL: rootURL, 1538 } 1539 err = o.fs.pacer.CallNoRetry(func() (bool, error) { 1540 resp, err = o.fs.srv.Call(ctx, &opts) 1541 return o.fs.shouldRetry(ctx, resp, err) 1542 }) 1543 if err != nil { 1544 // Give the WebDAV server a chance to get its internal state in order after the 1545 // error. The error may have been local in which case we closed the connection. 1546 // The server may still be dealing with it for a moment. A sleep isn't ideal but I 1547 // haven't been able to think of a better method to find out if the server has 1548 // finished - ncw 1549 time.Sleep(1 * time.Second) 1550 // Remove failed upload 1551 _ = o.Remove(ctx) 1552 return err 1553 } 1554 return nil 1555 } 1556 1557 // Remove an object 1558 func (o *Object) Remove(ctx context.Context) error { 1559 opts := rest.Opts{ 1560 Method: "DELETE", 1561 Path: o.filePath(), 1562 NoResponse: true, 1563 } 1564 return o.fs.pacer.Call(func() (bool, error) { 1565 resp, err := o.fs.srv.Call(ctx, &opts) 1566 return o.fs.shouldRetry(ctx, resp, err) 1567 }) 1568 } 1569 1570 // Check the interfaces are satisfied 1571 var ( 1572 _ fs.Fs = (*Fs)(nil) 1573 _ fs.Purger = (*Fs)(nil) 1574 _ fs.PutStreamer = (*Fs)(nil) 1575 _ fs.Copier = (*Fs)(nil) 1576 _ fs.Mover = (*Fs)(nil) 1577 _ fs.DirMover = (*Fs)(nil) 1578 _ fs.Abouter = (*Fs)(nil) 1579 _ fs.Object = (*Object)(nil) 1580 )