github.com/ncw/rclone@v1.48.1-0.20190724201158-a35aa1360e3e/backend/azureblob/azureblob.go (about) 1 // Package azureblob provides an interface to the Microsoft Azure blob object storage system 2 3 // +build !plan9,!solaris 4 5 package azureblob 6 7 import ( 8 "bytes" 9 "context" 10 "crypto/md5" 11 "encoding/base64" 12 "encoding/binary" 13 "encoding/hex" 14 "fmt" 15 "io" 16 "net/http" 17 "net/url" 18 "path" 19 "regexp" 20 "strconv" 21 "strings" 22 "sync" 23 "time" 24 25 "github.com/Azure/azure-pipeline-go/pipeline" 26 "github.com/Azure/azure-storage-blob-go/azblob" 27 "github.com/ncw/rclone/fs" 28 "github.com/ncw/rclone/fs/accounting" 29 "github.com/ncw/rclone/fs/config/configmap" 30 "github.com/ncw/rclone/fs/config/configstruct" 31 "github.com/ncw/rclone/fs/fserrors" 32 "github.com/ncw/rclone/fs/fshttp" 33 "github.com/ncw/rclone/fs/hash" 34 "github.com/ncw/rclone/fs/walk" 35 "github.com/ncw/rclone/lib/pacer" 36 "github.com/pkg/errors" 37 ) 38 39 const ( 40 minSleep = 10 * time.Millisecond 41 maxSleep = 10 * time.Second 42 decayConstant = 1 // bigger for slower decay, exponential 43 maxListChunkSize = 5000 // number of items to read at once 44 modTimeKey = "mtime" 45 timeFormatIn = time.RFC3339 46 timeFormatOut = "2006-01-02T15:04:05.000000000Z07:00" 47 maxTotalParts = 50000 // in multipart upload 48 storageDefaultBaseURL = "blob.core.windows.net" 49 // maxUncommittedSize = 9 << 30 // can't upload bigger than this 50 defaultChunkSize = 4 * fs.MebiByte 51 maxChunkSize = 100 * fs.MebiByte 52 defaultUploadCutoff = 256 * fs.MebiByte 53 maxUploadCutoff = 256 * fs.MebiByte 54 defaultAccessTier = azblob.AccessTierNone 55 maxTryTimeout = time.Hour * 24 * 365 //max time of an azure web request response window (whether or not data is flowing) 56 // Default storage account, key and blob endpoint for emulator support, 57 // though it is a base64 key checked in here, it is publicly available secret. 58 emulatorAccount = "devstoreaccount1" 59 emulatorAccountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" 60 emulatorBlobEndpoint = "http://127.0.0.1:10000/devstoreaccount1" 61 ) 62 63 // Register with Fs 64 func init() { 65 fs.Register(&fs.RegInfo{ 66 Name: "azureblob", 67 Description: "Microsoft Azure Blob Storage", 68 NewFs: NewFs, 69 Options: []fs.Option{{ 70 Name: "account", 71 Help: "Storage Account Name (leave blank to use SAS URL or Emulator)", 72 }, { 73 Name: "key", 74 Help: "Storage Account Key (leave blank to use SAS URL or Emulator)", 75 }, { 76 Name: "sas_url", 77 Help: "SAS URL for container level access only\n(leave blank if using account/key or Emulator)", 78 }, { 79 Name: "use_emulator", 80 Help: "Uses local storage emulator if provided as 'true' (leave blank if using real azure storage endpoint)", 81 Default: false, 82 }, { 83 Name: "endpoint", 84 Help: "Endpoint for the service\nLeave blank normally.", 85 Advanced: true, 86 }, { 87 Name: "upload_cutoff", 88 Help: "Cutoff for switching to chunked upload (<= 256MB).", 89 Default: defaultUploadCutoff, 90 Advanced: true, 91 }, { 92 Name: "chunk_size", 93 Help: `Upload chunk size (<= 100MB). 94 95 Note that this is stored in memory and there may be up to 96 "--transfers" chunks stored at once in memory.`, 97 Default: defaultChunkSize, 98 Advanced: true, 99 }, { 100 Name: "list_chunk", 101 Help: `Size of blob list. 102 103 This sets the number of blobs requested in each listing chunk. Default 104 is the maximum, 5000. "List blobs" requests are permitted 2 minutes 105 per megabyte to complete. If an operation is taking longer than 2 106 minutes per megabyte on average, it will time out ( 107 [source](https://docs.microsoft.com/en-us/rest/api/storageservices/setting-timeouts-for-blob-service-operations#exceptions-to-default-timeout-interval) 108 ). This can be used to limit the number of blobs items to return, to 109 avoid the time out.`, 110 Default: maxListChunkSize, 111 Advanced: true, 112 }, { 113 Name: "access_tier", 114 Help: `Access tier of blob: hot, cool or archive. 115 116 Archived blobs can be restored by setting access tier to hot or 117 cool. Leave blank if you intend to use default access tier, which is 118 set at account level 119 120 If there is no "access tier" specified, rclone doesn't apply any tier. 121 rclone performs "Set Tier" operation on blobs while uploading, if objects 122 are not modified, specifying "access tier" to new one will have no effect. 123 If blobs are in "archive tier" at remote, trying to perform data transfer 124 operations from remote will not be allowed. User should first restore by 125 tiering blob to "Hot" or "Cool".`, 126 Advanced: true, 127 }}, 128 }) 129 } 130 131 // Options defines the configuration for this backend 132 type Options struct { 133 Account string `config:"account"` 134 Key string `config:"key"` 135 Endpoint string `config:"endpoint"` 136 SASURL string `config:"sas_url"` 137 UploadCutoff fs.SizeSuffix `config:"upload_cutoff"` 138 ChunkSize fs.SizeSuffix `config:"chunk_size"` 139 ListChunkSize uint `config:"list_chunk"` 140 AccessTier string `config:"access_tier"` 141 UseEmulator bool `config:"use_emulator"` 142 } 143 144 // Fs represents a remote azure server 145 type Fs struct { 146 name string // name of this remote 147 root string // the path we are working on if any 148 opt Options // parsed config options 149 features *fs.Features // optional features 150 client *http.Client // http client we are using 151 svcURL *azblob.ServiceURL // reference to serviceURL 152 cntURL *azblob.ContainerURL // reference to containerURL 153 container string // the container we are working on 154 containerOKMu sync.Mutex // mutex to protect container OK 155 containerOK bool // true if we have created the container 156 containerDeleted bool // true if we have deleted the container 157 pacer *fs.Pacer // To pace and retry the API calls 158 uploadToken *pacer.TokenDispenser // control concurrency 159 } 160 161 // Object describes a azure object 162 type Object struct { 163 fs *Fs // what this object is part of 164 remote string // The remote path 165 modTime time.Time // The modified time of the object if known 166 md5 string // MD5 hash if known 167 size int64 // Size of the object 168 mimeType string // Content-Type of the object 169 accessTier azblob.AccessTierType // Blob Access Tier 170 meta map[string]string // blob metadata 171 } 172 173 // ------------------------------------------------------------ 174 175 // Name of the remote (as passed into NewFs) 176 func (f *Fs) Name() string { 177 return f.name 178 } 179 180 // Root of the remote (as passed into NewFs) 181 func (f *Fs) Root() string { 182 if f.root == "" { 183 return f.container 184 } 185 return f.container + "/" + f.root 186 } 187 188 // String converts this Fs to a string 189 func (f *Fs) String() string { 190 if f.root == "" { 191 return fmt.Sprintf("Azure container %s", f.container) 192 } 193 return fmt.Sprintf("Azure container %s path %s", f.container, f.root) 194 } 195 196 // Features returns the optional features of this Fs 197 func (f *Fs) Features() *fs.Features { 198 return f.features 199 } 200 201 // Pattern to match a azure path 202 var matcher = regexp.MustCompile(`^/*([^/]*)(.*)$`) 203 204 // parseParse parses a azure 'url' 205 func parsePath(path string) (container, directory string, err error) { 206 parts := matcher.FindStringSubmatch(path) 207 if parts == nil { 208 err = errors.Errorf("couldn't find container in azure path %q", path) 209 } else { 210 container, directory = parts[1], parts[2] 211 directory = strings.Trim(directory, "/") 212 } 213 return 214 } 215 216 // validateAccessTier checks if azureblob supports user supplied tier 217 func validateAccessTier(tier string) bool { 218 switch tier { 219 case string(azblob.AccessTierHot), 220 string(azblob.AccessTierCool), 221 string(azblob.AccessTierArchive): 222 // valid cases 223 return true 224 default: 225 return false 226 } 227 } 228 229 // retryErrorCodes is a slice of error codes that we will retry 230 var retryErrorCodes = []int{ 231 401, // Unauthorized (eg "Token has expired") 232 408, // Request Timeout 233 429, // Rate exceeded. 234 500, // Get occasional 500 Internal Server Error 235 503, // Service Unavailable 236 504, // Gateway Time-out 237 } 238 239 // shouldRetry returns a boolean as to whether this resp and err 240 // deserve to be retried. It returns the err as a convenience 241 func (f *Fs) shouldRetry(err error) (bool, error) { 242 // FIXME interpret special errors - more to do here 243 if storageErr, ok := err.(azblob.StorageError); ok { 244 statusCode := storageErr.Response().StatusCode 245 for _, e := range retryErrorCodes { 246 if statusCode == e { 247 return true, err 248 } 249 } 250 } 251 return fserrors.ShouldRetry(err), err 252 } 253 254 func checkUploadChunkSize(cs fs.SizeSuffix) error { 255 const minChunkSize = fs.Byte 256 if cs < minChunkSize { 257 return errors.Errorf("%s is less than %s", cs, minChunkSize) 258 } 259 if cs > maxChunkSize { 260 return errors.Errorf("%s is greater than %s", cs, maxChunkSize) 261 } 262 return nil 263 } 264 265 func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) { 266 err = checkUploadChunkSize(cs) 267 if err == nil { 268 old, f.opt.ChunkSize = f.opt.ChunkSize, cs 269 } 270 return 271 } 272 273 func checkUploadCutoff(cs fs.SizeSuffix) error { 274 if cs > maxUploadCutoff { 275 return errors.Errorf("%v must be less than or equal to %v", cs, maxUploadCutoff) 276 } 277 return nil 278 } 279 280 func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) { 281 err = checkUploadCutoff(cs) 282 if err == nil { 283 old, f.opt.UploadCutoff = f.opt.UploadCutoff, cs 284 } 285 return 286 } 287 288 // httpClientFactory creates a Factory object that sends HTTP requests 289 // to a rclone's http.Client. 290 // 291 // copied from azblob.newDefaultHTTPClientFactory 292 func httpClientFactory(client *http.Client) pipeline.Factory { 293 return pipeline.FactoryFunc(func(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.PolicyFunc { 294 return func(ctx context.Context, request pipeline.Request) (pipeline.Response, error) { 295 r, err := client.Do(request.WithContext(ctx)) 296 if err != nil { 297 err = pipeline.NewError(err, "HTTP request failed") 298 } 299 return pipeline.NewHTTPResponse(r), err 300 } 301 }) 302 } 303 304 // newPipeline creates a Pipeline using the specified credentials and options. 305 // 306 // this code was copied from azblob.NewPipeline 307 func (f *Fs) newPipeline(c azblob.Credential, o azblob.PipelineOptions) pipeline.Pipeline { 308 // Closest to API goes first; closest to the wire goes last 309 factories := []pipeline.Factory{ 310 azblob.NewTelemetryPolicyFactory(o.Telemetry), 311 azblob.NewUniqueRequestIDPolicyFactory(), 312 azblob.NewRetryPolicyFactory(o.Retry), 313 c, 314 pipeline.MethodFactoryMarker(), // indicates at what stage in the pipeline the method factory is invoked 315 azblob.NewRequestLogPolicyFactory(o.RequestLog), 316 } 317 return pipeline.NewPipeline(factories, pipeline.Options{HTTPSender: httpClientFactory(f.client), Log: o.Log}) 318 } 319 320 // NewFs constructs an Fs from the path, container:path 321 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { 322 ctx := context.Background() 323 // Parse config into Options struct 324 opt := new(Options) 325 err := configstruct.Set(m, opt) 326 if err != nil { 327 return nil, err 328 } 329 330 err = checkUploadCutoff(opt.UploadCutoff) 331 if err != nil { 332 return nil, errors.Wrap(err, "azure: upload cutoff") 333 } 334 err = checkUploadChunkSize(opt.ChunkSize) 335 if err != nil { 336 return nil, errors.Wrap(err, "azure: chunk size") 337 } 338 if opt.ListChunkSize > maxListChunkSize { 339 return nil, errors.Errorf("azure: blob list size can't be greater than %v - was %v", maxListChunkSize, opt.ListChunkSize) 340 } 341 container, directory, err := parsePath(root) 342 if err != nil { 343 return nil, err 344 } 345 if opt.Endpoint == "" { 346 opt.Endpoint = storageDefaultBaseURL 347 } 348 349 if opt.AccessTier == "" { 350 opt.AccessTier = string(defaultAccessTier) 351 } else if !validateAccessTier(opt.AccessTier) { 352 return nil, errors.Errorf("Azure Blob: Supported access tiers are %s, %s and %s", 353 string(azblob.AccessTierHot), string(azblob.AccessTierCool), string(azblob.AccessTierArchive)) 354 } 355 356 f := &Fs{ 357 name: name, 358 opt: *opt, 359 container: container, 360 root: directory, 361 pacer: fs.NewPacer(pacer.NewS3(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))), 362 uploadToken: pacer.NewTokenDispenser(fs.Config.Transfers), 363 client: fshttp.NewClient(fs.Config), 364 } 365 f.features = (&fs.Features{ 366 ReadMimeType: true, 367 WriteMimeType: true, 368 BucketBased: true, 369 SetTier: true, 370 GetTier: true, 371 }).Fill(f) 372 373 var ( 374 u *url.URL 375 serviceURL azblob.ServiceURL 376 containerURL azblob.ContainerURL 377 ) 378 switch { 379 case opt.UseEmulator: 380 credential, err := azblob.NewSharedKeyCredential(emulatorAccount, emulatorAccountKey) 381 if err != nil { 382 return nil, errors.Wrapf(err, "Failed to parse credentials") 383 } 384 u, err = url.Parse(emulatorBlobEndpoint) 385 if err != nil { 386 return nil, errors.Wrap(err, "failed to make azure storage url from account and endpoint") 387 } 388 pipeline := f.newPipeline(credential, azblob.PipelineOptions{Retry: azblob.RetryOptions{TryTimeout: maxTryTimeout}}) 389 serviceURL = azblob.NewServiceURL(*u, pipeline) 390 containerURL = serviceURL.NewContainerURL(container) 391 case opt.Account != "" && opt.Key != "": 392 credential, err := azblob.NewSharedKeyCredential(opt.Account, opt.Key) 393 if err != nil { 394 return nil, errors.Wrapf(err, "Failed to parse credentials") 395 } 396 397 u, err = url.Parse(fmt.Sprintf("https://%s.%s", opt.Account, opt.Endpoint)) 398 if err != nil { 399 return nil, errors.Wrap(err, "failed to make azure storage url from account and endpoint") 400 } 401 pipeline := f.newPipeline(credential, azblob.PipelineOptions{Retry: azblob.RetryOptions{TryTimeout: maxTryTimeout}}) 402 serviceURL = azblob.NewServiceURL(*u, pipeline) 403 containerURL = serviceURL.NewContainerURL(container) 404 case opt.SASURL != "": 405 u, err = url.Parse(opt.SASURL) 406 if err != nil { 407 return nil, errors.Wrapf(err, "failed to parse SAS URL") 408 } 409 // use anonymous credentials in case of sas url 410 pipeline := f.newPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{Retry: azblob.RetryOptions{TryTimeout: maxTryTimeout}}) 411 // Check if we have container level SAS or account level sas 412 parts := azblob.NewBlobURLParts(*u) 413 if parts.ContainerName != "" { 414 if container != "" && parts.ContainerName != container { 415 return nil, errors.New("Container name in SAS URL and container provided in command do not match") 416 } 417 418 f.container = parts.ContainerName 419 containerURL = azblob.NewContainerURL(*u, pipeline) 420 } else { 421 serviceURL = azblob.NewServiceURL(*u, pipeline) 422 containerURL = serviceURL.NewContainerURL(container) 423 } 424 default: 425 return nil, errors.New("Need account+key or connectionString or sasURL") 426 } 427 f.svcURL = &serviceURL 428 f.cntURL = &containerURL 429 430 if f.root != "" { 431 f.root += "/" 432 // Check to see if the (container,directory) is actually an existing file 433 oldRoot := f.root 434 remote := path.Base(directory) 435 f.root = path.Dir(directory) 436 if f.root == "." { 437 f.root = "" 438 } else { 439 f.root += "/" 440 } 441 _, err := f.NewObject(ctx, remote) 442 if err != nil { 443 if err == fs.ErrorObjectNotFound || err == fs.ErrorNotAFile { 444 // File doesn't exist or is a directory so return old f 445 f.root = oldRoot 446 return f, nil 447 } 448 return nil, err 449 } 450 // return an error with an fs which points to the parent 451 return f, fs.ErrorIsFile 452 } 453 return f, nil 454 } 455 456 // Return an Object from a path 457 // 458 // If it can't be found it returns the error fs.ErrorObjectNotFound. 459 func (f *Fs) newObjectWithInfo(remote string, info *azblob.BlobItem) (fs.Object, error) { 460 o := &Object{ 461 fs: f, 462 remote: remote, 463 } 464 if info != nil { 465 err := o.decodeMetaDataFromBlob(info) 466 if err != nil { 467 return nil, err 468 } 469 } else { 470 err := o.readMetaData() // reads info and headers, returning an error 471 if err != nil { 472 return nil, err 473 } 474 } 475 return o, nil 476 } 477 478 // NewObject finds the Object at remote. If it can't be found 479 // it returns the error fs.ErrorObjectNotFound. 480 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { 481 return f.newObjectWithInfo(remote, nil) 482 } 483 484 // getBlobReference creates an empty blob reference with no metadata 485 func (f *Fs) getBlobReference(remote string) azblob.BlobURL { 486 return f.cntURL.NewBlobURL(f.root + remote) 487 } 488 489 // updateMetadataWithModTime adds the modTime passed in to o.meta. 490 func (o *Object) updateMetadataWithModTime(modTime time.Time) { 491 // Make sure o.meta is not nil 492 if o.meta == nil { 493 o.meta = make(map[string]string, 1) 494 } 495 496 // Set modTimeKey in it 497 o.meta[modTimeKey] = modTime.Format(timeFormatOut) 498 } 499 500 // Returns whether file is a directory marker or not 501 func isDirectoryMarker(size int64, metadata azblob.Metadata, remote string) bool { 502 // Directory markers are 0 length 503 if size == 0 { 504 // Note that metadata with hdi_isfolder = true seems to be a 505 // defacto standard for marking blobs as directories. 506 endsWithSlash := strings.HasSuffix(remote, "/") 507 if endsWithSlash || remote == "" || metadata["hdi_isfolder"] == "true" { 508 return true 509 } 510 511 } 512 return false 513 } 514 515 // listFn is called from list to handle an object 516 type listFn func(remote string, object *azblob.BlobItem, isDirectory bool) error 517 518 // list lists the objects into the function supplied from 519 // the container and root supplied 520 // 521 // dir is the starting directory, "" for root 522 func (f *Fs) list(ctx context.Context, dir string, recurse bool, maxResults uint, fn listFn) error { 523 f.containerOKMu.Lock() 524 deleted := f.containerDeleted 525 f.containerOKMu.Unlock() 526 if deleted { 527 return fs.ErrorDirNotFound 528 } 529 root := f.root 530 if dir != "" { 531 root += dir + "/" 532 } 533 delimiter := "" 534 if !recurse { 535 delimiter = "/" 536 } 537 538 options := azblob.ListBlobsSegmentOptions{ 539 Details: azblob.BlobListingDetails{ 540 Copy: false, 541 Metadata: true, 542 Snapshots: false, 543 UncommittedBlobs: false, 544 Deleted: false, 545 }, 546 Prefix: root, 547 MaxResults: int32(maxResults), 548 } 549 directoryMarkers := map[string]struct{}{} 550 for marker := (azblob.Marker{}); marker.NotDone(); { 551 var response *azblob.ListBlobsHierarchySegmentResponse 552 err := f.pacer.Call(func() (bool, error) { 553 var err error 554 response, err = f.cntURL.ListBlobsHierarchySegment(ctx, marker, delimiter, options) 555 return f.shouldRetry(err) 556 }) 557 558 if err != nil { 559 // Check http error code along with service code, current SDK doesn't populate service code correctly sometimes 560 if storageErr, ok := err.(azblob.StorageError); ok && (storageErr.ServiceCode() == azblob.ServiceCodeContainerNotFound || storageErr.Response().StatusCode == http.StatusNotFound) { 561 return fs.ErrorDirNotFound 562 } 563 return err 564 } 565 // Advance marker to next 566 marker = response.NextMarker 567 568 for i := range response.Segment.BlobItems { 569 file := &response.Segment.BlobItems[i] 570 // Finish if file name no longer has prefix 571 // if prefix != "" && !strings.HasPrefix(file.Name, prefix) { 572 // return nil 573 // } 574 if !strings.HasPrefix(file.Name, f.root) { 575 fs.Debugf(f, "Odd name received %q", file.Name) 576 continue 577 } 578 remote := file.Name[len(f.root):] 579 if isDirectoryMarker(*file.Properties.ContentLength, file.Metadata, remote) { 580 if strings.HasSuffix(remote, "/") { 581 remote = remote[:len(remote)-1] 582 } 583 err = fn(remote, file, true) 584 if err != nil { 585 return err 586 } 587 // Keep track of directory markers. If recursing then 588 // there will be no Prefixes so no need to keep track 589 if !recurse { 590 directoryMarkers[remote] = struct{}{} 591 } 592 continue // skip directory marker 593 } 594 // Send object 595 err = fn(remote, file, false) 596 if err != nil { 597 return err 598 } 599 } 600 // Send the subdirectories 601 for _, remote := range response.Segment.BlobPrefixes { 602 remote := strings.TrimRight(remote.Name, "/") 603 if !strings.HasPrefix(remote, f.root) { 604 fs.Debugf(f, "Odd directory name received %q", remote) 605 continue 606 } 607 remote = remote[len(f.root):] 608 // Don't send if already sent as a directory marker 609 if _, found := directoryMarkers[remote]; found { 610 continue 611 } 612 // Send object 613 err = fn(remote, nil, true) 614 if err != nil { 615 return err 616 } 617 } 618 } 619 return nil 620 } 621 622 // Convert a list item into a DirEntry 623 func (f *Fs) itemToDirEntry(remote string, object *azblob.BlobItem, isDirectory bool) (fs.DirEntry, error) { 624 if isDirectory { 625 d := fs.NewDir(remote, time.Time{}) 626 return d, nil 627 } 628 o, err := f.newObjectWithInfo(remote, object) 629 if err != nil { 630 return nil, err 631 } 632 return o, nil 633 } 634 635 // mark the container as being OK 636 func (f *Fs) markContainerOK() { 637 if f.container != "" { 638 f.containerOKMu.Lock() 639 f.containerOK = true 640 f.containerDeleted = false 641 f.containerOKMu.Unlock() 642 } 643 } 644 645 // listDir lists a single directory 646 func (f *Fs) listDir(ctx context.Context, dir string) (entries fs.DirEntries, err error) { 647 err = f.list(ctx, dir, false, f.opt.ListChunkSize, func(remote string, object *azblob.BlobItem, isDirectory bool) error { 648 entry, err := f.itemToDirEntry(remote, object, isDirectory) 649 if err != nil { 650 return err 651 } 652 if entry != nil { 653 entries = append(entries, entry) 654 } 655 return nil 656 }) 657 if err != nil { 658 return nil, err 659 } 660 // container must be present if listing succeeded 661 f.markContainerOK() 662 return entries, nil 663 } 664 665 // listContainers returns all the containers to out 666 func (f *Fs) listContainers(dir string) (entries fs.DirEntries, err error) { 667 if dir != "" { 668 return nil, fs.ErrorListBucketRequired 669 } 670 err = f.listContainersToFn(func(container *azblob.ContainerItem) error { 671 d := fs.NewDir(container.Name, container.Properties.LastModified) 672 entries = append(entries, d) 673 return nil 674 }) 675 if err != nil { 676 return nil, err 677 } 678 return entries, nil 679 } 680 681 // List the objects and directories in dir into entries. The 682 // entries can be returned in any order but should be for a 683 // complete directory. 684 // 685 // dir should be "" to list the root, and should not have 686 // trailing slashes. 687 // 688 // This should return ErrDirNotFound if the directory isn't 689 // found. 690 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { 691 if f.container == "" { 692 return f.listContainers(dir) 693 } 694 return f.listDir(ctx, dir) 695 } 696 697 // ListR lists the objects and directories of the Fs starting 698 // from dir recursively into out. 699 // 700 // dir should be "" to start from the root, and should not 701 // have trailing slashes. 702 // 703 // This should return ErrDirNotFound if the directory isn't 704 // found. 705 // 706 // It should call callback for each tranche of entries read. 707 // These need not be returned in any particular order. If 708 // callback returns an error then the listing will stop 709 // immediately. 710 // 711 // Don't implement this unless you have a more efficient way 712 // of listing recursively that doing a directory traversal. 713 func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) { 714 if f.container == "" { 715 return fs.ErrorListBucketRequired 716 } 717 list := walk.NewListRHelper(callback) 718 err = f.list(ctx, dir, true, f.opt.ListChunkSize, func(remote string, object *azblob.BlobItem, isDirectory bool) error { 719 entry, err := f.itemToDirEntry(remote, object, isDirectory) 720 if err != nil { 721 return err 722 } 723 return list.Add(entry) 724 }) 725 if err != nil { 726 return err 727 } 728 // container must be present if listing succeeded 729 f.markContainerOK() 730 return list.Flush() 731 } 732 733 // listContainerFn is called from listContainersToFn to handle a container 734 type listContainerFn func(*azblob.ContainerItem) error 735 736 // listContainersToFn lists the containers to the function supplied 737 func (f *Fs) listContainersToFn(fn listContainerFn) error { 738 params := azblob.ListContainersSegmentOptions{ 739 MaxResults: int32(f.opt.ListChunkSize), 740 } 741 ctx := context.Background() 742 for marker := (azblob.Marker{}); marker.NotDone(); { 743 var response *azblob.ListContainersSegmentResponse 744 err := f.pacer.Call(func() (bool, error) { 745 var err error 746 response, err = f.svcURL.ListContainersSegment(ctx, marker, params) 747 return f.shouldRetry(err) 748 }) 749 if err != nil { 750 return err 751 } 752 753 for i := range response.ContainerItems { 754 err = fn(&response.ContainerItems[i]) 755 if err != nil { 756 return err 757 } 758 } 759 marker = response.NextMarker 760 } 761 762 return nil 763 } 764 765 // Put the object into the container 766 // 767 // Copy the reader in to the new object which is returned 768 // 769 // The new object may have been created if an error is returned 770 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { 771 // Temporary Object under construction 772 fs := &Object{ 773 fs: f, 774 remote: src.Remote(), 775 } 776 return fs, fs.Update(ctx, in, src, options...) 777 } 778 779 // Check if the container exists 780 // 781 // NB this can return incorrect results if called immediately after container deletion 782 func (f *Fs) dirExists() (bool, error) { 783 options := azblob.ListBlobsSegmentOptions{ 784 Details: azblob.BlobListingDetails{ 785 Copy: false, 786 Metadata: false, 787 Snapshots: false, 788 UncommittedBlobs: false, 789 Deleted: false, 790 }, 791 MaxResults: 1, 792 } 793 err := f.pacer.Call(func() (bool, error) { 794 ctx := context.Background() 795 _, err := f.cntURL.ListBlobsHierarchySegment(ctx, azblob.Marker{}, "", options) 796 return f.shouldRetry(err) 797 }) 798 if err == nil { 799 return true, nil 800 } 801 // Check http error code along with service code, current SDK doesn't populate service code correctly sometimes 802 if storageErr, ok := err.(azblob.StorageError); ok && (storageErr.ServiceCode() == azblob.ServiceCodeContainerNotFound || storageErr.Response().StatusCode == http.StatusNotFound) { 803 return false, nil 804 } 805 return false, err 806 } 807 808 // Mkdir creates the container if it doesn't exist 809 func (f *Fs) Mkdir(ctx context.Context, dir string) error { 810 f.containerOKMu.Lock() 811 defer f.containerOKMu.Unlock() 812 if f.containerOK { 813 return nil 814 } 815 if !f.containerDeleted { 816 exists, err := f.dirExists() 817 if err == nil { 818 f.containerOK = exists 819 } 820 if err != nil || exists { 821 return err 822 } 823 } 824 825 // now try to create the container 826 err := f.pacer.Call(func() (bool, error) { 827 ctx := context.Background() 828 _, err := f.cntURL.Create(ctx, azblob.Metadata{}, azblob.PublicAccessNone) 829 if err != nil { 830 if storageErr, ok := err.(azblob.StorageError); ok { 831 switch storageErr.ServiceCode() { 832 case azblob.ServiceCodeContainerAlreadyExists: 833 f.containerOK = true 834 return false, nil 835 case azblob.ServiceCodeContainerBeingDeleted: 836 // From https://docs.microsoft.com/en-us/rest/api/storageservices/delete-container 837 // When a container is deleted, a container with the same name cannot be created 838 // for at least 30 seconds; the container may not be available for more than 30 839 // seconds if the service is still processing the request. 840 time.Sleep(6 * time.Second) // default 10 retries will be 60 seconds 841 f.containerDeleted = true 842 return true, err 843 } 844 } 845 } 846 return f.shouldRetry(err) 847 }) 848 if err == nil { 849 f.containerOK = true 850 f.containerDeleted = false 851 } 852 return errors.Wrap(err, "failed to make container") 853 } 854 855 // isEmpty checks to see if a given directory is empty and returns an error if not 856 func (f *Fs) isEmpty(ctx context.Context, dir string) (err error) { 857 empty := true 858 err = f.list(ctx, dir, true, 1, func(remote string, object *azblob.BlobItem, isDirectory bool) error { 859 empty = false 860 return nil 861 }) 862 if err != nil { 863 return err 864 } 865 if !empty { 866 return fs.ErrorDirectoryNotEmpty 867 } 868 return nil 869 } 870 871 // deleteContainer deletes the container. It can delete a full 872 // container so use isEmpty if you don't want that. 873 func (f *Fs) deleteContainer() error { 874 f.containerOKMu.Lock() 875 defer f.containerOKMu.Unlock() 876 options := azblob.ContainerAccessConditions{} 877 ctx := context.Background() 878 err := f.pacer.Call(func() (bool, error) { 879 _, err := f.cntURL.GetProperties(ctx, azblob.LeaseAccessConditions{}) 880 if err == nil { 881 _, err = f.cntURL.Delete(ctx, options) 882 } 883 884 if err != nil { 885 // Check http error code along with service code, current SDK doesn't populate service code correctly sometimes 886 if storageErr, ok := err.(azblob.StorageError); ok && (storageErr.ServiceCode() == azblob.ServiceCodeContainerNotFound || storageErr.Response().StatusCode == http.StatusNotFound) { 887 return false, fs.ErrorDirNotFound 888 } 889 890 return f.shouldRetry(err) 891 } 892 893 return f.shouldRetry(err) 894 }) 895 if err == nil { 896 f.containerOK = false 897 f.containerDeleted = true 898 } 899 return errors.Wrap(err, "failed to delete container") 900 } 901 902 // Rmdir deletes the container if the fs is at the root 903 // 904 // Returns an error if it isn't empty 905 func (f *Fs) Rmdir(ctx context.Context, dir string) error { 906 err := f.isEmpty(ctx, dir) 907 if err != nil { 908 return err 909 } 910 if f.root != "" || dir != "" { 911 return nil 912 } 913 return f.deleteContainer() 914 } 915 916 // Precision of the remote 917 func (f *Fs) Precision() time.Duration { 918 return time.Nanosecond 919 } 920 921 // Hashes returns the supported hash sets. 922 func (f *Fs) Hashes() hash.Set { 923 return hash.Set(hash.MD5) 924 } 925 926 // Purge deletes all the files and directories including the old versions. 927 func (f *Fs) Purge(ctx context.Context) error { 928 dir := "" // forward compat! 929 if f.root != "" || dir != "" { 930 // Delegate to caller if not root container 931 return fs.ErrorCantPurge 932 } 933 return f.deleteContainer() 934 } 935 936 // Copy src to this remote using server side copy operations. 937 // 938 // This is stored with the remote path given 939 // 940 // It returns the destination Object and a possible error 941 // 942 // Will only be called if src.Fs().Name() == f.Name() 943 // 944 // If it isn't possible then return fs.ErrorCantCopy 945 func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { 946 err := f.Mkdir(ctx, "") 947 if err != nil { 948 return nil, err 949 } 950 srcObj, ok := src.(*Object) 951 if !ok { 952 fs.Debugf(src, "Can't copy - not same remote type") 953 return nil, fs.ErrorCantCopy 954 } 955 dstBlobURL := f.getBlobReference(remote) 956 srcBlobURL := srcObj.getBlobReference() 957 958 source, err := url.Parse(srcBlobURL.String()) 959 if err != nil { 960 return nil, err 961 } 962 963 options := azblob.BlobAccessConditions{} 964 var startCopy *azblob.BlobStartCopyFromURLResponse 965 966 err = f.pacer.Call(func() (bool, error) { 967 startCopy, err = dstBlobURL.StartCopyFromURL(ctx, *source, nil, azblob.ModifiedAccessConditions{}, options) 968 return f.shouldRetry(err) 969 }) 970 if err != nil { 971 return nil, err 972 } 973 974 copyStatus := startCopy.CopyStatus() 975 for copyStatus == azblob.CopyStatusPending { 976 time.Sleep(1 * time.Second) 977 getMetadata, err := dstBlobURL.GetProperties(ctx, options) 978 if err != nil { 979 return nil, err 980 } 981 copyStatus = getMetadata.CopyStatus() 982 } 983 984 return f.NewObject(ctx, remote) 985 } 986 987 // ------------------------------------------------------------ 988 989 // Fs returns the parent Fs 990 func (o *Object) Fs() fs.Info { 991 return o.fs 992 } 993 994 // Return a string version 995 func (o *Object) String() string { 996 if o == nil { 997 return "<nil>" 998 } 999 return o.remote 1000 } 1001 1002 // Remote returns the remote path 1003 func (o *Object) Remote() string { 1004 return o.remote 1005 } 1006 1007 // Hash returns the MD5 of an object returning a lowercase hex string 1008 func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) { 1009 if t != hash.MD5 { 1010 return "", hash.ErrUnsupported 1011 } 1012 // Convert base64 encoded md5 into lower case hex 1013 if o.md5 == "" { 1014 return "", nil 1015 } 1016 data, err := base64.StdEncoding.DecodeString(o.md5) 1017 if err != nil { 1018 return "", errors.Wrapf(err, "Failed to decode Content-MD5: %q", o.md5) 1019 } 1020 return hex.EncodeToString(data), nil 1021 } 1022 1023 // Size returns the size of an object in bytes 1024 func (o *Object) Size() int64 { 1025 return o.size 1026 } 1027 1028 func (o *Object) setMetadata(metadata azblob.Metadata) { 1029 if len(metadata) > 0 { 1030 o.meta = metadata 1031 if modTime, ok := metadata[modTimeKey]; ok { 1032 when, err := time.Parse(timeFormatIn, modTime) 1033 if err != nil { 1034 fs.Debugf(o, "Couldn't parse %v = %q: %v", modTimeKey, modTime, err) 1035 } 1036 o.modTime = when 1037 } 1038 } else { 1039 o.meta = nil 1040 } 1041 } 1042 1043 // decodeMetaDataFromPropertiesResponse sets the metadata from the data passed in 1044 // 1045 // Sets 1046 // o.id 1047 // o.modTime 1048 // o.size 1049 // o.md5 1050 // o.meta 1051 func (o *Object) decodeMetaDataFromPropertiesResponse(info *azblob.BlobGetPropertiesResponse) (err error) { 1052 metadata := info.NewMetadata() 1053 size := info.ContentLength() 1054 if isDirectoryMarker(size, metadata, o.remote) { 1055 return fs.ErrorNotAFile 1056 } 1057 // NOTE - Client library always returns MD5 as base64 decoded string, Object needs to maintain 1058 // this as base64 encoded string. 1059 o.md5 = base64.StdEncoding.EncodeToString(info.ContentMD5()) 1060 o.mimeType = info.ContentType() 1061 o.size = size 1062 o.modTime = info.LastModified() 1063 o.accessTier = azblob.AccessTierType(info.AccessTier()) 1064 o.setMetadata(metadata) 1065 1066 return nil 1067 } 1068 1069 func (o *Object) decodeMetaDataFromBlob(info *azblob.BlobItem) (err error) { 1070 metadata := info.Metadata 1071 size := *info.Properties.ContentLength 1072 if isDirectoryMarker(size, metadata, o.remote) { 1073 return fs.ErrorNotAFile 1074 } 1075 // NOTE - Client library always returns MD5 as base64 decoded string, Object needs to maintain 1076 // this as base64 encoded string. 1077 o.md5 = base64.StdEncoding.EncodeToString(info.Properties.ContentMD5) 1078 o.mimeType = *info.Properties.ContentType 1079 o.size = size 1080 o.modTime = info.Properties.LastModified 1081 o.accessTier = info.Properties.AccessTier 1082 o.setMetadata(metadata) 1083 return nil 1084 } 1085 1086 // getBlobReference creates an empty blob reference with no metadata 1087 func (o *Object) getBlobReference() azblob.BlobURL { 1088 return o.fs.getBlobReference(o.remote) 1089 } 1090 1091 // clearMetaData clears enough metadata so readMetaData will re-read it 1092 func (o *Object) clearMetaData() { 1093 o.modTime = time.Time{} 1094 } 1095 1096 // readMetaData gets the metadata if it hasn't already been fetched 1097 // 1098 // Sets 1099 // o.id 1100 // o.modTime 1101 // o.size 1102 // o.md5 1103 func (o *Object) readMetaData() (err error) { 1104 if !o.modTime.IsZero() { 1105 return nil 1106 } 1107 blob := o.getBlobReference() 1108 1109 // Read metadata (this includes metadata) 1110 options := azblob.BlobAccessConditions{} 1111 ctx := context.Background() 1112 var blobProperties *azblob.BlobGetPropertiesResponse 1113 err = o.fs.pacer.Call(func() (bool, error) { 1114 blobProperties, err = blob.GetProperties(ctx, options) 1115 return o.fs.shouldRetry(err) 1116 }) 1117 if err != nil { 1118 // On directories - GetProperties does not work and current SDK does not populate service code correctly hence check regular http response as well 1119 if storageErr, ok := err.(azblob.StorageError); ok && (storageErr.ServiceCode() == azblob.ServiceCodeBlobNotFound || storageErr.Response().StatusCode == http.StatusNotFound) { 1120 return fs.ErrorObjectNotFound 1121 } 1122 return err 1123 } 1124 1125 return o.decodeMetaDataFromPropertiesResponse(blobProperties) 1126 } 1127 1128 // parseTimeString converts a decimal string number of milliseconds 1129 // elapsed since January 1, 1970 UTC into a time.Time and stores it in 1130 // the modTime variable. 1131 func (o *Object) parseTimeString(timeString string) (err error) { 1132 if timeString == "" { 1133 return nil 1134 } 1135 unixMilliseconds, err := strconv.ParseInt(timeString, 10, 64) 1136 if err != nil { 1137 fs.Debugf(o, "Failed to parse mod time string %q: %v", timeString, err) 1138 return err 1139 } 1140 o.modTime = time.Unix(unixMilliseconds/1E3, (unixMilliseconds%1E3)*1E6).UTC() 1141 return nil 1142 } 1143 1144 // ModTime returns the modification time of the object 1145 // 1146 // It attempts to read the objects mtime and if that isn't present the 1147 // LastModified returned in the http headers 1148 func (o *Object) ModTime(ctx context.Context) (result time.Time) { 1149 // The error is logged in readMetaData 1150 _ = o.readMetaData() 1151 return o.modTime 1152 } 1153 1154 // SetModTime sets the modification time of the local fs object 1155 func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error { 1156 // Make sure o.meta is not nil 1157 if o.meta == nil { 1158 o.meta = make(map[string]string, 1) 1159 } 1160 // Set modTimeKey in it 1161 o.meta[modTimeKey] = modTime.Format(timeFormatOut) 1162 1163 blob := o.getBlobReference() 1164 err := o.fs.pacer.Call(func() (bool, error) { 1165 _, err := blob.SetMetadata(ctx, o.meta, azblob.BlobAccessConditions{}) 1166 return o.fs.shouldRetry(err) 1167 }) 1168 if err != nil { 1169 return err 1170 } 1171 o.modTime = modTime 1172 return nil 1173 } 1174 1175 // Storable returns if this object is storable 1176 func (o *Object) Storable() bool { 1177 return true 1178 } 1179 1180 // Open an object for read 1181 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) { 1182 // Offset and Count for range download 1183 var offset int64 1184 var count int64 1185 if o.AccessTier() == azblob.AccessTierArchive { 1186 return nil, errors.Errorf("Blob in archive tier, you need to set tier to hot or cool first") 1187 } 1188 1189 for _, option := range options { 1190 switch x := option.(type) { 1191 case *fs.RangeOption: 1192 offset, count = x.Decode(o.size) 1193 if count < 0 { 1194 count = o.size - offset 1195 } 1196 case *fs.SeekOption: 1197 offset = x.Offset 1198 default: 1199 if option.Mandatory() { 1200 fs.Logf(o, "Unsupported mandatory option: %v", option) 1201 } 1202 } 1203 } 1204 blob := o.getBlobReference() 1205 ac := azblob.BlobAccessConditions{} 1206 var dowloadResponse *azblob.DownloadResponse 1207 err = o.fs.pacer.Call(func() (bool, error) { 1208 dowloadResponse, err = blob.Download(ctx, offset, count, ac, false) 1209 return o.fs.shouldRetry(err) 1210 }) 1211 if err != nil { 1212 return nil, errors.Wrap(err, "failed to open for download") 1213 } 1214 in = dowloadResponse.Body(azblob.RetryReaderOptions{}) 1215 return in, nil 1216 } 1217 1218 // dontEncode is the characters that do not need percent-encoding 1219 // 1220 // The characters that do not need percent-encoding are a subset of 1221 // the printable ASCII characters: upper-case letters, lower-case 1222 // letters, digits, ".", "_", "-", "/", "~", "!", "$", "'", "(", ")", 1223 // "*", ";", "=", ":", and "@". All other byte values in a UTF-8 must 1224 // be replaced with "%" and the two-digit hex value of the byte. 1225 const dontEncode = (`abcdefghijklmnopqrstuvwxyz` + 1226 `ABCDEFGHIJKLMNOPQRSTUVWXYZ` + 1227 `0123456789` + 1228 `._-/~!$'()*;=:@`) 1229 1230 // noNeedToEncode is a bitmap of characters which don't need % encoding 1231 var noNeedToEncode [256]bool 1232 1233 func init() { 1234 for _, c := range dontEncode { 1235 noNeedToEncode[c] = true 1236 } 1237 } 1238 1239 // readSeeker joins an io.Reader and an io.Seeker 1240 type readSeeker struct { 1241 io.Reader 1242 io.Seeker 1243 } 1244 1245 // uploadMultipart uploads a file using multipart upload 1246 // 1247 // Write a larger blob, using CreateBlockBlob, PutBlock, and PutBlockList. 1248 func (o *Object) uploadMultipart(in io.Reader, size int64, blob *azblob.BlobURL, httpHeaders *azblob.BlobHTTPHeaders) (err error) { 1249 // Calculate correct chunkSize 1250 chunkSize := int64(o.fs.opt.ChunkSize) 1251 var totalParts int64 1252 for { 1253 // Calculate number of parts 1254 var remainder int64 1255 totalParts, remainder = size/chunkSize, size%chunkSize 1256 if remainder != 0 { 1257 totalParts++ 1258 } 1259 if totalParts < maxTotalParts { 1260 break 1261 } 1262 // Double chunk size if the number of parts is too big 1263 chunkSize *= 2 1264 if chunkSize > int64(maxChunkSize) { 1265 return errors.Errorf("can't upload as it is too big %v - takes more than %d chunks of %v", fs.SizeSuffix(size), totalParts, fs.SizeSuffix(chunkSize/2)) 1266 } 1267 } 1268 fs.Debugf(o, "Multipart upload session started for %d parts of size %v", totalParts, fs.SizeSuffix(chunkSize)) 1269 1270 // https://godoc.org/github.com/Azure/azure-storage-blob-go/2017-07-29/azblob#example-BlockBlobURL 1271 // Utilities are cloned from above example 1272 // These helper functions convert a binary block ID to a base-64 string and vice versa 1273 // NOTE: The blockID must be <= 64 bytes and ALL blockIDs for the block must be the same length 1274 blockIDBinaryToBase64 := func(blockID []byte) string { return base64.StdEncoding.EncodeToString(blockID) } 1275 // These helper functions convert an int block ID to a base-64 string and vice versa 1276 blockIDIntToBase64 := func(blockID uint64) string { 1277 binaryBlockID := (&[8]byte{})[:] // All block IDs are 8 bytes long 1278 binary.LittleEndian.PutUint64(binaryBlockID, blockID) 1279 return blockIDBinaryToBase64(binaryBlockID) 1280 } 1281 1282 // block ID variables 1283 var ( 1284 rawID uint64 1285 blockID = "" // id in base64 encoded form 1286 blocks []string 1287 ) 1288 1289 // increment the blockID 1290 nextID := func() { 1291 rawID++ 1292 blockID = blockIDIntToBase64(rawID) 1293 blocks = append(blocks, blockID) 1294 } 1295 1296 // Get BlockBlobURL, we will use default pipeline here 1297 blockBlobURL := blob.ToBlockBlobURL() 1298 ctx := context.Background() 1299 ac := azblob.LeaseAccessConditions{} // Use default lease access conditions 1300 1301 // unwrap the accounting from the input, we use wrap to put it 1302 // back on after the buffering 1303 in, wrap := accounting.UnWrap(in) 1304 1305 // Upload the chunks 1306 remaining := size 1307 position := int64(0) 1308 errs := make(chan error, 1) 1309 var wg sync.WaitGroup 1310 outer: 1311 for part := 0; part < int(totalParts); part++ { 1312 // Check any errors 1313 select { 1314 case err = <-errs: 1315 break outer 1316 default: 1317 } 1318 1319 reqSize := remaining 1320 if reqSize >= chunkSize { 1321 reqSize = chunkSize 1322 } 1323 1324 // Make a block of memory 1325 buf := make([]byte, reqSize) 1326 1327 // Read the chunk 1328 _, err = io.ReadFull(in, buf) 1329 if err != nil { 1330 err = errors.Wrap(err, "multipart upload failed to read source") 1331 break outer 1332 } 1333 1334 // Transfer the chunk 1335 nextID() 1336 wg.Add(1) 1337 o.fs.uploadToken.Get() 1338 go func(part int, position int64, blockID string) { 1339 defer wg.Done() 1340 defer o.fs.uploadToken.Put() 1341 fs.Debugf(o, "Uploading part %d/%d offset %v/%v part size %v", part+1, totalParts, fs.SizeSuffix(position), fs.SizeSuffix(size), fs.SizeSuffix(chunkSize)) 1342 1343 // Upload the block, with MD5 for check 1344 md5sum := md5.Sum(buf) 1345 transactionalMD5 := md5sum[:] 1346 err = o.fs.pacer.Call(func() (bool, error) { 1347 bufferReader := bytes.NewReader(buf) 1348 wrappedReader := wrap(bufferReader) 1349 rs := readSeeker{wrappedReader, bufferReader} 1350 _, err = blockBlobURL.StageBlock(ctx, blockID, &rs, ac, transactionalMD5) 1351 return o.fs.shouldRetry(err) 1352 }) 1353 1354 if err != nil { 1355 err = errors.Wrap(err, "multipart upload failed to upload part") 1356 select { 1357 case errs <- err: 1358 default: 1359 } 1360 return 1361 } 1362 }(part, position, blockID) 1363 1364 // ready for next block 1365 remaining -= chunkSize 1366 position += chunkSize 1367 } 1368 wg.Wait() 1369 if err == nil { 1370 select { 1371 case err = <-errs: 1372 default: 1373 } 1374 } 1375 if err != nil { 1376 return err 1377 } 1378 1379 // Finalise the upload session 1380 err = o.fs.pacer.Call(func() (bool, error) { 1381 _, err := blockBlobURL.CommitBlockList(ctx, blocks, *httpHeaders, o.meta, azblob.BlobAccessConditions{}) 1382 return o.fs.shouldRetry(err) 1383 }) 1384 if err != nil { 1385 return errors.Wrap(err, "multipart upload failed to finalize") 1386 } 1387 return nil 1388 } 1389 1390 // Update the object with the contents of the io.Reader, modTime and size 1391 // 1392 // The new object may have been created if an error is returned 1393 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { 1394 err = o.fs.Mkdir(ctx, "") 1395 if err != nil { 1396 return err 1397 } 1398 size := src.Size() 1399 // Update Mod time 1400 o.updateMetadataWithModTime(src.ModTime(ctx)) 1401 if err != nil { 1402 return err 1403 } 1404 1405 blob := o.getBlobReference() 1406 httpHeaders := azblob.BlobHTTPHeaders{} 1407 httpHeaders.ContentType = fs.MimeType(ctx, o) 1408 // Compute the Content-MD5 of the file, for multiparts uploads it 1409 // will be set in PutBlockList API call using the 'x-ms-blob-content-md5' header 1410 // Note: If multipart, a MD5 checksum will also be computed for each uploaded block 1411 // in order to validate its integrity during transport 1412 if sourceMD5, _ := src.Hash(ctx, hash.MD5); sourceMD5 != "" { 1413 sourceMD5bytes, err := hex.DecodeString(sourceMD5) 1414 if err == nil { 1415 httpHeaders.ContentMD5 = sourceMD5bytes 1416 } else { 1417 fs.Debugf(o, "Failed to decode %q as MD5: %v", sourceMD5, err) 1418 } 1419 } 1420 1421 putBlobOptions := azblob.UploadStreamToBlockBlobOptions{ 1422 BufferSize: int(o.fs.opt.ChunkSize), 1423 MaxBuffers: 4, 1424 Metadata: o.meta, 1425 BlobHTTPHeaders: httpHeaders, 1426 } 1427 // FIXME Until https://github.com/Azure/azure-storage-blob-go/pull/75 1428 // is merged the SDK can't upload a single blob of exactly the chunk 1429 // size, so upload with a multpart upload to work around. 1430 // See: https://github.com/ncw/rclone/issues/2653 1431 multipartUpload := size >= int64(o.fs.opt.UploadCutoff) 1432 if size == int64(o.fs.opt.ChunkSize) { 1433 multipartUpload = true 1434 fs.Debugf(o, "Setting multipart upload for file of chunk size (%d) to work around SDK bug", size) 1435 } 1436 1437 // Don't retry, return a retry error instead 1438 err = o.fs.pacer.CallNoRetry(func() (bool, error) { 1439 if multipartUpload { 1440 // If a large file upload in chunks 1441 err = o.uploadMultipart(in, size, &blob, &httpHeaders) 1442 } else { 1443 // Write a small blob in one transaction 1444 blockBlobURL := blob.ToBlockBlobURL() 1445 _, err = azblob.UploadStreamToBlockBlob(ctx, in, blockBlobURL, putBlobOptions) 1446 } 1447 return o.fs.shouldRetry(err) 1448 }) 1449 if err != nil { 1450 return err 1451 } 1452 // Refresh metadata on object 1453 o.clearMetaData() 1454 err = o.readMetaData() 1455 if err != nil { 1456 return err 1457 } 1458 1459 // If tier is not changed or not specified, do not attempt to invoke `SetBlobTier` operation 1460 if o.fs.opt.AccessTier == string(defaultAccessTier) || o.fs.opt.AccessTier == string(o.AccessTier()) { 1461 return nil 1462 } 1463 1464 // Now, set blob tier based on configured access tier 1465 return o.SetTier(o.fs.opt.AccessTier) 1466 } 1467 1468 // Remove an object 1469 func (o *Object) Remove(ctx context.Context) error { 1470 blob := o.getBlobReference() 1471 snapShotOptions := azblob.DeleteSnapshotsOptionNone 1472 ac := azblob.BlobAccessConditions{} 1473 return o.fs.pacer.Call(func() (bool, error) { 1474 _, err := blob.Delete(ctx, snapShotOptions, ac) 1475 return o.fs.shouldRetry(err) 1476 }) 1477 } 1478 1479 // MimeType of an Object if known, "" otherwise 1480 func (o *Object) MimeType(ctx context.Context) string { 1481 return o.mimeType 1482 } 1483 1484 // AccessTier of an object, default is of type none 1485 func (o *Object) AccessTier() azblob.AccessTierType { 1486 return o.accessTier 1487 } 1488 1489 // SetTier performs changing object tier 1490 func (o *Object) SetTier(tier string) error { 1491 if !validateAccessTier(tier) { 1492 return errors.Errorf("Tier %s not supported by Azure Blob Storage", tier) 1493 } 1494 1495 // Check if current tier already matches with desired tier 1496 if o.GetTier() == tier { 1497 return nil 1498 } 1499 desiredAccessTier := azblob.AccessTierType(tier) 1500 blob := o.getBlobReference() 1501 ctx := context.Background() 1502 err := o.fs.pacer.Call(func() (bool, error) { 1503 _, err := blob.SetTier(ctx, desiredAccessTier, azblob.LeaseAccessConditions{}) 1504 return o.fs.shouldRetry(err) 1505 }) 1506 1507 if err != nil { 1508 return errors.Wrap(err, "Failed to set Blob Tier") 1509 } 1510 1511 // Set access tier on local object also, this typically 1512 // gets updated on get blob properties 1513 o.accessTier = desiredAccessTier 1514 fs.Debugf(o, "Successfully changed object tier to %s", tier) 1515 1516 return nil 1517 } 1518 1519 // GetTier returns object tier in azure as string 1520 func (o *Object) GetTier() string { 1521 return string(o.accessTier) 1522 } 1523 1524 // Check the interfaces are satisfied 1525 var ( 1526 _ fs.Fs = &Fs{} 1527 _ fs.Copier = &Fs{} 1528 _ fs.Purger = &Fs{} 1529 _ fs.ListRer = &Fs{} 1530 _ fs.Object = &Object{} 1531 _ fs.MimeTyper = &Object{} 1532 )