github.com/ncw/rclone@v1.48.1-0.20190724201158-a35aa1360e3e/backend/azureblob/azureblob.go (about)

     1  // Package azureblob provides an interface to the Microsoft Azure blob object storage system
     2  
     3  // +build !plan9,!solaris
     4  
     5  package azureblob
     6  
     7  import (
     8  	"bytes"
     9  	"context"
    10  	"crypto/md5"
    11  	"encoding/base64"
    12  	"encoding/binary"
    13  	"encoding/hex"
    14  	"fmt"
    15  	"io"
    16  	"net/http"
    17  	"net/url"
    18  	"path"
    19  	"regexp"
    20  	"strconv"
    21  	"strings"
    22  	"sync"
    23  	"time"
    24  
    25  	"github.com/Azure/azure-pipeline-go/pipeline"
    26  	"github.com/Azure/azure-storage-blob-go/azblob"
    27  	"github.com/ncw/rclone/fs"
    28  	"github.com/ncw/rclone/fs/accounting"
    29  	"github.com/ncw/rclone/fs/config/configmap"
    30  	"github.com/ncw/rclone/fs/config/configstruct"
    31  	"github.com/ncw/rclone/fs/fserrors"
    32  	"github.com/ncw/rclone/fs/fshttp"
    33  	"github.com/ncw/rclone/fs/hash"
    34  	"github.com/ncw/rclone/fs/walk"
    35  	"github.com/ncw/rclone/lib/pacer"
    36  	"github.com/pkg/errors"
    37  )
    38  
    39  const (
    40  	minSleep              = 10 * time.Millisecond
    41  	maxSleep              = 10 * time.Second
    42  	decayConstant         = 1    // bigger for slower decay, exponential
    43  	maxListChunkSize      = 5000 // number of items to read at once
    44  	modTimeKey            = "mtime"
    45  	timeFormatIn          = time.RFC3339
    46  	timeFormatOut         = "2006-01-02T15:04:05.000000000Z07:00"
    47  	maxTotalParts         = 50000 // in multipart upload
    48  	storageDefaultBaseURL = "blob.core.windows.net"
    49  	// maxUncommittedSize = 9 << 30 // can't upload bigger than this
    50  	defaultChunkSize    = 4 * fs.MebiByte
    51  	maxChunkSize        = 100 * fs.MebiByte
    52  	defaultUploadCutoff = 256 * fs.MebiByte
    53  	maxUploadCutoff     = 256 * fs.MebiByte
    54  	defaultAccessTier   = azblob.AccessTierNone
    55  	maxTryTimeout       = time.Hour * 24 * 365 //max time of an azure web request response window (whether or not data is flowing)
    56  	// Default storage account, key and blob endpoint for emulator support,
    57  	// though it is a base64 key checked in here, it is publicly available secret.
    58  	emulatorAccount      = "devstoreaccount1"
    59  	emulatorAccountKey   = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
    60  	emulatorBlobEndpoint = "http://127.0.0.1:10000/devstoreaccount1"
    61  )
    62  
    63  // Register with Fs
    64  func init() {
    65  	fs.Register(&fs.RegInfo{
    66  		Name:        "azureblob",
    67  		Description: "Microsoft Azure Blob Storage",
    68  		NewFs:       NewFs,
    69  		Options: []fs.Option{{
    70  			Name: "account",
    71  			Help: "Storage Account Name (leave blank to use SAS URL or Emulator)",
    72  		}, {
    73  			Name: "key",
    74  			Help: "Storage Account Key (leave blank to use SAS URL or Emulator)",
    75  		}, {
    76  			Name: "sas_url",
    77  			Help: "SAS URL for container level access only\n(leave blank if using account/key or Emulator)",
    78  		}, {
    79  			Name:    "use_emulator",
    80  			Help:    "Uses local storage emulator if provided as 'true' (leave blank if using real azure storage endpoint)",
    81  			Default: false,
    82  		}, {
    83  			Name:     "endpoint",
    84  			Help:     "Endpoint for the service\nLeave blank normally.",
    85  			Advanced: true,
    86  		}, {
    87  			Name:     "upload_cutoff",
    88  			Help:     "Cutoff for switching to chunked upload (<= 256MB).",
    89  			Default:  defaultUploadCutoff,
    90  			Advanced: true,
    91  		}, {
    92  			Name: "chunk_size",
    93  			Help: `Upload chunk size (<= 100MB).
    94  
    95  Note that this is stored in memory and there may be up to
    96  "--transfers" chunks stored at once in memory.`,
    97  			Default:  defaultChunkSize,
    98  			Advanced: true,
    99  		}, {
   100  			Name: "list_chunk",
   101  			Help: `Size of blob list.
   102  
   103  This sets the number of blobs requested in each listing chunk. Default
   104  is the maximum, 5000. "List blobs" requests are permitted 2 minutes
   105  per megabyte to complete. If an operation is taking longer than 2
   106  minutes per megabyte on average, it will time out (
   107  [source](https://docs.microsoft.com/en-us/rest/api/storageservices/setting-timeouts-for-blob-service-operations#exceptions-to-default-timeout-interval)
   108  ). This can be used to limit the number of blobs items to return, to
   109  avoid the time out.`,
   110  			Default:  maxListChunkSize,
   111  			Advanced: true,
   112  		}, {
   113  			Name: "access_tier",
   114  			Help: `Access tier of blob: hot, cool or archive.
   115  
   116  Archived blobs can be restored by setting access tier to hot or
   117  cool. Leave blank if you intend to use default access tier, which is
   118  set at account level
   119  
   120  If there is no "access tier" specified, rclone doesn't apply any tier.
   121  rclone performs "Set Tier" operation on blobs while uploading, if objects
   122  are not modified, specifying "access tier" to new one will have no effect.
   123  If blobs are in "archive tier" at remote, trying to perform data transfer
   124  operations from remote will not be allowed. User should first restore by
   125  tiering blob to "Hot" or "Cool".`,
   126  			Advanced: true,
   127  		}},
   128  	})
   129  }
   130  
   131  // Options defines the configuration for this backend
   132  type Options struct {
   133  	Account       string        `config:"account"`
   134  	Key           string        `config:"key"`
   135  	Endpoint      string        `config:"endpoint"`
   136  	SASURL        string        `config:"sas_url"`
   137  	UploadCutoff  fs.SizeSuffix `config:"upload_cutoff"`
   138  	ChunkSize     fs.SizeSuffix `config:"chunk_size"`
   139  	ListChunkSize uint          `config:"list_chunk"`
   140  	AccessTier    string        `config:"access_tier"`
   141  	UseEmulator   bool          `config:"use_emulator"`
   142  }
   143  
   144  // Fs represents a remote azure server
   145  type Fs struct {
   146  	name             string                // name of this remote
   147  	root             string                // the path we are working on if any
   148  	opt              Options               // parsed config options
   149  	features         *fs.Features          // optional features
   150  	client           *http.Client          // http client we are using
   151  	svcURL           *azblob.ServiceURL    // reference to serviceURL
   152  	cntURL           *azblob.ContainerURL  // reference to containerURL
   153  	container        string                // the container we are working on
   154  	containerOKMu    sync.Mutex            // mutex to protect container OK
   155  	containerOK      bool                  // true if we have created the container
   156  	containerDeleted bool                  // true if we have deleted the container
   157  	pacer            *fs.Pacer             // To pace and retry the API calls
   158  	uploadToken      *pacer.TokenDispenser // control concurrency
   159  }
   160  
   161  // Object describes a azure object
   162  type Object struct {
   163  	fs         *Fs                   // what this object is part of
   164  	remote     string                // The remote path
   165  	modTime    time.Time             // The modified time of the object if known
   166  	md5        string                // MD5 hash if known
   167  	size       int64                 // Size of the object
   168  	mimeType   string                // Content-Type of the object
   169  	accessTier azblob.AccessTierType // Blob Access Tier
   170  	meta       map[string]string     // blob metadata
   171  }
   172  
   173  // ------------------------------------------------------------
   174  
   175  // Name of the remote (as passed into NewFs)
   176  func (f *Fs) Name() string {
   177  	return f.name
   178  }
   179  
   180  // Root of the remote (as passed into NewFs)
   181  func (f *Fs) Root() string {
   182  	if f.root == "" {
   183  		return f.container
   184  	}
   185  	return f.container + "/" + f.root
   186  }
   187  
   188  // String converts this Fs to a string
   189  func (f *Fs) String() string {
   190  	if f.root == "" {
   191  		return fmt.Sprintf("Azure container %s", f.container)
   192  	}
   193  	return fmt.Sprintf("Azure container %s path %s", f.container, f.root)
   194  }
   195  
   196  // Features returns the optional features of this Fs
   197  func (f *Fs) Features() *fs.Features {
   198  	return f.features
   199  }
   200  
   201  // Pattern to match a azure path
   202  var matcher = regexp.MustCompile(`^/*([^/]*)(.*)$`)
   203  
   204  // parseParse parses a azure 'url'
   205  func parsePath(path string) (container, directory string, err error) {
   206  	parts := matcher.FindStringSubmatch(path)
   207  	if parts == nil {
   208  		err = errors.Errorf("couldn't find container in azure path %q", path)
   209  	} else {
   210  		container, directory = parts[1], parts[2]
   211  		directory = strings.Trim(directory, "/")
   212  	}
   213  	return
   214  }
   215  
   216  // validateAccessTier checks if azureblob supports user supplied tier
   217  func validateAccessTier(tier string) bool {
   218  	switch tier {
   219  	case string(azblob.AccessTierHot),
   220  		string(azblob.AccessTierCool),
   221  		string(azblob.AccessTierArchive):
   222  		// valid cases
   223  		return true
   224  	default:
   225  		return false
   226  	}
   227  }
   228  
   229  // retryErrorCodes is a slice of error codes that we will retry
   230  var retryErrorCodes = []int{
   231  	401, // Unauthorized (eg "Token has expired")
   232  	408, // Request Timeout
   233  	429, // Rate exceeded.
   234  	500, // Get occasional 500 Internal Server Error
   235  	503, // Service Unavailable
   236  	504, // Gateway Time-out
   237  }
   238  
   239  // shouldRetry returns a boolean as to whether this resp and err
   240  // deserve to be retried.  It returns the err as a convenience
   241  func (f *Fs) shouldRetry(err error) (bool, error) {
   242  	// FIXME interpret special errors - more to do here
   243  	if storageErr, ok := err.(azblob.StorageError); ok {
   244  		statusCode := storageErr.Response().StatusCode
   245  		for _, e := range retryErrorCodes {
   246  			if statusCode == e {
   247  				return true, err
   248  			}
   249  		}
   250  	}
   251  	return fserrors.ShouldRetry(err), err
   252  }
   253  
   254  func checkUploadChunkSize(cs fs.SizeSuffix) error {
   255  	const minChunkSize = fs.Byte
   256  	if cs < minChunkSize {
   257  		return errors.Errorf("%s is less than %s", cs, minChunkSize)
   258  	}
   259  	if cs > maxChunkSize {
   260  		return errors.Errorf("%s is greater than %s", cs, maxChunkSize)
   261  	}
   262  	return nil
   263  }
   264  
   265  func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
   266  	err = checkUploadChunkSize(cs)
   267  	if err == nil {
   268  		old, f.opt.ChunkSize = f.opt.ChunkSize, cs
   269  	}
   270  	return
   271  }
   272  
   273  func checkUploadCutoff(cs fs.SizeSuffix) error {
   274  	if cs > maxUploadCutoff {
   275  		return errors.Errorf("%v must be less than or equal to %v", cs, maxUploadCutoff)
   276  	}
   277  	return nil
   278  }
   279  
   280  func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
   281  	err = checkUploadCutoff(cs)
   282  	if err == nil {
   283  		old, f.opt.UploadCutoff = f.opt.UploadCutoff, cs
   284  	}
   285  	return
   286  }
   287  
   288  // httpClientFactory creates a Factory object that sends HTTP requests
   289  // to a rclone's http.Client.
   290  //
   291  // copied from azblob.newDefaultHTTPClientFactory
   292  func httpClientFactory(client *http.Client) pipeline.Factory {
   293  	return pipeline.FactoryFunc(func(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.PolicyFunc {
   294  		return func(ctx context.Context, request pipeline.Request) (pipeline.Response, error) {
   295  			r, err := client.Do(request.WithContext(ctx))
   296  			if err != nil {
   297  				err = pipeline.NewError(err, "HTTP request failed")
   298  			}
   299  			return pipeline.NewHTTPResponse(r), err
   300  		}
   301  	})
   302  }
   303  
   304  // newPipeline creates a Pipeline using the specified credentials and options.
   305  //
   306  // this code was copied from azblob.NewPipeline
   307  func (f *Fs) newPipeline(c azblob.Credential, o azblob.PipelineOptions) pipeline.Pipeline {
   308  	// Closest to API goes first; closest to the wire goes last
   309  	factories := []pipeline.Factory{
   310  		azblob.NewTelemetryPolicyFactory(o.Telemetry),
   311  		azblob.NewUniqueRequestIDPolicyFactory(),
   312  		azblob.NewRetryPolicyFactory(o.Retry),
   313  		c,
   314  		pipeline.MethodFactoryMarker(), // indicates at what stage in the pipeline the method factory is invoked
   315  		azblob.NewRequestLogPolicyFactory(o.RequestLog),
   316  	}
   317  	return pipeline.NewPipeline(factories, pipeline.Options{HTTPSender: httpClientFactory(f.client), Log: o.Log})
   318  }
   319  
   320  // NewFs constructs an Fs from the path, container:path
   321  func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
   322  	ctx := context.Background()
   323  	// Parse config into Options struct
   324  	opt := new(Options)
   325  	err := configstruct.Set(m, opt)
   326  	if err != nil {
   327  		return nil, err
   328  	}
   329  
   330  	err = checkUploadCutoff(opt.UploadCutoff)
   331  	if err != nil {
   332  		return nil, errors.Wrap(err, "azure: upload cutoff")
   333  	}
   334  	err = checkUploadChunkSize(opt.ChunkSize)
   335  	if err != nil {
   336  		return nil, errors.Wrap(err, "azure: chunk size")
   337  	}
   338  	if opt.ListChunkSize > maxListChunkSize {
   339  		return nil, errors.Errorf("azure: blob list size can't be greater than %v - was %v", maxListChunkSize, opt.ListChunkSize)
   340  	}
   341  	container, directory, err := parsePath(root)
   342  	if err != nil {
   343  		return nil, err
   344  	}
   345  	if opt.Endpoint == "" {
   346  		opt.Endpoint = storageDefaultBaseURL
   347  	}
   348  
   349  	if opt.AccessTier == "" {
   350  		opt.AccessTier = string(defaultAccessTier)
   351  	} else if !validateAccessTier(opt.AccessTier) {
   352  		return nil, errors.Errorf("Azure Blob: Supported access tiers are %s, %s and %s",
   353  			string(azblob.AccessTierHot), string(azblob.AccessTierCool), string(azblob.AccessTierArchive))
   354  	}
   355  
   356  	f := &Fs{
   357  		name:        name,
   358  		opt:         *opt,
   359  		container:   container,
   360  		root:        directory,
   361  		pacer:       fs.NewPacer(pacer.NewS3(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
   362  		uploadToken: pacer.NewTokenDispenser(fs.Config.Transfers),
   363  		client:      fshttp.NewClient(fs.Config),
   364  	}
   365  	f.features = (&fs.Features{
   366  		ReadMimeType:  true,
   367  		WriteMimeType: true,
   368  		BucketBased:   true,
   369  		SetTier:       true,
   370  		GetTier:       true,
   371  	}).Fill(f)
   372  
   373  	var (
   374  		u            *url.URL
   375  		serviceURL   azblob.ServiceURL
   376  		containerURL azblob.ContainerURL
   377  	)
   378  	switch {
   379  	case opt.UseEmulator:
   380  		credential, err := azblob.NewSharedKeyCredential(emulatorAccount, emulatorAccountKey)
   381  		if err != nil {
   382  			return nil, errors.Wrapf(err, "Failed to parse credentials")
   383  		}
   384  		u, err = url.Parse(emulatorBlobEndpoint)
   385  		if err != nil {
   386  			return nil, errors.Wrap(err, "failed to make azure storage url from account and endpoint")
   387  		}
   388  		pipeline := f.newPipeline(credential, azblob.PipelineOptions{Retry: azblob.RetryOptions{TryTimeout: maxTryTimeout}})
   389  		serviceURL = azblob.NewServiceURL(*u, pipeline)
   390  		containerURL = serviceURL.NewContainerURL(container)
   391  	case opt.Account != "" && opt.Key != "":
   392  		credential, err := azblob.NewSharedKeyCredential(opt.Account, opt.Key)
   393  		if err != nil {
   394  			return nil, errors.Wrapf(err, "Failed to parse credentials")
   395  		}
   396  
   397  		u, err = url.Parse(fmt.Sprintf("https://%s.%s", opt.Account, opt.Endpoint))
   398  		if err != nil {
   399  			return nil, errors.Wrap(err, "failed to make azure storage url from account and endpoint")
   400  		}
   401  		pipeline := f.newPipeline(credential, azblob.PipelineOptions{Retry: azblob.RetryOptions{TryTimeout: maxTryTimeout}})
   402  		serviceURL = azblob.NewServiceURL(*u, pipeline)
   403  		containerURL = serviceURL.NewContainerURL(container)
   404  	case opt.SASURL != "":
   405  		u, err = url.Parse(opt.SASURL)
   406  		if err != nil {
   407  			return nil, errors.Wrapf(err, "failed to parse SAS URL")
   408  		}
   409  		// use anonymous credentials in case of sas url
   410  		pipeline := f.newPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{Retry: azblob.RetryOptions{TryTimeout: maxTryTimeout}})
   411  		// Check if we have container level SAS or account level sas
   412  		parts := azblob.NewBlobURLParts(*u)
   413  		if parts.ContainerName != "" {
   414  			if container != "" && parts.ContainerName != container {
   415  				return nil, errors.New("Container name in SAS URL and container provided in command do not match")
   416  			}
   417  
   418  			f.container = parts.ContainerName
   419  			containerURL = azblob.NewContainerURL(*u, pipeline)
   420  		} else {
   421  			serviceURL = azblob.NewServiceURL(*u, pipeline)
   422  			containerURL = serviceURL.NewContainerURL(container)
   423  		}
   424  	default:
   425  		return nil, errors.New("Need account+key or connectionString or sasURL")
   426  	}
   427  	f.svcURL = &serviceURL
   428  	f.cntURL = &containerURL
   429  
   430  	if f.root != "" {
   431  		f.root += "/"
   432  		// Check to see if the (container,directory) is actually an existing file
   433  		oldRoot := f.root
   434  		remote := path.Base(directory)
   435  		f.root = path.Dir(directory)
   436  		if f.root == "." {
   437  			f.root = ""
   438  		} else {
   439  			f.root += "/"
   440  		}
   441  		_, err := f.NewObject(ctx, remote)
   442  		if err != nil {
   443  			if err == fs.ErrorObjectNotFound || err == fs.ErrorNotAFile {
   444  				// File doesn't exist or is a directory so return old f
   445  				f.root = oldRoot
   446  				return f, nil
   447  			}
   448  			return nil, err
   449  		}
   450  		// return an error with an fs which points to the parent
   451  		return f, fs.ErrorIsFile
   452  	}
   453  	return f, nil
   454  }
   455  
   456  // Return an Object from a path
   457  //
   458  // If it can't be found it returns the error fs.ErrorObjectNotFound.
   459  func (f *Fs) newObjectWithInfo(remote string, info *azblob.BlobItem) (fs.Object, error) {
   460  	o := &Object{
   461  		fs:     f,
   462  		remote: remote,
   463  	}
   464  	if info != nil {
   465  		err := o.decodeMetaDataFromBlob(info)
   466  		if err != nil {
   467  			return nil, err
   468  		}
   469  	} else {
   470  		err := o.readMetaData() // reads info and headers, returning an error
   471  		if err != nil {
   472  			return nil, err
   473  		}
   474  	}
   475  	return o, nil
   476  }
   477  
   478  // NewObject finds the Object at remote.  If it can't be found
   479  // it returns the error fs.ErrorObjectNotFound.
   480  func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
   481  	return f.newObjectWithInfo(remote, nil)
   482  }
   483  
   484  // getBlobReference creates an empty blob reference with no metadata
   485  func (f *Fs) getBlobReference(remote string) azblob.BlobURL {
   486  	return f.cntURL.NewBlobURL(f.root + remote)
   487  }
   488  
   489  // updateMetadataWithModTime adds the modTime passed in to o.meta.
   490  func (o *Object) updateMetadataWithModTime(modTime time.Time) {
   491  	// Make sure o.meta is not nil
   492  	if o.meta == nil {
   493  		o.meta = make(map[string]string, 1)
   494  	}
   495  
   496  	// Set modTimeKey in it
   497  	o.meta[modTimeKey] = modTime.Format(timeFormatOut)
   498  }
   499  
   500  // Returns whether file is a directory marker or not
   501  func isDirectoryMarker(size int64, metadata azblob.Metadata, remote string) bool {
   502  	// Directory markers are 0 length
   503  	if size == 0 {
   504  		// Note that metadata with hdi_isfolder = true seems to be a
   505  		// defacto standard for marking blobs as directories.
   506  		endsWithSlash := strings.HasSuffix(remote, "/")
   507  		if endsWithSlash || remote == "" || metadata["hdi_isfolder"] == "true" {
   508  			return true
   509  		}
   510  
   511  	}
   512  	return false
   513  }
   514  
   515  // listFn is called from list to handle an object
   516  type listFn func(remote string, object *azblob.BlobItem, isDirectory bool) error
   517  
   518  // list lists the objects into the function supplied from
   519  // the container and root supplied
   520  //
   521  // dir is the starting directory, "" for root
   522  func (f *Fs) list(ctx context.Context, dir string, recurse bool, maxResults uint, fn listFn) error {
   523  	f.containerOKMu.Lock()
   524  	deleted := f.containerDeleted
   525  	f.containerOKMu.Unlock()
   526  	if deleted {
   527  		return fs.ErrorDirNotFound
   528  	}
   529  	root := f.root
   530  	if dir != "" {
   531  		root += dir + "/"
   532  	}
   533  	delimiter := ""
   534  	if !recurse {
   535  		delimiter = "/"
   536  	}
   537  
   538  	options := azblob.ListBlobsSegmentOptions{
   539  		Details: azblob.BlobListingDetails{
   540  			Copy:             false,
   541  			Metadata:         true,
   542  			Snapshots:        false,
   543  			UncommittedBlobs: false,
   544  			Deleted:          false,
   545  		},
   546  		Prefix:     root,
   547  		MaxResults: int32(maxResults),
   548  	}
   549  	directoryMarkers := map[string]struct{}{}
   550  	for marker := (azblob.Marker{}); marker.NotDone(); {
   551  		var response *azblob.ListBlobsHierarchySegmentResponse
   552  		err := f.pacer.Call(func() (bool, error) {
   553  			var err error
   554  			response, err = f.cntURL.ListBlobsHierarchySegment(ctx, marker, delimiter, options)
   555  			return f.shouldRetry(err)
   556  		})
   557  
   558  		if err != nil {
   559  			// Check http error code along with service code, current SDK doesn't populate service code correctly sometimes
   560  			if storageErr, ok := err.(azblob.StorageError); ok && (storageErr.ServiceCode() == azblob.ServiceCodeContainerNotFound || storageErr.Response().StatusCode == http.StatusNotFound) {
   561  				return fs.ErrorDirNotFound
   562  			}
   563  			return err
   564  		}
   565  		// Advance marker to next
   566  		marker = response.NextMarker
   567  
   568  		for i := range response.Segment.BlobItems {
   569  			file := &response.Segment.BlobItems[i]
   570  			// Finish if file name no longer has prefix
   571  			// if prefix != "" && !strings.HasPrefix(file.Name, prefix) {
   572  			// 	return nil
   573  			// }
   574  			if !strings.HasPrefix(file.Name, f.root) {
   575  				fs.Debugf(f, "Odd name received %q", file.Name)
   576  				continue
   577  			}
   578  			remote := file.Name[len(f.root):]
   579  			if isDirectoryMarker(*file.Properties.ContentLength, file.Metadata, remote) {
   580  				if strings.HasSuffix(remote, "/") {
   581  					remote = remote[:len(remote)-1]
   582  				}
   583  				err = fn(remote, file, true)
   584  				if err != nil {
   585  					return err
   586  				}
   587  				// Keep track of directory markers. If recursing then
   588  				// there will be no Prefixes so no need to keep track
   589  				if !recurse {
   590  					directoryMarkers[remote] = struct{}{}
   591  				}
   592  				continue // skip directory marker
   593  			}
   594  			// Send object
   595  			err = fn(remote, file, false)
   596  			if err != nil {
   597  				return err
   598  			}
   599  		}
   600  		// Send the subdirectories
   601  		for _, remote := range response.Segment.BlobPrefixes {
   602  			remote := strings.TrimRight(remote.Name, "/")
   603  			if !strings.HasPrefix(remote, f.root) {
   604  				fs.Debugf(f, "Odd directory name received %q", remote)
   605  				continue
   606  			}
   607  			remote = remote[len(f.root):]
   608  			// Don't send if already sent as a directory marker
   609  			if _, found := directoryMarkers[remote]; found {
   610  				continue
   611  			}
   612  			// Send object
   613  			err = fn(remote, nil, true)
   614  			if err != nil {
   615  				return err
   616  			}
   617  		}
   618  	}
   619  	return nil
   620  }
   621  
   622  // Convert a list item into a DirEntry
   623  func (f *Fs) itemToDirEntry(remote string, object *azblob.BlobItem, isDirectory bool) (fs.DirEntry, error) {
   624  	if isDirectory {
   625  		d := fs.NewDir(remote, time.Time{})
   626  		return d, nil
   627  	}
   628  	o, err := f.newObjectWithInfo(remote, object)
   629  	if err != nil {
   630  		return nil, err
   631  	}
   632  	return o, nil
   633  }
   634  
   635  // mark the container as being OK
   636  func (f *Fs) markContainerOK() {
   637  	if f.container != "" {
   638  		f.containerOKMu.Lock()
   639  		f.containerOK = true
   640  		f.containerDeleted = false
   641  		f.containerOKMu.Unlock()
   642  	}
   643  }
   644  
   645  // listDir lists a single directory
   646  func (f *Fs) listDir(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
   647  	err = f.list(ctx, dir, false, f.opt.ListChunkSize, func(remote string, object *azblob.BlobItem, isDirectory bool) error {
   648  		entry, err := f.itemToDirEntry(remote, object, isDirectory)
   649  		if err != nil {
   650  			return err
   651  		}
   652  		if entry != nil {
   653  			entries = append(entries, entry)
   654  		}
   655  		return nil
   656  	})
   657  	if err != nil {
   658  		return nil, err
   659  	}
   660  	// container must be present if listing succeeded
   661  	f.markContainerOK()
   662  	return entries, nil
   663  }
   664  
   665  // listContainers returns all the containers to out
   666  func (f *Fs) listContainers(dir string) (entries fs.DirEntries, err error) {
   667  	if dir != "" {
   668  		return nil, fs.ErrorListBucketRequired
   669  	}
   670  	err = f.listContainersToFn(func(container *azblob.ContainerItem) error {
   671  		d := fs.NewDir(container.Name, container.Properties.LastModified)
   672  		entries = append(entries, d)
   673  		return nil
   674  	})
   675  	if err != nil {
   676  		return nil, err
   677  	}
   678  	return entries, nil
   679  }
   680  
   681  // List the objects and directories in dir into entries.  The
   682  // entries can be returned in any order but should be for a
   683  // complete directory.
   684  //
   685  // dir should be "" to list the root, and should not have
   686  // trailing slashes.
   687  //
   688  // This should return ErrDirNotFound if the directory isn't
   689  // found.
   690  func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
   691  	if f.container == "" {
   692  		return f.listContainers(dir)
   693  	}
   694  	return f.listDir(ctx, dir)
   695  }
   696  
   697  // ListR lists the objects and directories of the Fs starting
   698  // from dir recursively into out.
   699  //
   700  // dir should be "" to start from the root, and should not
   701  // have trailing slashes.
   702  //
   703  // This should return ErrDirNotFound if the directory isn't
   704  // found.
   705  //
   706  // It should call callback for each tranche of entries read.
   707  // These need not be returned in any particular order.  If
   708  // callback returns an error then the listing will stop
   709  // immediately.
   710  //
   711  // Don't implement this unless you have a more efficient way
   712  // of listing recursively that doing a directory traversal.
   713  func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
   714  	if f.container == "" {
   715  		return fs.ErrorListBucketRequired
   716  	}
   717  	list := walk.NewListRHelper(callback)
   718  	err = f.list(ctx, dir, true, f.opt.ListChunkSize, func(remote string, object *azblob.BlobItem, isDirectory bool) error {
   719  		entry, err := f.itemToDirEntry(remote, object, isDirectory)
   720  		if err != nil {
   721  			return err
   722  		}
   723  		return list.Add(entry)
   724  	})
   725  	if err != nil {
   726  		return err
   727  	}
   728  	// container must be present if listing succeeded
   729  	f.markContainerOK()
   730  	return list.Flush()
   731  }
   732  
   733  // listContainerFn is called from listContainersToFn to handle a container
   734  type listContainerFn func(*azblob.ContainerItem) error
   735  
   736  // listContainersToFn lists the containers to the function supplied
   737  func (f *Fs) listContainersToFn(fn listContainerFn) error {
   738  	params := azblob.ListContainersSegmentOptions{
   739  		MaxResults: int32(f.opt.ListChunkSize),
   740  	}
   741  	ctx := context.Background()
   742  	for marker := (azblob.Marker{}); marker.NotDone(); {
   743  		var response *azblob.ListContainersSegmentResponse
   744  		err := f.pacer.Call(func() (bool, error) {
   745  			var err error
   746  			response, err = f.svcURL.ListContainersSegment(ctx, marker, params)
   747  			return f.shouldRetry(err)
   748  		})
   749  		if err != nil {
   750  			return err
   751  		}
   752  
   753  		for i := range response.ContainerItems {
   754  			err = fn(&response.ContainerItems[i])
   755  			if err != nil {
   756  				return err
   757  			}
   758  		}
   759  		marker = response.NextMarker
   760  	}
   761  
   762  	return nil
   763  }
   764  
   765  // Put the object into the container
   766  //
   767  // Copy the reader in to the new object which is returned
   768  //
   769  // The new object may have been created if an error is returned
   770  func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
   771  	// Temporary Object under construction
   772  	fs := &Object{
   773  		fs:     f,
   774  		remote: src.Remote(),
   775  	}
   776  	return fs, fs.Update(ctx, in, src, options...)
   777  }
   778  
   779  // Check if the container exists
   780  //
   781  // NB this can return incorrect results if called immediately after container deletion
   782  func (f *Fs) dirExists() (bool, error) {
   783  	options := azblob.ListBlobsSegmentOptions{
   784  		Details: azblob.BlobListingDetails{
   785  			Copy:             false,
   786  			Metadata:         false,
   787  			Snapshots:        false,
   788  			UncommittedBlobs: false,
   789  			Deleted:          false,
   790  		},
   791  		MaxResults: 1,
   792  	}
   793  	err := f.pacer.Call(func() (bool, error) {
   794  		ctx := context.Background()
   795  		_, err := f.cntURL.ListBlobsHierarchySegment(ctx, azblob.Marker{}, "", options)
   796  		return f.shouldRetry(err)
   797  	})
   798  	if err == nil {
   799  		return true, nil
   800  	}
   801  	// Check http error code along with service code, current SDK doesn't populate service code correctly sometimes
   802  	if storageErr, ok := err.(azblob.StorageError); ok && (storageErr.ServiceCode() == azblob.ServiceCodeContainerNotFound || storageErr.Response().StatusCode == http.StatusNotFound) {
   803  		return false, nil
   804  	}
   805  	return false, err
   806  }
   807  
   808  // Mkdir creates the container if it doesn't exist
   809  func (f *Fs) Mkdir(ctx context.Context, dir string) error {
   810  	f.containerOKMu.Lock()
   811  	defer f.containerOKMu.Unlock()
   812  	if f.containerOK {
   813  		return nil
   814  	}
   815  	if !f.containerDeleted {
   816  		exists, err := f.dirExists()
   817  		if err == nil {
   818  			f.containerOK = exists
   819  		}
   820  		if err != nil || exists {
   821  			return err
   822  		}
   823  	}
   824  
   825  	// now try to create the container
   826  	err := f.pacer.Call(func() (bool, error) {
   827  		ctx := context.Background()
   828  		_, err := f.cntURL.Create(ctx, azblob.Metadata{}, azblob.PublicAccessNone)
   829  		if err != nil {
   830  			if storageErr, ok := err.(azblob.StorageError); ok {
   831  				switch storageErr.ServiceCode() {
   832  				case azblob.ServiceCodeContainerAlreadyExists:
   833  					f.containerOK = true
   834  					return false, nil
   835  				case azblob.ServiceCodeContainerBeingDeleted:
   836  					// From https://docs.microsoft.com/en-us/rest/api/storageservices/delete-container
   837  					// When a container is deleted, a container with the same name cannot be created
   838  					// for at least 30 seconds; the container may not be available for more than 30
   839  					// seconds if the service is still processing the request.
   840  					time.Sleep(6 * time.Second) // default 10 retries will be 60 seconds
   841  					f.containerDeleted = true
   842  					return true, err
   843  				}
   844  			}
   845  		}
   846  		return f.shouldRetry(err)
   847  	})
   848  	if err == nil {
   849  		f.containerOK = true
   850  		f.containerDeleted = false
   851  	}
   852  	return errors.Wrap(err, "failed to make container")
   853  }
   854  
   855  // isEmpty checks to see if a given directory is empty and returns an error if not
   856  func (f *Fs) isEmpty(ctx context.Context, dir string) (err error) {
   857  	empty := true
   858  	err = f.list(ctx, dir, true, 1, func(remote string, object *azblob.BlobItem, isDirectory bool) error {
   859  		empty = false
   860  		return nil
   861  	})
   862  	if err != nil {
   863  		return err
   864  	}
   865  	if !empty {
   866  		return fs.ErrorDirectoryNotEmpty
   867  	}
   868  	return nil
   869  }
   870  
   871  // deleteContainer deletes the container.  It can delete a full
   872  // container so use isEmpty if you don't want that.
   873  func (f *Fs) deleteContainer() error {
   874  	f.containerOKMu.Lock()
   875  	defer f.containerOKMu.Unlock()
   876  	options := azblob.ContainerAccessConditions{}
   877  	ctx := context.Background()
   878  	err := f.pacer.Call(func() (bool, error) {
   879  		_, err := f.cntURL.GetProperties(ctx, azblob.LeaseAccessConditions{})
   880  		if err == nil {
   881  			_, err = f.cntURL.Delete(ctx, options)
   882  		}
   883  
   884  		if err != nil {
   885  			// Check http error code along with service code, current SDK doesn't populate service code correctly sometimes
   886  			if storageErr, ok := err.(azblob.StorageError); ok && (storageErr.ServiceCode() == azblob.ServiceCodeContainerNotFound || storageErr.Response().StatusCode == http.StatusNotFound) {
   887  				return false, fs.ErrorDirNotFound
   888  			}
   889  
   890  			return f.shouldRetry(err)
   891  		}
   892  
   893  		return f.shouldRetry(err)
   894  	})
   895  	if err == nil {
   896  		f.containerOK = false
   897  		f.containerDeleted = true
   898  	}
   899  	return errors.Wrap(err, "failed to delete container")
   900  }
   901  
   902  // Rmdir deletes the container if the fs is at the root
   903  //
   904  // Returns an error if it isn't empty
   905  func (f *Fs) Rmdir(ctx context.Context, dir string) error {
   906  	err := f.isEmpty(ctx, dir)
   907  	if err != nil {
   908  		return err
   909  	}
   910  	if f.root != "" || dir != "" {
   911  		return nil
   912  	}
   913  	return f.deleteContainer()
   914  }
   915  
   916  // Precision of the remote
   917  func (f *Fs) Precision() time.Duration {
   918  	return time.Nanosecond
   919  }
   920  
   921  // Hashes returns the supported hash sets.
   922  func (f *Fs) Hashes() hash.Set {
   923  	return hash.Set(hash.MD5)
   924  }
   925  
   926  // Purge deletes all the files and directories including the old versions.
   927  func (f *Fs) Purge(ctx context.Context) error {
   928  	dir := "" // forward compat!
   929  	if f.root != "" || dir != "" {
   930  		// Delegate to caller if not root container
   931  		return fs.ErrorCantPurge
   932  	}
   933  	return f.deleteContainer()
   934  }
   935  
   936  // Copy src to this remote using server side copy operations.
   937  //
   938  // This is stored with the remote path given
   939  //
   940  // It returns the destination Object and a possible error
   941  //
   942  // Will only be called if src.Fs().Name() == f.Name()
   943  //
   944  // If it isn't possible then return fs.ErrorCantCopy
   945  func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
   946  	err := f.Mkdir(ctx, "")
   947  	if err != nil {
   948  		return nil, err
   949  	}
   950  	srcObj, ok := src.(*Object)
   951  	if !ok {
   952  		fs.Debugf(src, "Can't copy - not same remote type")
   953  		return nil, fs.ErrorCantCopy
   954  	}
   955  	dstBlobURL := f.getBlobReference(remote)
   956  	srcBlobURL := srcObj.getBlobReference()
   957  
   958  	source, err := url.Parse(srcBlobURL.String())
   959  	if err != nil {
   960  		return nil, err
   961  	}
   962  
   963  	options := azblob.BlobAccessConditions{}
   964  	var startCopy *azblob.BlobStartCopyFromURLResponse
   965  
   966  	err = f.pacer.Call(func() (bool, error) {
   967  		startCopy, err = dstBlobURL.StartCopyFromURL(ctx, *source, nil, azblob.ModifiedAccessConditions{}, options)
   968  		return f.shouldRetry(err)
   969  	})
   970  	if err != nil {
   971  		return nil, err
   972  	}
   973  
   974  	copyStatus := startCopy.CopyStatus()
   975  	for copyStatus == azblob.CopyStatusPending {
   976  		time.Sleep(1 * time.Second)
   977  		getMetadata, err := dstBlobURL.GetProperties(ctx, options)
   978  		if err != nil {
   979  			return nil, err
   980  		}
   981  		copyStatus = getMetadata.CopyStatus()
   982  	}
   983  
   984  	return f.NewObject(ctx, remote)
   985  }
   986  
   987  // ------------------------------------------------------------
   988  
   989  // Fs returns the parent Fs
   990  func (o *Object) Fs() fs.Info {
   991  	return o.fs
   992  }
   993  
   994  // Return a string version
   995  func (o *Object) String() string {
   996  	if o == nil {
   997  		return "<nil>"
   998  	}
   999  	return o.remote
  1000  }
  1001  
  1002  // Remote returns the remote path
  1003  func (o *Object) Remote() string {
  1004  	return o.remote
  1005  }
  1006  
  1007  // Hash returns the MD5 of an object returning a lowercase hex string
  1008  func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
  1009  	if t != hash.MD5 {
  1010  		return "", hash.ErrUnsupported
  1011  	}
  1012  	// Convert base64 encoded md5 into lower case hex
  1013  	if o.md5 == "" {
  1014  		return "", nil
  1015  	}
  1016  	data, err := base64.StdEncoding.DecodeString(o.md5)
  1017  	if err != nil {
  1018  		return "", errors.Wrapf(err, "Failed to decode Content-MD5: %q", o.md5)
  1019  	}
  1020  	return hex.EncodeToString(data), nil
  1021  }
  1022  
  1023  // Size returns the size of an object in bytes
  1024  func (o *Object) Size() int64 {
  1025  	return o.size
  1026  }
  1027  
  1028  func (o *Object) setMetadata(metadata azblob.Metadata) {
  1029  	if len(metadata) > 0 {
  1030  		o.meta = metadata
  1031  		if modTime, ok := metadata[modTimeKey]; ok {
  1032  			when, err := time.Parse(timeFormatIn, modTime)
  1033  			if err != nil {
  1034  				fs.Debugf(o, "Couldn't parse %v = %q: %v", modTimeKey, modTime, err)
  1035  			}
  1036  			o.modTime = when
  1037  		}
  1038  	} else {
  1039  		o.meta = nil
  1040  	}
  1041  }
  1042  
  1043  // decodeMetaDataFromPropertiesResponse sets the metadata from the data passed in
  1044  //
  1045  // Sets
  1046  //  o.id
  1047  //  o.modTime
  1048  //  o.size
  1049  //  o.md5
  1050  //  o.meta
  1051  func (o *Object) decodeMetaDataFromPropertiesResponse(info *azblob.BlobGetPropertiesResponse) (err error) {
  1052  	metadata := info.NewMetadata()
  1053  	size := info.ContentLength()
  1054  	if isDirectoryMarker(size, metadata, o.remote) {
  1055  		return fs.ErrorNotAFile
  1056  	}
  1057  	// NOTE - Client library always returns MD5 as base64 decoded string, Object needs to maintain
  1058  	// this as base64 encoded string.
  1059  	o.md5 = base64.StdEncoding.EncodeToString(info.ContentMD5())
  1060  	o.mimeType = info.ContentType()
  1061  	o.size = size
  1062  	o.modTime = info.LastModified()
  1063  	o.accessTier = azblob.AccessTierType(info.AccessTier())
  1064  	o.setMetadata(metadata)
  1065  
  1066  	return nil
  1067  }
  1068  
  1069  func (o *Object) decodeMetaDataFromBlob(info *azblob.BlobItem) (err error) {
  1070  	metadata := info.Metadata
  1071  	size := *info.Properties.ContentLength
  1072  	if isDirectoryMarker(size, metadata, o.remote) {
  1073  		return fs.ErrorNotAFile
  1074  	}
  1075  	// NOTE - Client library always returns MD5 as base64 decoded string, Object needs to maintain
  1076  	// this as base64 encoded string.
  1077  	o.md5 = base64.StdEncoding.EncodeToString(info.Properties.ContentMD5)
  1078  	o.mimeType = *info.Properties.ContentType
  1079  	o.size = size
  1080  	o.modTime = info.Properties.LastModified
  1081  	o.accessTier = info.Properties.AccessTier
  1082  	o.setMetadata(metadata)
  1083  	return nil
  1084  }
  1085  
  1086  // getBlobReference creates an empty blob reference with no metadata
  1087  func (o *Object) getBlobReference() azblob.BlobURL {
  1088  	return o.fs.getBlobReference(o.remote)
  1089  }
  1090  
  1091  // clearMetaData clears enough metadata so readMetaData will re-read it
  1092  func (o *Object) clearMetaData() {
  1093  	o.modTime = time.Time{}
  1094  }
  1095  
  1096  // readMetaData gets the metadata if it hasn't already been fetched
  1097  //
  1098  // Sets
  1099  //  o.id
  1100  //  o.modTime
  1101  //  o.size
  1102  //  o.md5
  1103  func (o *Object) readMetaData() (err error) {
  1104  	if !o.modTime.IsZero() {
  1105  		return nil
  1106  	}
  1107  	blob := o.getBlobReference()
  1108  
  1109  	// Read metadata (this includes metadata)
  1110  	options := azblob.BlobAccessConditions{}
  1111  	ctx := context.Background()
  1112  	var blobProperties *azblob.BlobGetPropertiesResponse
  1113  	err = o.fs.pacer.Call(func() (bool, error) {
  1114  		blobProperties, err = blob.GetProperties(ctx, options)
  1115  		return o.fs.shouldRetry(err)
  1116  	})
  1117  	if err != nil {
  1118  		// On directories - GetProperties does not work and current SDK does not populate service code correctly hence check regular http response as well
  1119  		if storageErr, ok := err.(azblob.StorageError); ok && (storageErr.ServiceCode() == azblob.ServiceCodeBlobNotFound || storageErr.Response().StatusCode == http.StatusNotFound) {
  1120  			return fs.ErrorObjectNotFound
  1121  		}
  1122  		return err
  1123  	}
  1124  
  1125  	return o.decodeMetaDataFromPropertiesResponse(blobProperties)
  1126  }
  1127  
  1128  // parseTimeString converts a decimal string number of milliseconds
  1129  // elapsed since January 1, 1970 UTC into a time.Time and stores it in
  1130  // the modTime variable.
  1131  func (o *Object) parseTimeString(timeString string) (err error) {
  1132  	if timeString == "" {
  1133  		return nil
  1134  	}
  1135  	unixMilliseconds, err := strconv.ParseInt(timeString, 10, 64)
  1136  	if err != nil {
  1137  		fs.Debugf(o, "Failed to parse mod time string %q: %v", timeString, err)
  1138  		return err
  1139  	}
  1140  	o.modTime = time.Unix(unixMilliseconds/1E3, (unixMilliseconds%1E3)*1E6).UTC()
  1141  	return nil
  1142  }
  1143  
  1144  // ModTime returns the modification time of the object
  1145  //
  1146  // It attempts to read the objects mtime and if that isn't present the
  1147  // LastModified returned in the http headers
  1148  func (o *Object) ModTime(ctx context.Context) (result time.Time) {
  1149  	// The error is logged in readMetaData
  1150  	_ = o.readMetaData()
  1151  	return o.modTime
  1152  }
  1153  
  1154  // SetModTime sets the modification time of the local fs object
  1155  func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
  1156  	// Make sure o.meta is not nil
  1157  	if o.meta == nil {
  1158  		o.meta = make(map[string]string, 1)
  1159  	}
  1160  	// Set modTimeKey in it
  1161  	o.meta[modTimeKey] = modTime.Format(timeFormatOut)
  1162  
  1163  	blob := o.getBlobReference()
  1164  	err := o.fs.pacer.Call(func() (bool, error) {
  1165  		_, err := blob.SetMetadata(ctx, o.meta, azblob.BlobAccessConditions{})
  1166  		return o.fs.shouldRetry(err)
  1167  	})
  1168  	if err != nil {
  1169  		return err
  1170  	}
  1171  	o.modTime = modTime
  1172  	return nil
  1173  }
  1174  
  1175  // Storable returns if this object is storable
  1176  func (o *Object) Storable() bool {
  1177  	return true
  1178  }
  1179  
  1180  // Open an object for read
  1181  func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
  1182  	// Offset and Count for range download
  1183  	var offset int64
  1184  	var count int64
  1185  	if o.AccessTier() == azblob.AccessTierArchive {
  1186  		return nil, errors.Errorf("Blob in archive tier, you need to set tier to hot or cool first")
  1187  	}
  1188  
  1189  	for _, option := range options {
  1190  		switch x := option.(type) {
  1191  		case *fs.RangeOption:
  1192  			offset, count = x.Decode(o.size)
  1193  			if count < 0 {
  1194  				count = o.size - offset
  1195  			}
  1196  		case *fs.SeekOption:
  1197  			offset = x.Offset
  1198  		default:
  1199  			if option.Mandatory() {
  1200  				fs.Logf(o, "Unsupported mandatory option: %v", option)
  1201  			}
  1202  		}
  1203  	}
  1204  	blob := o.getBlobReference()
  1205  	ac := azblob.BlobAccessConditions{}
  1206  	var dowloadResponse *azblob.DownloadResponse
  1207  	err = o.fs.pacer.Call(func() (bool, error) {
  1208  		dowloadResponse, err = blob.Download(ctx, offset, count, ac, false)
  1209  		return o.fs.shouldRetry(err)
  1210  	})
  1211  	if err != nil {
  1212  		return nil, errors.Wrap(err, "failed to open for download")
  1213  	}
  1214  	in = dowloadResponse.Body(azblob.RetryReaderOptions{})
  1215  	return in, nil
  1216  }
  1217  
  1218  // dontEncode is the characters that do not need percent-encoding
  1219  //
  1220  // The characters that do not need percent-encoding are a subset of
  1221  // the printable ASCII characters: upper-case letters, lower-case
  1222  // letters, digits, ".", "_", "-", "/", "~", "!", "$", "'", "(", ")",
  1223  // "*", ";", "=", ":", and "@". All other byte values in a UTF-8 must
  1224  // be replaced with "%" and the two-digit hex value of the byte.
  1225  const dontEncode = (`abcdefghijklmnopqrstuvwxyz` +
  1226  	`ABCDEFGHIJKLMNOPQRSTUVWXYZ` +
  1227  	`0123456789` +
  1228  	`._-/~!$'()*;=:@`)
  1229  
  1230  // noNeedToEncode is a bitmap of characters which don't need % encoding
  1231  var noNeedToEncode [256]bool
  1232  
  1233  func init() {
  1234  	for _, c := range dontEncode {
  1235  		noNeedToEncode[c] = true
  1236  	}
  1237  }
  1238  
  1239  // readSeeker joins an io.Reader and an io.Seeker
  1240  type readSeeker struct {
  1241  	io.Reader
  1242  	io.Seeker
  1243  }
  1244  
  1245  // uploadMultipart uploads a file using multipart upload
  1246  //
  1247  // Write a larger blob, using CreateBlockBlob, PutBlock, and PutBlockList.
  1248  func (o *Object) uploadMultipart(in io.Reader, size int64, blob *azblob.BlobURL, httpHeaders *azblob.BlobHTTPHeaders) (err error) {
  1249  	// Calculate correct chunkSize
  1250  	chunkSize := int64(o.fs.opt.ChunkSize)
  1251  	var totalParts int64
  1252  	for {
  1253  		// Calculate number of parts
  1254  		var remainder int64
  1255  		totalParts, remainder = size/chunkSize, size%chunkSize
  1256  		if remainder != 0 {
  1257  			totalParts++
  1258  		}
  1259  		if totalParts < maxTotalParts {
  1260  			break
  1261  		}
  1262  		// Double chunk size if the number of parts is too big
  1263  		chunkSize *= 2
  1264  		if chunkSize > int64(maxChunkSize) {
  1265  			return errors.Errorf("can't upload as it is too big %v - takes more than %d chunks of %v", fs.SizeSuffix(size), totalParts, fs.SizeSuffix(chunkSize/2))
  1266  		}
  1267  	}
  1268  	fs.Debugf(o, "Multipart upload session started for %d parts of size %v", totalParts, fs.SizeSuffix(chunkSize))
  1269  
  1270  	// https://godoc.org/github.com/Azure/azure-storage-blob-go/2017-07-29/azblob#example-BlockBlobURL
  1271  	// Utilities are cloned from above example
  1272  	// These helper functions convert a binary block ID to a base-64 string and vice versa
  1273  	// NOTE: The blockID must be <= 64 bytes and ALL blockIDs for the block must be the same length
  1274  	blockIDBinaryToBase64 := func(blockID []byte) string { return base64.StdEncoding.EncodeToString(blockID) }
  1275  	// These helper functions convert an int block ID to a base-64 string and vice versa
  1276  	blockIDIntToBase64 := func(blockID uint64) string {
  1277  		binaryBlockID := (&[8]byte{})[:] // All block IDs are 8 bytes long
  1278  		binary.LittleEndian.PutUint64(binaryBlockID, blockID)
  1279  		return blockIDBinaryToBase64(binaryBlockID)
  1280  	}
  1281  
  1282  	// block ID variables
  1283  	var (
  1284  		rawID   uint64
  1285  		blockID = "" // id in base64 encoded form
  1286  		blocks  []string
  1287  	)
  1288  
  1289  	// increment the blockID
  1290  	nextID := func() {
  1291  		rawID++
  1292  		blockID = blockIDIntToBase64(rawID)
  1293  		blocks = append(blocks, blockID)
  1294  	}
  1295  
  1296  	// Get BlockBlobURL, we will use default pipeline here
  1297  	blockBlobURL := blob.ToBlockBlobURL()
  1298  	ctx := context.Background()
  1299  	ac := azblob.LeaseAccessConditions{} // Use default lease access conditions
  1300  
  1301  	// unwrap the accounting from the input, we use wrap to put it
  1302  	// back on after the buffering
  1303  	in, wrap := accounting.UnWrap(in)
  1304  
  1305  	// Upload the chunks
  1306  	remaining := size
  1307  	position := int64(0)
  1308  	errs := make(chan error, 1)
  1309  	var wg sync.WaitGroup
  1310  outer:
  1311  	for part := 0; part < int(totalParts); part++ {
  1312  		// Check any errors
  1313  		select {
  1314  		case err = <-errs:
  1315  			break outer
  1316  		default:
  1317  		}
  1318  
  1319  		reqSize := remaining
  1320  		if reqSize >= chunkSize {
  1321  			reqSize = chunkSize
  1322  		}
  1323  
  1324  		// Make a block of memory
  1325  		buf := make([]byte, reqSize)
  1326  
  1327  		// Read the chunk
  1328  		_, err = io.ReadFull(in, buf)
  1329  		if err != nil {
  1330  			err = errors.Wrap(err, "multipart upload failed to read source")
  1331  			break outer
  1332  		}
  1333  
  1334  		// Transfer the chunk
  1335  		nextID()
  1336  		wg.Add(1)
  1337  		o.fs.uploadToken.Get()
  1338  		go func(part int, position int64, blockID string) {
  1339  			defer wg.Done()
  1340  			defer o.fs.uploadToken.Put()
  1341  			fs.Debugf(o, "Uploading part %d/%d offset %v/%v part size %v", part+1, totalParts, fs.SizeSuffix(position), fs.SizeSuffix(size), fs.SizeSuffix(chunkSize))
  1342  
  1343  			// Upload the block, with MD5 for check
  1344  			md5sum := md5.Sum(buf)
  1345  			transactionalMD5 := md5sum[:]
  1346  			err = o.fs.pacer.Call(func() (bool, error) {
  1347  				bufferReader := bytes.NewReader(buf)
  1348  				wrappedReader := wrap(bufferReader)
  1349  				rs := readSeeker{wrappedReader, bufferReader}
  1350  				_, err = blockBlobURL.StageBlock(ctx, blockID, &rs, ac, transactionalMD5)
  1351  				return o.fs.shouldRetry(err)
  1352  			})
  1353  
  1354  			if err != nil {
  1355  				err = errors.Wrap(err, "multipart upload failed to upload part")
  1356  				select {
  1357  				case errs <- err:
  1358  				default:
  1359  				}
  1360  				return
  1361  			}
  1362  		}(part, position, blockID)
  1363  
  1364  		// ready for next block
  1365  		remaining -= chunkSize
  1366  		position += chunkSize
  1367  	}
  1368  	wg.Wait()
  1369  	if err == nil {
  1370  		select {
  1371  		case err = <-errs:
  1372  		default:
  1373  		}
  1374  	}
  1375  	if err != nil {
  1376  		return err
  1377  	}
  1378  
  1379  	// Finalise the upload session
  1380  	err = o.fs.pacer.Call(func() (bool, error) {
  1381  		_, err := blockBlobURL.CommitBlockList(ctx, blocks, *httpHeaders, o.meta, azblob.BlobAccessConditions{})
  1382  		return o.fs.shouldRetry(err)
  1383  	})
  1384  	if err != nil {
  1385  		return errors.Wrap(err, "multipart upload failed to finalize")
  1386  	}
  1387  	return nil
  1388  }
  1389  
  1390  // Update the object with the contents of the io.Reader, modTime and size
  1391  //
  1392  // The new object may have been created if an error is returned
  1393  func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
  1394  	err = o.fs.Mkdir(ctx, "")
  1395  	if err != nil {
  1396  		return err
  1397  	}
  1398  	size := src.Size()
  1399  	// Update Mod time
  1400  	o.updateMetadataWithModTime(src.ModTime(ctx))
  1401  	if err != nil {
  1402  		return err
  1403  	}
  1404  
  1405  	blob := o.getBlobReference()
  1406  	httpHeaders := azblob.BlobHTTPHeaders{}
  1407  	httpHeaders.ContentType = fs.MimeType(ctx, o)
  1408  	// Compute the Content-MD5 of the file, for multiparts uploads it
  1409  	// will be set in PutBlockList API call using the 'x-ms-blob-content-md5' header
  1410  	// Note: If multipart, a MD5 checksum will also be computed for each uploaded block
  1411  	// 		 in order to validate its integrity during transport
  1412  	if sourceMD5, _ := src.Hash(ctx, hash.MD5); sourceMD5 != "" {
  1413  		sourceMD5bytes, err := hex.DecodeString(sourceMD5)
  1414  		if err == nil {
  1415  			httpHeaders.ContentMD5 = sourceMD5bytes
  1416  		} else {
  1417  			fs.Debugf(o, "Failed to decode %q as MD5: %v", sourceMD5, err)
  1418  		}
  1419  	}
  1420  
  1421  	putBlobOptions := azblob.UploadStreamToBlockBlobOptions{
  1422  		BufferSize:      int(o.fs.opt.ChunkSize),
  1423  		MaxBuffers:      4,
  1424  		Metadata:        o.meta,
  1425  		BlobHTTPHeaders: httpHeaders,
  1426  	}
  1427  	// FIXME Until https://github.com/Azure/azure-storage-blob-go/pull/75
  1428  	// is merged the SDK can't upload a single blob of exactly the chunk
  1429  	// size, so upload with a multpart upload to work around.
  1430  	// See: https://github.com/ncw/rclone/issues/2653
  1431  	multipartUpload := size >= int64(o.fs.opt.UploadCutoff)
  1432  	if size == int64(o.fs.opt.ChunkSize) {
  1433  		multipartUpload = true
  1434  		fs.Debugf(o, "Setting multipart upload for file of chunk size (%d) to work around SDK bug", size)
  1435  	}
  1436  
  1437  	// Don't retry, return a retry error instead
  1438  	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
  1439  		if multipartUpload {
  1440  			// If a large file upload in chunks
  1441  			err = o.uploadMultipart(in, size, &blob, &httpHeaders)
  1442  		} else {
  1443  			// Write a small blob in one transaction
  1444  			blockBlobURL := blob.ToBlockBlobURL()
  1445  			_, err = azblob.UploadStreamToBlockBlob(ctx, in, blockBlobURL, putBlobOptions)
  1446  		}
  1447  		return o.fs.shouldRetry(err)
  1448  	})
  1449  	if err != nil {
  1450  		return err
  1451  	}
  1452  	// Refresh metadata on object
  1453  	o.clearMetaData()
  1454  	err = o.readMetaData()
  1455  	if err != nil {
  1456  		return err
  1457  	}
  1458  
  1459  	// If tier is not changed or not specified, do not attempt to invoke `SetBlobTier` operation
  1460  	if o.fs.opt.AccessTier == string(defaultAccessTier) || o.fs.opt.AccessTier == string(o.AccessTier()) {
  1461  		return nil
  1462  	}
  1463  
  1464  	// Now, set blob tier based on configured access tier
  1465  	return o.SetTier(o.fs.opt.AccessTier)
  1466  }
  1467  
  1468  // Remove an object
  1469  func (o *Object) Remove(ctx context.Context) error {
  1470  	blob := o.getBlobReference()
  1471  	snapShotOptions := azblob.DeleteSnapshotsOptionNone
  1472  	ac := azblob.BlobAccessConditions{}
  1473  	return o.fs.pacer.Call(func() (bool, error) {
  1474  		_, err := blob.Delete(ctx, snapShotOptions, ac)
  1475  		return o.fs.shouldRetry(err)
  1476  	})
  1477  }
  1478  
  1479  // MimeType of an Object if known, "" otherwise
  1480  func (o *Object) MimeType(ctx context.Context) string {
  1481  	return o.mimeType
  1482  }
  1483  
  1484  // AccessTier of an object, default is of type none
  1485  func (o *Object) AccessTier() azblob.AccessTierType {
  1486  	return o.accessTier
  1487  }
  1488  
  1489  // SetTier performs changing object tier
  1490  func (o *Object) SetTier(tier string) error {
  1491  	if !validateAccessTier(tier) {
  1492  		return errors.Errorf("Tier %s not supported by Azure Blob Storage", tier)
  1493  	}
  1494  
  1495  	// Check if current tier already matches with desired tier
  1496  	if o.GetTier() == tier {
  1497  		return nil
  1498  	}
  1499  	desiredAccessTier := azblob.AccessTierType(tier)
  1500  	blob := o.getBlobReference()
  1501  	ctx := context.Background()
  1502  	err := o.fs.pacer.Call(func() (bool, error) {
  1503  		_, err := blob.SetTier(ctx, desiredAccessTier, azblob.LeaseAccessConditions{})
  1504  		return o.fs.shouldRetry(err)
  1505  	})
  1506  
  1507  	if err != nil {
  1508  		return errors.Wrap(err, "Failed to set Blob Tier")
  1509  	}
  1510  
  1511  	// Set access tier on local object also, this typically
  1512  	// gets updated on get blob properties
  1513  	o.accessTier = desiredAccessTier
  1514  	fs.Debugf(o, "Successfully changed object tier to %s", tier)
  1515  
  1516  	return nil
  1517  }
  1518  
  1519  // GetTier returns object tier in azure as string
  1520  func (o *Object) GetTier() string {
  1521  	return string(o.accessTier)
  1522  }
  1523  
  1524  // Check the interfaces are satisfied
  1525  var (
  1526  	_ fs.Fs        = &Fs{}
  1527  	_ fs.Copier    = &Fs{}
  1528  	_ fs.Purger    = &Fs{}
  1529  	_ fs.ListRer   = &Fs{}
  1530  	_ fs.Object    = &Object{}
  1531  	_ fs.MimeTyper = &Object{}
  1532  )