github.com/StarfishStorage/goofys@v0.23.2-0.20200415030923-535558486b34/internal/backend_azblob.go (about) 1 // Copyright 2019 Ka-Hing Cheung 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package internal 16 17 import ( 18 . "github.com/kahing/goofys/api/common" 19 20 "bytes" 21 "context" 22 "encoding/base64" 23 "fmt" 24 "net/http" 25 "net/url" 26 "sort" 27 "strings" 28 "sync" 29 "sync/atomic" 30 "syscall" 31 "time" 32 33 "github.com/Azure/azure-pipeline-go/pipeline" 34 "github.com/Azure/azure-storage-blob-go/azblob" 35 36 "github.com/google/uuid" 37 "github.com/jacobsa/fuse" 38 "github.com/sirupsen/logrus" 39 ) 40 41 const AzuriteEndpoint = "http://127.0.0.1:8080/devstoreaccount1/" 42 const AzureDirBlobMetadataKey = "hdi_isfolder" 43 const AzureBlobMetaDataHeaderPrefix = "x-ms-meta-" 44 45 // Azure Blob Store API does not not treat headers as case insensitive. 46 // This is particularly a problem with `AzureDirBlobMetadataKey` header. 47 // pipelineWrapper wraps around an implementation of `Pipeline` and 48 // changes the Do function to update the input request headers before invoking 49 // Do on the wrapping Pipeline onject. 50 type pipelineWrapper struct { 51 p pipeline.Pipeline 52 } 53 54 type requestWrapper struct { 55 pipeline.Request 56 } 57 58 var pipelineHTTPClient = newDefaultHTTPClient() 59 60 // Clone of https://github.com/Azure/azure-pipeline-go/blob/master/pipeline/core.go#L202 61 func newDefaultHTTPClient() *http.Client { 62 return &http.Client{ 63 Transport: GetHTTPTransport(), 64 } 65 } 66 67 // Creates a pipeline.Factory object that fixes headers related to azure blob store 68 // and sends HTTP requests to Go's default http.Client. 69 func newAzBlobHTTPClientFactory() pipeline.Factory { 70 return pipeline.FactoryFunc( 71 func(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.PolicyFunc { 72 return func(ctx context.Context, request pipeline.Request) (pipeline.Response, error) { 73 // Fix the Azure Blob store metadata headers. 74 // Problem: 75 // - Golang canonicalizes headers and converts them into camel case 76 // because HTTP headers are supposed to be case insensitive. E.g After 77 // canonicalization, 'foo-bar' becomes 'Foo-Bar'. 78 // - Azure API treats HTTP headers in case sensitive manner. 79 // Solution: Convert the problematic headers to lower case. 80 for key, value := range request.Header { 81 keyLower := strings.ToLower(key) 82 // We are mofifying the map while iterating on it. So we check for 83 // keyLower != key to avoid potential infinite loop. 84 // See https://golang.org/ref/spec#RangeClause for more info. 85 if keyLower != key && strings.Contains(keyLower, AzureBlobMetaDataHeaderPrefix) { 86 request.Header.Del(key) 87 request.Header[keyLower] = value 88 } 89 } 90 // Send the HTTP request. 91 r, err := pipelineHTTPClient.Do(request.WithContext(ctx)) 92 if err != nil { 93 err = pipeline.NewError(err, "HTTP request failed") 94 } 95 return pipeline.NewHTTPResponse(r), err 96 } 97 }) 98 } 99 100 type AZBlob struct { 101 config *AZBlobConfig 102 cap Capabilities 103 104 mu sync.Mutex 105 u *azblob.ServiceURL 106 c *azblob.ContainerURL 107 108 pipeline pipeline.Pipeline 109 110 bucket string 111 bareURL string 112 sasTokenProvider SASTokenProvider 113 tokenExpire time.Time 114 tokenRenewBuffer time.Duration 115 tokenRenewGate *Ticket 116 } 117 118 var azbLog = GetLogger("azblob") 119 120 func NewAZBlob(container string, config *AZBlobConfig) (*AZBlob, error) { 121 po := azblob.PipelineOptions{ 122 Log: pipeline.LogOptions{ 123 Log: func(level pipeline.LogLevel, msg string) { 124 // naive casting kind of works because pipeline.INFO maps 125 // to 5 which is logrus.DEBUG 126 if level == pipeline.LogError { 127 // somehow some http errors 128 // are logged at Error, we 129 // already log unhandled 130 // errors so no need to do 131 // that here 132 level = pipeline.LogInfo 133 } 134 azbLog.Log(logrus.Level(uint32(level)), msg) 135 }, 136 ShouldLog: func(level pipeline.LogLevel) bool { 137 if level == pipeline.LogError { 138 // somehow some http errors 139 // are logged at Error, we 140 // already log unhandled 141 // errors so no need to do 142 // that here 143 level = pipeline.LogInfo 144 } 145 return azbLog.IsLevelEnabled(logrus.Level(uint32(level))) 146 }, 147 }, 148 RequestLog: azblob.RequestLogOptions{ 149 LogWarningIfTryOverThreshold: time.Duration(-1), 150 }, 151 HTTPSender: newAzBlobHTTPClientFactory(), 152 } 153 154 p := azblob.NewPipeline(azblob.NewAnonymousCredential(), po) 155 bareURL := config.Endpoint 156 157 var bu *azblob.ServiceURL 158 var bc *azblob.ContainerURL 159 160 if config.SasToken == nil { 161 credential, err := azblob.NewSharedKeyCredential(config.AccountName, config.AccountKey) 162 if err != nil { 163 return nil, fmt.Errorf("Unable to construct credential: %v", err) 164 } 165 166 p = azblob.NewPipeline(credential, po) 167 168 u, err := url.Parse(bareURL) 169 if err != nil { 170 return nil, err 171 } 172 173 serviceURL := azblob.NewServiceURL(*u, p) 174 containerURL := serviceURL.NewContainerURL(container) 175 176 bu = &serviceURL 177 bc = &containerURL 178 } 179 180 b := &AZBlob{ 181 config: config, 182 cap: Capabilities{ 183 MaxMultipartSize: 100 * 1024 * 1024, 184 Name: "wasb", 185 }, 186 pipeline: p, 187 bucket: container, 188 bareURL: bareURL, 189 sasTokenProvider: config.SasToken, 190 u: bu, 191 c: bc, 192 tokenRenewBuffer: config.TokenRenewBuffer, 193 tokenRenewGate: Ticket{Total: 1}.Init(), 194 } 195 196 return b, nil 197 } 198 199 func (b *AZBlob) Delegate() interface{} { 200 return b 201 } 202 203 func (b *AZBlob) Capabilities() *Capabilities { 204 return &b.cap 205 } 206 207 func (b *AZBlob) Bucket() string { 208 return b.bucket 209 } 210 211 func (b *AZBlob) refreshToken() (*azblob.ContainerURL, error) { 212 if b.sasTokenProvider == nil { 213 return b.c, nil 214 } 215 216 b.mu.Lock() 217 218 if b.c == nil { 219 b.mu.Unlock() 220 return b.updateToken() 221 } else if b.tokenExpire.Before(time.Now().UTC()) { 222 // our token totally expired, renew inline before using it 223 b.mu.Unlock() 224 b.tokenRenewGate.Take(1, true) 225 defer b.tokenRenewGate.Return(1) 226 227 b.mu.Lock() 228 // check again, because in the mean time maybe it's renewed 229 if b.tokenExpire.Before(time.Now().UTC()) { 230 b.mu.Unlock() 231 azbLog.Warnf("token expired: %v", b.tokenExpire) 232 _, err := b.updateToken() 233 if err != nil { 234 azbLog.Errorf("Unable to refresh token: %v", err) 235 return nil, syscall.EACCES 236 } 237 } else { 238 // another concurrent goroutine renewed it for us 239 b.mu.Unlock() 240 } 241 } else if b.tokenExpire.Add(b.tokenRenewBuffer).Before(time.Now().UTC()) { 242 b.mu.Unlock() 243 // only allow one token renew at a time 244 if b.tokenRenewGate.Take(1, false) { 245 246 go func() { 247 defer b.tokenRenewGate.Return(1) 248 _, err := b.updateToken() 249 if err != nil { 250 azbLog.Errorf("Unable to refresh token: %v", err) 251 } 252 }() 253 254 // if we cannot renew token, treat it as a 255 // transient failure because the token is 256 // still valid for a while. When the grace 257 // period is over we will get an error when we 258 // actually access the blob store 259 } else { 260 // another goroutine is already renewing 261 azbLog.Infof("token renewal already in progress") 262 } 263 } else { 264 b.mu.Unlock() 265 } 266 return b.c, nil 267 } 268 269 func parseSasToken(token string) (expire time.Time) { 270 expire = TIME_MAX 271 272 parts, err := url.ParseQuery(token) 273 if err != nil { 274 return 275 } 276 277 se := parts.Get("se") 278 if se == "" { 279 azbLog.Error("token missing 'se' param") 280 return 281 } 282 283 expire, err = time.Parse("2006-01-02T15:04:05Z", se) 284 if err != nil { 285 // sometimes they only have the date 286 expire, err = time.Parse("2006-01-02", se) 287 if err != nil { 288 expire = TIME_MAX 289 } 290 } 291 return 292 } 293 294 func (b *AZBlob) updateToken() (*azblob.ContainerURL, error) { 295 token, err := b.sasTokenProvider() 296 if err != nil { 297 azbLog.Errorf("Unable to generate SAS token: %v", err) 298 return nil, syscall.EACCES 299 } 300 301 expire := parseSasToken(token) 302 azbLog.Infof("token for %v refreshed, next expire at %v", b.bucket, expire.String()) 303 304 sUrl := b.bareURL + "?" + token 305 u, err := url.Parse(sUrl) 306 if err != nil { 307 azbLog.Errorf("Unable to construct service URL: %v", sUrl) 308 return nil, fuse.EINVAL 309 } 310 311 serviceURL := azblob.NewServiceURL(*u, b.pipeline) 312 containerURL := serviceURL.NewContainerURL(b.bucket) 313 314 b.mu.Lock() 315 defer b.mu.Unlock() 316 317 b.u = &serviceURL 318 b.c = &containerURL 319 b.tokenExpire = expire 320 321 return b.c, nil 322 } 323 324 func (b *AZBlob) testBucket(key string) (err error) { 325 _, err = b.HeadBlob(&HeadBlobInput{Key: key}) 326 if err != nil { 327 err = mapAZBError(err) 328 if err == fuse.ENOENT { 329 err = nil 330 } 331 } 332 333 return 334 } 335 336 func (b *AZBlob) Init(key string) error { 337 _, err := b.refreshToken() 338 if err != nil { 339 return err 340 } 341 342 err = b.testBucket(key) 343 return err 344 } 345 346 func mapAZBError(err error) error { 347 if err == nil { 348 return nil 349 } 350 351 if stgErr, ok := err.(azblob.StorageError); ok { 352 switch stgErr.ServiceCode() { 353 case azblob.ServiceCodeBlobAlreadyExists: 354 return syscall.EACCES 355 case azblob.ServiceCodeBlobNotFound: 356 return fuse.ENOENT 357 case azblob.ServiceCodeContainerAlreadyExists: 358 return syscall.EEXIST 359 case azblob.ServiceCodeContainerBeingDeleted: 360 return syscall.EAGAIN 361 case azblob.ServiceCodeContainerDisabled: 362 return syscall.EACCES 363 case azblob.ServiceCodeContainerNotFound: 364 return syscall.ENODEV 365 case azblob.ServiceCodeCopyAcrossAccountsNotSupported: 366 return fuse.EINVAL 367 case azblob.ServiceCodeSourceConditionNotMet: 368 return fuse.EINVAL 369 case azblob.ServiceCodeSystemInUse: 370 return syscall.EAGAIN 371 case azblob.ServiceCodeTargetConditionNotMet: 372 return fuse.EINVAL 373 case azblob.ServiceCodeBlobBeingRehydrated: 374 return syscall.EAGAIN 375 case azblob.ServiceCodeBlobArchived: 376 return fuse.EINVAL 377 case azblob.ServiceCodeAccountBeingCreated: 378 return syscall.EAGAIN 379 case azblob.ServiceCodeAuthenticationFailed: 380 return syscall.EACCES 381 case azblob.ServiceCodeConditionNotMet: 382 return syscall.EBUSY 383 case azblob.ServiceCodeInternalError: 384 return syscall.EAGAIN 385 case azblob.ServiceCodeInvalidAuthenticationInfo: 386 return syscall.EACCES 387 case azblob.ServiceCodeOperationTimedOut: 388 return syscall.EAGAIN 389 case azblob.ServiceCodeResourceNotFound: 390 return fuse.ENOENT 391 case azblob.ServiceCodeServerBusy: 392 return syscall.EAGAIN 393 case "AuthorizationFailure": // from Azurite emulator 394 return syscall.EACCES 395 default: 396 err = mapHttpError(stgErr.Response().StatusCode) 397 if err != nil { 398 return err 399 } else { 400 azbLog.Errorf("code=%v status=%v err=%v", stgErr.ServiceCode(), stgErr.Response().Status, stgErr) 401 return stgErr 402 } 403 } 404 } else { 405 return err 406 } 407 } 408 409 func pMetadata(m map[string]string) map[string]*string { 410 metadata := make(map[string]*string) 411 for k, _ := range m { 412 k = strings.ToLower(k) 413 v := m[k] 414 metadata[k] = &v 415 } 416 return metadata 417 } 418 419 func nilMetadata(m map[string]*string) map[string]string { 420 metadata := make(map[string]string) 421 for k, v := range m { 422 k = strings.ToLower(k) 423 metadata[k] = nilStr(v) 424 } 425 return metadata 426 } 427 428 func (b *AZBlob) HeadBlob(param *HeadBlobInput) (*HeadBlobOutput, error) { 429 c, err := b.refreshToken() 430 if err != nil { 431 return nil, err 432 } 433 434 if strings.HasSuffix(param.Key, "/") { 435 dirBlob, err := b.HeadBlob(&HeadBlobInput{Key: param.Key[:len(param.Key)-1]}) 436 if err == nil { 437 if !dirBlob.IsDirBlob { 438 // we requested for a dir suffix, but this isn't one 439 err = fuse.ENOENT 440 } 441 } 442 return dirBlob, err 443 } 444 445 blob := c.NewBlobURL(param.Key) 446 resp, err := blob.GetProperties(context.TODO(), azblob.BlobAccessConditions{}) 447 if err != nil { 448 return nil, mapAZBError(err) 449 } 450 451 metadata := resp.NewMetadata() 452 isDir := strings.HasSuffix(param.Key, "/") 453 if !isDir && metadata != nil { 454 _, isDir = metadata[AzureDirBlobMetadataKey] 455 } 456 // don't expose this to user land 457 delete(metadata, AzureDirBlobMetadataKey) 458 459 return &HeadBlobOutput{ 460 BlobItemOutput: BlobItemOutput{ 461 Key: ¶m.Key, 462 ETag: PString(string(resp.ETag())), 463 LastModified: PTime(resp.LastModified()), 464 Size: uint64(resp.ContentLength()), 465 StorageClass: PString(resp.AccessTier()), 466 }, 467 ContentType: PString(resp.ContentType()), 468 Metadata: pMetadata(metadata), 469 IsDirBlob: isDir, 470 }, nil 471 } 472 473 func nilStr(v *string) string { 474 if v == nil { 475 return "" 476 } else { 477 return *v 478 } 479 } 480 func nilUint32(v *uint32) uint32 { 481 if v == nil { 482 return 0 483 } else { 484 return *v 485 } 486 } 487 488 func (b *AZBlob) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error) { 489 // azure blob does not support startAfter 490 if param.StartAfter != nil { 491 return nil, syscall.ENOTSUP 492 } 493 494 c, err := b.refreshToken() 495 if err != nil { 496 return nil, err 497 } 498 499 prefixes := make([]BlobPrefixOutput, 0) 500 items := make([]BlobItemOutput, 0) 501 502 var blobItems []azblob.BlobItem 503 var nextMarker *string 504 505 options := azblob.ListBlobsSegmentOptions{ 506 Prefix: nilStr(param.Prefix), 507 MaxResults: int32(nilUint32(param.MaxKeys)), 508 Details: azblob.BlobListingDetails{ 509 // blobfuse (following wasb) convention uses 510 // an empty blob with "hdi_isfolder" metadata 511 // set to represent a folder. So we include 512 // metadaata in listing to discover that and 513 // convert the result back to what we expect 514 // (which is a "dir/" blob) 515 // https://github.com/Azure/azure-storage-fuse/issues/222 516 // https://blogs.msdn.microsoft.com/mostlytrue/2014/04/22/wasb-back-stories-masquerading-a-key-value-store/ 517 Metadata: true, 518 }, 519 } 520 521 if param.Delimiter != nil { 522 resp, err := c.ListBlobsHierarchySegment(context.TODO(), 523 azblob.Marker{ 524 param.ContinuationToken, 525 }, 526 nilStr(param.Delimiter), 527 options) 528 if err != nil { 529 return nil, mapAZBError(err) 530 } 531 532 for i, _ := range resp.Segment.BlobPrefixes { 533 p := resp.Segment.BlobPrefixes[i] 534 prefixes = append(prefixes, BlobPrefixOutput{Prefix: &p.Name}) 535 } 536 537 if b.config.Endpoint == AzuriteEndpoint && 538 // XXX in Azurite this is not sorted 539 !sort.IsSorted(sortBlobPrefixOutput(prefixes)) { 540 sort.Sort(sortBlobPrefixOutput(prefixes)) 541 } 542 543 blobItems = resp.Segment.BlobItems 544 nextMarker = resp.NextMarker.Val 545 } else { 546 resp, err := c.ListBlobsFlatSegment(context.TODO(), 547 azblob.Marker{ 548 param.ContinuationToken, 549 }, 550 options) 551 if err != nil { 552 return nil, mapAZBError(err) 553 } 554 555 blobItems = resp.Segment.BlobItems 556 nextMarker = resp.NextMarker.Val 557 558 if b.config.Endpoint == AzuriteEndpoint && 559 !sort.IsSorted(sortBlobItemOutput(items)) { 560 sort.Sort(sortBlobItemOutput(items)) 561 } 562 } 563 564 var sortItems bool 565 566 for idx, _ := range blobItems { 567 i := &blobItems[idx] 568 p := &i.Properties 569 570 if i.Metadata[AzureDirBlobMetadataKey] != "" { 571 i.Name = i.Name + "/" 572 573 if param.Delimiter != nil { 574 // do we already have such a prefix? 575 n := len(prefixes) 576 if idx := sort.Search(n, func(idx int) bool { 577 return *prefixes[idx].Prefix >= i.Name 578 }); idx >= n || *prefixes[idx].Prefix != i.Name { 579 if idx >= n { 580 prefixes = append(prefixes, BlobPrefixOutput{ 581 Prefix: &i.Name, 582 }) 583 } else { 584 prefixes = append(prefixes, BlobPrefixOutput{}) 585 copy(prefixes[idx+1:], prefixes[idx:]) 586 prefixes[idx].Prefix = &i.Name 587 } 588 } 589 continue 590 } else { 591 sortItems = true 592 } 593 } 594 595 items = append(items, BlobItemOutput{ 596 Key: &i.Name, 597 ETag: PString(string(p.Etag)), 598 LastModified: PTime(p.LastModified), 599 Size: uint64(*p.ContentLength), 600 StorageClass: PString(string(p.AccessTier)), 601 }) 602 } 603 604 if strings.HasSuffix(options.Prefix, "/") { 605 // because azure doesn't use dir/ blobs, dir/ would not show up 606 // so we make another request to fill that in 607 dirBlob, err := b.HeadBlob(&HeadBlobInput{options.Prefix}) 608 if err == nil { 609 *dirBlob.Key += "/" 610 items = append(items, dirBlob.BlobItemOutput) 611 sortItems = true 612 } else if err == fuse.ENOENT { 613 err = nil 614 } else { 615 return nil, err 616 } 617 } 618 619 // items are supposed to be alphabetical, but if there was a directory we would 620 // have changed the ordering. XXX re-sort this for now but we can probably 621 // insert smarter instead 622 if sortItems { 623 sort.Sort(sortBlobItemOutput(items)) 624 } 625 626 if nextMarker != nil && *nextMarker == "" { 627 nextMarker = nil 628 } 629 630 return &ListBlobsOutput{ 631 Prefixes: prefixes, 632 Items: items, 633 NextContinuationToken: nextMarker, 634 IsTruncated: nextMarker != nil, 635 }, nil 636 } 637 638 func (b *AZBlob) DeleteBlob(param *DeleteBlobInput) (*DeleteBlobOutput, error) { 639 c, err := b.refreshToken() 640 if err != nil { 641 return nil, err 642 } 643 644 if strings.HasSuffix(param.Key, "/") { 645 return b.DeleteBlob(&DeleteBlobInput{Key: param.Key[:len(param.Key)-1]}) 646 } 647 648 blob := c.NewBlobURL(param.Key) 649 _, err = blob.Delete(context.TODO(), azblob.DeleteSnapshotsOptionInclude, azblob.BlobAccessConditions{}) 650 if err != nil { 651 return nil, mapAZBError(err) 652 } 653 return &DeleteBlobOutput{}, nil 654 } 655 656 func (b *AZBlob) DeleteBlobs(param *DeleteBlobsInput) (ret *DeleteBlobsOutput, deleteError error) { 657 var wg sync.WaitGroup 658 defer func() { 659 wg.Wait() 660 if deleteError != nil { 661 ret = nil 662 } else { 663 ret = &DeleteBlobsOutput{} 664 } 665 }() 666 667 for _, i := range param.Items { 668 SmallActionsGate.Take(1, true) 669 wg.Add(1) 670 671 go func(key string) { 672 defer func() { 673 SmallActionsGate.Return(1) 674 wg.Done() 675 }() 676 677 _, err := b.DeleteBlob(&DeleteBlobInput{key}) 678 if err != nil { 679 err = mapAZBError(err) 680 if err != fuse.ENOENT { 681 deleteError = err 682 } 683 } 684 }(i) 685 686 if deleteError != nil { 687 return 688 } 689 } 690 691 return 692 } 693 694 func (b *AZBlob) RenameBlob(param *RenameBlobInput) (*RenameBlobOutput, error) { 695 return nil, syscall.ENOTSUP 696 } 697 698 func (b *AZBlob) CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error) { 699 if strings.HasSuffix(param.Source, "/") && strings.HasSuffix(param.Destination, "/") { 700 param.Source = param.Source[:len(param.Source)-1] 701 param.Destination = param.Destination[:len(param.Destination)-1] 702 return b.CopyBlob(param) 703 } 704 705 c, err := b.refreshToken() 706 if err != nil { 707 return nil, err 708 } 709 710 src := c.NewBlobURL(param.Source) 711 dest := c.NewBlobURL(param.Destination) 712 resp, err := dest.StartCopyFromURL(context.TODO(), src.URL(), nilMetadata(param.Metadata), 713 azblob.ModifiedAccessConditions{}, azblob.BlobAccessConditions{}) 714 if err != nil { 715 return nil, mapAZBError(err) 716 } 717 718 if resp.CopyStatus() == azblob.CopyStatusPending { 719 time.Sleep(50 * time.Millisecond) 720 721 var copy *azblob.BlobGetPropertiesResponse 722 for copy, err = dest.GetProperties(context.TODO(), azblob.BlobAccessConditions{}); err == nil; copy, err = dest.GetProperties(context.TODO(), azblob.BlobAccessConditions{}) { 723 // if there's a new copy, we can only assume the last one was done 724 if copy.CopyStatus() != azblob.CopyStatusPending || copy.CopyID() != resp.CopyID() { 725 break 726 } 727 } 728 if err != nil { 729 return nil, mapAZBError(err) 730 } 731 } 732 733 return &CopyBlobOutput{}, nil 734 } 735 736 func (b *AZBlob) GetBlob(param *GetBlobInput) (*GetBlobOutput, error) { 737 c, err := b.refreshToken() 738 if err != nil { 739 return nil, err 740 } 741 742 blob := c.NewBlobURL(param.Key) 743 var ifMatch azblob.ETag 744 if param.IfMatch != nil { 745 ifMatch = azblob.ETag(*param.IfMatch) 746 } 747 748 resp, err := blob.Download(context.TODO(), 749 int64(param.Start), int64(param.Count), 750 azblob.BlobAccessConditions{ 751 ModifiedAccessConditions: azblob.ModifiedAccessConditions{ 752 IfMatch: ifMatch, 753 }, 754 }, false) 755 if err != nil { 756 return nil, mapAZBError(err) 757 } 758 759 metadata := pMetadata(resp.NewMetadata()) 760 delete(metadata, AzureDirBlobMetadataKey) 761 762 return &GetBlobOutput{ 763 HeadBlobOutput: HeadBlobOutput{ 764 BlobItemOutput: BlobItemOutput{ 765 Key: ¶m.Key, 766 ETag: PString(string(resp.ETag())), 767 LastModified: PTime(resp.LastModified()), 768 Size: uint64(resp.ContentLength()), 769 }, 770 ContentType: PString(resp.ContentType()), 771 Metadata: metadata, 772 }, 773 Body: resp.Body(azblob.RetryReaderOptions{}), 774 }, nil 775 } 776 777 func (b *AZBlob) PutBlob(param *PutBlobInput) (*PutBlobOutput, error) { 778 c, err := b.refreshToken() 779 if err != nil { 780 return nil, err 781 } 782 783 if param.DirBlob && strings.HasSuffix(param.Key, "/") { 784 // turn this into an empty blob with "hdi_isfolder" metadata 785 param.Key = param.Key[:len(param.Key)-1] 786 if param.Metadata != nil { 787 param.Metadata[AzureDirBlobMetadataKey] = PString("true") 788 } else { 789 param.Metadata = map[string]*string{ 790 AzureDirBlobMetadataKey: PString("true"), 791 } 792 } 793 return b.PutBlob(param) 794 } 795 796 body := param.Body 797 if body == nil { 798 body = bytes.NewReader([]byte("")) 799 } 800 801 blob := c.NewBlobURL(param.Key).ToBlockBlobURL() 802 resp, err := blob.Upload(context.TODO(), 803 body, 804 azblob.BlobHTTPHeaders{ 805 ContentType: nilStr(param.ContentType), 806 }, 807 nilMetadata(param.Metadata), azblob.BlobAccessConditions{}) 808 if err != nil { 809 return nil, mapAZBError(err) 810 } 811 812 return &PutBlobOutput{ 813 ETag: PString(string(resp.ETag())), 814 }, nil 815 } 816 817 func (b *AZBlob) MultipartBlobBegin(param *MultipartBlobBeginInput) (*MultipartBlobCommitInput, error) { 818 // we can have up to 50K parts, so %05d should be sufficient 819 uploadId := uuid.New().String() + "::%05d" 820 821 // this is implicitly done on the server side 822 return &MultipartBlobCommitInput{ 823 Key: ¶m.Key, 824 Metadata: param.Metadata, 825 UploadId: &uploadId, 826 Parts: make([]*string, 50000), // at most 50K parts 827 }, nil 828 } 829 830 func (b *AZBlob) MultipartBlobAdd(param *MultipartBlobAddInput) (*MultipartBlobAddOutput, error) { 831 c, err := b.refreshToken() 832 if err != nil { 833 return nil, err 834 } 835 836 blob := c.NewBlockBlobURL(*param.Commit.Key) 837 blockId := fmt.Sprintf(*param.Commit.UploadId, param.PartNumber) 838 base64BlockId := base64.StdEncoding.EncodeToString([]byte(blockId)) 839 840 atomic.AddUint32(¶m.Commit.NumParts, 1) 841 842 _, err = blob.StageBlock(context.TODO(), base64BlockId, param.Body, 843 azblob.LeaseAccessConditions{}, nil) 844 if err != nil { 845 return nil, mapAZBError(err) 846 } 847 848 param.Commit.Parts[param.PartNumber-1] = &base64BlockId 849 850 return &MultipartBlobAddOutput{}, nil 851 } 852 853 func (b *AZBlob) MultipartBlobAbort(param *MultipartBlobCommitInput) (*MultipartBlobAbortOutput, error) { 854 // no-op, server will garbage collect them 855 return &MultipartBlobAbortOutput{}, nil 856 } 857 858 func (b *AZBlob) MultipartBlobCommit(param *MultipartBlobCommitInput) (*MultipartBlobCommitOutput, error) { 859 c, err := b.refreshToken() 860 if err != nil { 861 return nil, err 862 } 863 864 blob := c.NewBlockBlobURL(*param.Key) 865 parts := make([]string, param.NumParts) 866 867 for i := uint32(0); i < param.NumParts; i++ { 868 parts[i] = *param.Parts[i] 869 } 870 871 resp, err := blob.CommitBlockList(context.TODO(), parts, 872 azblob.BlobHTTPHeaders{}, nilMetadata(param.Metadata), 873 azblob.BlobAccessConditions{}) 874 if err != nil { 875 return nil, mapAZBError(err) 876 } 877 878 return &MultipartBlobCommitOutput{ 879 ETag: PString(string(resp.ETag())), 880 }, nil 881 } 882 883 func (b *AZBlob) MultipartExpire(param *MultipartExpireInput) (*MultipartExpireOutput, error) { 884 return nil, syscall.ENOTSUP 885 } 886 887 func (b *AZBlob) RemoveBucket(param *RemoveBucketInput) (*RemoveBucketOutput, error) { 888 c, err := b.refreshToken() 889 if err != nil { 890 return nil, err 891 } 892 893 _, err = c.Delete(context.TODO(), azblob.ContainerAccessConditions{}) 894 if err != nil { 895 return nil, mapAZBError(err) 896 } 897 return &RemoveBucketOutput{}, nil 898 } 899 900 func (b *AZBlob) MakeBucket(param *MakeBucketInput) (*MakeBucketOutput, error) { 901 c, err := b.refreshToken() 902 if err != nil { 903 return nil, err 904 } 905 906 _, err = c.Create(context.TODO(), nil, azblob.PublicAccessNone) 907 if err != nil { 908 return nil, mapAZBError(err) 909 } 910 return &MakeBucketOutput{}, nil 911 }