github.com/philwinder/goofys@v0.24.0/internal/backend_adlv1.go (about) 1 // Copyright 2019 Databricks 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package internal 16 17 import ( 18 . "github.com/kahing/goofys/api/common" 19 20 "bytes" 21 "context" 22 "encoding/json" 23 "fmt" 24 "net/http" 25 "strconv" 26 "strings" 27 "syscall" 28 "time" 29 30 "github.com/jacobsa/fuse" 31 uuid "github.com/satori/go.uuid" 32 "github.com/sirupsen/logrus" 33 34 adl "github.com/Azure/azure-sdk-for-go/services/datalake/store/2016-11-01/filesystem" 35 "github.com/Azure/go-autorest/autorest" 36 ) 37 38 type ADLv1 struct { 39 cap Capabilities 40 41 flags *FlagStorage 42 config *ADLv1Config 43 44 client *adl.Client 45 account string 46 // ADLv1 doesn't actually have the concept of buckets (defined 47 // by me as a top level container that can be created with 48 // existing credentials). We could create new adl filesystems 49 // but that seems more involved. This bucket is more like a 50 // backend level prefix mostly to ease testing 51 bucket string 52 } 53 54 type ADLv1Err struct { 55 RemoteException struct { 56 Exception string 57 Message string 58 JavaClassName string 59 } 60 resp *http.Response 61 } 62 63 func (err ADLv1Err) Error() string { 64 return fmt.Sprintf("%v %v", err.resp.Status, err.RemoteException) 65 } 66 67 const ADL1_REQUEST_ID = "X-Ms-Request-Id" 68 69 var adls1Log = GetLogger("adlv1") 70 71 type ADLv1MultipartBlobCommitInput struct { 72 Size uint64 73 } 74 75 func IsADLv1Endpoint(endpoint string) bool { 76 return strings.HasPrefix(endpoint, "adl://") 77 //return strings.HasSuffix(endpoint, ".azuredatalakestore.net") 78 } 79 80 func adlLogResp(level logrus.Level, r *http.Response) { 81 if adls1Log.IsLevelEnabled(level) { 82 op := r.Request.URL.Query().Get("op") 83 requestId := r.Request.Header.Get(ADL1_REQUEST_ID) 84 respId := r.Header.Get(ADL1_REQUEST_ID) 85 adls1Log.Logf(level, "%v %v %v %v %v", op, r.Request.URL.String(), 86 requestId, r.Status, respId) 87 } 88 } 89 90 func NewADLv1(bucket string, flags *FlagStorage, config *ADLv1Config) (*ADLv1, error) { 91 parts := strings.SplitN(config.Endpoint, ".", 2) 92 if len(parts) != 2 { 93 return nil, fmt.Errorf("Invalid endpoint: %v", config.Endpoint) 94 } 95 96 LogRequest := func(p autorest.Preparer) autorest.Preparer { 97 return autorest.PreparerFunc(func(r *http.Request) (*http.Request, error) { 98 // the autogenerated permission bits are 99 // incorrect, it should be a string in base 8 100 // instead of base 10 101 q := r.URL.Query() 102 if perm := q.Get("permission"); perm != "" { 103 perm, err := strconv.ParseInt(perm, 10, 32) 104 if err == nil { 105 q.Set("permission", 106 fmt.Sprintf("0%o", perm)) 107 r.URL.RawQuery = q.Encode() 108 } 109 } 110 111 u, _ := uuid.NewV4() 112 r.Header.Add(ADL1_REQUEST_ID, u.String()) 113 114 if adls1Log.IsLevelEnabled(logrus.DebugLevel) { 115 op := r.URL.Query().Get("op") 116 requestId := r.Header.Get(ADL1_REQUEST_ID) 117 adls1Log.Debugf("%v %v %v", op, r.URL.String(), requestId) 118 } 119 120 r, err := p.Prepare(r) 121 if err != nil { 122 log.Error(err) 123 } 124 return r, err 125 }) 126 } 127 128 LogResponse := func(p autorest.Responder) autorest.Responder { 129 return autorest.ResponderFunc(func(r *http.Response) error { 130 adlLogResp(logrus.DebugLevel, r) 131 err := p.Respond(r) 132 if err != nil { 133 log.Error(err) 134 } 135 return err 136 }) 137 } 138 139 adlClient := adl.NewClient() 140 adlClient.BaseClient.Client.Authorizer = config.Authorizer 141 adlClient.BaseClient.Client.RequestInspector = LogRequest 142 adlClient.BaseClient.Client.ResponseInspector = LogResponse 143 adlClient.BaseClient.AdlsFileSystemDNSSuffix = parts[1] 144 adlClient.BaseClient.Sender.(*http.Client).Transport = GetHTTPTransport() 145 146 b := &ADLv1{ 147 flags: flags, 148 config: config, 149 client: &adlClient, 150 account: parts[0], 151 bucket: bucket, 152 cap: Capabilities{ 153 NoParallelMultipart: true, 154 DirBlob: true, 155 Name: "adl", 156 // ADLv1 fails with 404 if we upload data 157 // larger than 30000000 bytes (28.6MB) (28MB 158 // also failed in at one point, but as of 159 // 2019-11-07 seems to work) 160 MaxMultipartSize: 20 * 1024 * 1024, 161 }, 162 } 163 164 return b, nil 165 } 166 167 func (b *ADLv1) Bucket() string { 168 return b.bucket 169 } 170 171 func (b *ADLv1) Delegate() interface{} { 172 return b 173 } 174 175 func mapADLv1Error(resp *http.Response, err error, rawError bool) error { 176 // TODO(dotslash/khc): Figure out a way to surface these errors before reducing 177 // them to syscall.E<SOMETHING>. The detailed errors can aid in better debugging 178 // without the need to explicitly enabling debug_s3 flag and trying to reproduce 179 // issues. 180 if resp == nil { 181 if err != nil { 182 return syscall.EAGAIN 183 } else { 184 return err 185 } 186 } 187 188 if resp.StatusCode < 200 || resp.StatusCode >= 300 { 189 defer resp.Body.Close() 190 if rawError { 191 decoder := json.NewDecoder(resp.Body) 192 var adlErr ADLv1Err 193 194 var err error 195 if err = decoder.Decode(&adlErr); err == nil { 196 adlErr.resp = resp 197 return adlErr 198 } else { 199 adls1Log.Errorf("cannot parse error: %v", err) 200 return syscall.EAGAIN 201 } 202 } else { 203 err = mapHttpError(resp.StatusCode) 204 if err != nil { 205 return err 206 } else { 207 adlLogResp(logrus.ErrorLevel, resp) 208 return syscall.EINVAL 209 } 210 } 211 } 212 return nil 213 } 214 215 func (b *ADLv1) path(key string) string { 216 key = strings.TrimLeft(key, "/") 217 if b.bucket != "" { 218 if key != "" { 219 key = b.bucket + "/" + key 220 } else { 221 key = b.bucket 222 } 223 } 224 return key 225 } 226 227 func (b *ADLv1) Init(key string) error { 228 res, err := b.client.GetFileStatus(context.TODO(), b.account, b.path(key), nil) 229 err = mapADLv1Error(res.Response.Response, err, true) 230 if adlErr, ok := err.(ADLv1Err); ok { 231 if adlErr.RemoteException.Exception == "FileNotFoundException" { 232 return nil 233 } 234 } 235 return err 236 } 237 238 func (b *ADLv1) Capabilities() *Capabilities { 239 return &b.cap 240 } 241 242 func adlv1LastModified(t int64) time.Time { 243 return time.Unix(t/1000, t%1000000) 244 } 245 246 func adlv1FileStatus2BlobItem(f *adl.FileStatusProperties, key *string) BlobItemOutput { 247 return BlobItemOutput{ 248 Key: key, 249 LastModified: PTime(adlv1LastModified(*f.ModificationTime)), 250 Size: uint64(*f.Length), 251 } 252 } 253 254 func (b *ADLv1) HeadBlob(param *HeadBlobInput) (*HeadBlobOutput, error) { 255 res, err := b.client.GetFileStatus(context.TODO(), b.account, b.path(param.Key), nil) 256 err = mapADLv1Error(res.Response.Response, err, false) 257 if err != nil { 258 return nil, err 259 } 260 261 return &HeadBlobOutput{ 262 BlobItemOutput: adlv1FileStatus2BlobItem(res.FileStatus, ¶m.Key), 263 IsDirBlob: res.FileStatus.Type == "DIRECTORY", 264 }, nil 265 266 } 267 268 func (b *ADLv1) appendToListResults(path string, recursive bool, startAfter string, 269 maxKeys *uint32, prefixes []BlobPrefixOutput, items []BlobItemOutput) (adl.FileStatusesResult, []BlobPrefixOutput, []BlobItemOutput, error) { 270 271 res, err := b.client.ListFileStatus(context.TODO(), b.account, b.path(path), 272 nil, "", "", nil) 273 err = mapADLv1Error(res.Response.Response, err, false) 274 if err != nil { 275 return adl.FileStatusesResult{}, nil, nil, err 276 } 277 278 if path != "" { 279 if len(*res.FileStatuses.FileStatus) == 1 && 280 *(*res.FileStatuses.FileStatus)[0].PathSuffix == "" { 281 // path is actually a file 282 if !strings.HasSuffix(path, "/") { 283 items = append(items, 284 adlv1FileStatus2BlobItem(&(*res.FileStatuses.FileStatus)[0], &path)) 285 } 286 return res, prefixes, items, nil 287 } 288 289 if !recursive { 290 if strings.HasSuffix(path, "/") { 291 // we listed for the dir object itself 292 items = append(items, BlobItemOutput{ 293 Key: PString(path), 294 }) 295 } else { 296 prefixes = append(prefixes, BlobPrefixOutput{ 297 PString(path + "/"), 298 }) 299 } 300 } 301 } 302 303 path = strings.TrimRight(path, "/") 304 305 if maxKeys != nil { 306 *maxKeys -= uint32(len(*res.FileStatuses.FileStatus)) 307 } 308 309 for _, i := range *res.FileStatuses.FileStatus { 310 key := *i.PathSuffix 311 if path != "" { 312 key = path + "/" + key 313 } 314 315 if i.Type == "DIRECTORY" { 316 if recursive { 317 // we shouldn't generate prefixes if 318 // it's a recursive listing 319 items = append(items, 320 adlv1FileStatus2BlobItem(&i, PString(key+"/"))) 321 322 _, prefixes, items, err = b.appendToListResults(key, 323 recursive, "", maxKeys, prefixes, items) 324 } else { 325 prefixes = append(prefixes, BlobPrefixOutput{ 326 Prefix: PString(key + "/"), 327 }) 328 } 329 } else { 330 items = append(items, adlv1FileStatus2BlobItem(&i, &key)) 331 } 332 } 333 334 return res, prefixes, items, nil 335 } 336 337 func (b *ADLv1) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error) { 338 var recursive bool 339 if param.Delimiter == nil { 340 // used by tests to cleanup (and also slurping, but 341 // that's only enabled on S3 right now) 342 recursive = true 343 // cannot emulate these 344 if param.ContinuationToken != nil || param.StartAfter != nil { 345 return nil, syscall.ENOTSUP 346 } 347 } else if *param.Delimiter != "/" { 348 return nil, syscall.ENOTSUP 349 } 350 continuationToken := param.ContinuationToken 351 if param.StartAfter != nil { 352 continuationToken = param.StartAfter 353 } 354 355 _, prefixes, items, err := b.appendToListResults(nilStr(param.Prefix), 356 recursive, nilStr(continuationToken), param.MaxKeys, nil, nil) 357 if err == fuse.ENOENT { 358 err = nil 359 } else if err != nil { 360 return nil, err 361 } 362 363 return &ListBlobsOutput{ 364 Prefixes: prefixes, 365 Items: items, 366 IsTruncated: false, 367 }, nil 368 } 369 370 func (b *ADLv1) DeleteBlob(param *DeleteBlobInput) (*DeleteBlobOutput, error) { 371 res, err := b.client.Delete(context.TODO(), b.account, b.path(strings.TrimRight(param.Key, "/")), PBool(false)) 372 err = mapADLv1Error(res.Response.Response, err, false) 373 if err != nil { 374 return nil, err 375 } 376 if !*res.OperationResult { 377 return nil, fuse.ENOENT 378 } 379 return &DeleteBlobOutput{}, nil 380 } 381 382 func (b *ADLv1) DeleteBlobs(param *DeleteBlobsInput) (*DeleteBlobsOutput, error) { 383 return nil, syscall.ENOTSUP 384 } 385 386 func (b *ADLv1) RenameBlob(param *RenameBlobInput) (*RenameBlobOutput, error) { 387 r, err := b.client.RenamePreparer(context.TODO(), b.account, b.path(param.Source), 388 b.path(param.Destination)) 389 err = mapADLv1Error(nil, err, false) 390 if err != nil { 391 return nil, err 392 } 393 394 params := r.URL.Query() 395 params.Add("renameoptions", "OVERWRITE") 396 r.URL.RawQuery = params.Encode() 397 398 resp, err := b.client.RenameSender(r) 399 err = mapADLv1Error(resp, err, false) 400 if err != nil { 401 return nil, err 402 } 403 404 res, err := b.client.RenameResponder(resp) 405 err = mapADLv1Error(resp, err, false) 406 if err != nil { 407 return nil, err 408 } 409 410 if !*res.OperationResult { 411 // ADLv1 returns false if we try to rename a dir to a 412 // file, or if the rename source doesn't exist. We 413 // should have prevented renaming a dir to a file at 414 // upper layer so this is probably the former 415 416 // (the reverse, renaming a file to a directory works 417 // in ADLv1 and is the same as moving the file into 418 // the directory) 419 return nil, fuse.ENOENT 420 } 421 422 return &RenameBlobOutput{}, nil 423 } 424 425 func (b *ADLv1) CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error) { 426 return nil, syscall.ENOTSUP 427 } 428 429 func (b *ADLv1) GetBlob(param *GetBlobInput) (*GetBlobOutput, error) { 430 var length *int64 431 var offset *int64 432 433 if param.Count != 0 { 434 length = PInt64(int64(param.Count)) 435 } 436 if param.Start != 0 { 437 offset = PInt64(int64(param.Start)) 438 } 439 440 var filesessionid *uuid.UUID 441 if param.IfMatch != nil { 442 b := make([]byte, 16) 443 copy(b, []byte(*param.IfMatch)) 444 u, err := uuid.FromBytes(b) 445 if err != nil { 446 return nil, err 447 } 448 filesessionid = &u 449 } 450 451 resp, err := b.client.Open(context.TODO(), b.account, b.path(param.Key), length, offset, 452 filesessionid) 453 err = mapADLv1Error(resp.Response.Response, err, false) 454 if err != nil { 455 return nil, err 456 } 457 if resp.Value != nil { 458 defer func() { 459 if resp.Value != nil { 460 (*resp.Value).Close() 461 } 462 }() 463 } 464 // WebHDFS specifies that Content-Length is returned but ADLv1 465 // doesn't return it. Thankfully we never actually use it in 466 // the context of GetBlobOutput 467 468 var contentType *string 469 // not very useful since ADLv1 always return application/octet-stream 470 if val, ok := resp.Header["Content-Type"]; ok && len(val) != 0 { 471 contentType = &val[len(val)-1] 472 } 473 474 res := GetBlobOutput{ 475 HeadBlobOutput: HeadBlobOutput{ 476 BlobItemOutput: BlobItemOutput{ 477 Key: ¶m.Key, 478 }, 479 ContentType: contentType, 480 IsDirBlob: false, 481 }, 482 Body: *resp.Value, 483 } 484 resp.Value = nil 485 486 return &res, nil 487 } 488 489 func (b *ADLv1) PutBlob(param *PutBlobInput) (*PutBlobOutput, error) { 490 if param.DirBlob { 491 err := b.mkdir(param.Key) 492 if err != nil { 493 return nil, err 494 } 495 } else { 496 res, err := b.client.Create(context.TODO(), b.account, b.path(param.Key), 497 &ReadSeekerCloser{param.Body}, PBool(true), adl.CLOSE, nil, 498 PInt32(int32(b.flags.FileMode))) 499 err = mapADLv1Error(res.Response, err, false) 500 if err != nil { 501 return nil, err 502 } 503 } 504 505 return &PutBlobOutput{}, nil 506 } 507 508 func (b *ADLv1) MultipartBlobBegin(param *MultipartBlobBeginInput) (*MultipartBlobCommitInput, error) { 509 // ADLv1 doesn't have the concept of atomic replacement which 510 // means that when we replace an object, readers may see 511 // intermediate results. Here we implement MPU by first 512 // sending a CREATE with 0 bytes and acquire a lease at the 513 // same time. much of these is not documented anywhere except 514 // in the SDKs: 515 // https://github.com/Azure/azure-data-lake-store-java/blob/f5c270b8cb2ac68536b2cb123d355a874cade34c/src/main/java/com/microsoft/azure/datalake/store/Core.java#L84 516 leaseId, err := uuid.NewV4() 517 if err != nil { 518 return nil, err 519 } 520 521 res, err := b.client.Create(context.TODO(), b.account, b.path(param.Key), 522 &ReadSeekerCloser{bytes.NewReader([]byte(""))}, PBool(true), adl.DATA, &leaseId, 523 PInt32(int32(b.flags.FileMode))) 524 err = mapADLv1Error(res.Response, err, false) 525 if err != nil { 526 return nil, err 527 } 528 529 return &MultipartBlobCommitInput{ 530 Key: PString(b.path(param.Key)), 531 UploadId: PString(leaseId.String()), 532 backendData: &ADLv1MultipartBlobCommitInput{}, 533 }, nil 534 } 535 536 func (b *ADLv1) uploadPart(param *MultipartBlobAddInput, offset uint64) error { 537 leaseId, err := uuid.FromString(*param.Commit.UploadId) 538 if err != nil { 539 return err 540 } 541 542 res, err := b.client.Append(context.TODO(), b.account, *param.Commit.Key, 543 &ReadSeekerCloser{param.Body}, PInt64(int64(offset-param.Size)), adl.DATA, 544 &leaseId, &leaseId) 545 err = mapADLv1Error(res.Response, err, true) 546 if err != nil { 547 if adlErr, ok := err.(ADLv1Err); ok { 548 if adlErr.resp.StatusCode == 404 { 549 // ADLv1 APPEND returns 404 if either: 550 // the request payload is too large: 551 // https://social.msdn.microsoft.com/Forums/azure/en-US/48e86ce8-79f8-4412-838f-8e2a60b5f387/notfound-error-on-call-to-data-lake-store-create?forum=AzureDataLake 552 553 // or if a second concurrent stream is 554 // created. The behavior is odd: seems 555 // like the first stream will error 556 // but the latter stream works fine 557 err = fuse.EINVAL 558 return err 559 } else if adlErr.resp.StatusCode == 400 && 560 adlErr.RemoteException.Exception == "BadOffsetException" { 561 appendErr := b.detectTransientError(param, offset) 562 if appendErr == nil { 563 return nil 564 } 565 } 566 err = mapADLv1Error(adlErr.resp, err, false) 567 } 568 } 569 return err 570 } 571 572 func (b *ADLv1) detectTransientError(param *MultipartBlobAddInput, offset uint64) error { 573 leaseId, err := uuid.FromString(*param.Commit.UploadId) 574 if err != nil { 575 return err 576 } 577 res, err := b.client.Append(context.TODO(), b.account, *param.Commit.Key, 578 &ReadSeekerCloser{bytes.NewReader([]byte(""))}, 579 PInt64(int64(offset)), adl.CLOSE, &leaseId, &leaseId) 580 err = mapADLv1Error(res.Response, err, false) 581 return err 582 } 583 584 func (b *ADLv1) MultipartBlobAdd(param *MultipartBlobAddInput) (*MultipartBlobAddOutput, error) { 585 // APPEND with the expected offsets (so we can detect 586 // concurrent updates to the same file and fail, in case lease 587 // is for some reason broken on the server side 588 589 var commitData *ADLv1MultipartBlobCommitInput 590 var ok bool 591 if commitData, ok = param.Commit.backendData.(*ADLv1MultipartBlobCommitInput); !ok { 592 panic("Incorrect commit data type") 593 } 594 595 commitData.Size += param.Size 596 err := b.uploadPart(param, commitData.Size) 597 if err != nil { 598 return nil, err 599 } 600 601 return &MultipartBlobAddOutput{}, nil 602 } 603 604 func (b *ADLv1) MultipartBlobAbort(param *MultipartBlobCommitInput) (*MultipartBlobAbortOutput, error) { 605 // there's no such thing as abort, but at least we should release the lease 606 // which technically is more like a commit than abort 607 leaseId, err := uuid.FromString(*param.UploadId) 608 if err != nil { 609 return nil, err 610 } 611 res, err := b.client.Append(context.TODO(), b.account, *param.Key, 612 &ReadSeekerCloser{bytes.NewReader([]byte(""))}, nil, adl.CLOSE, &leaseId, &leaseId) 613 err = mapADLv1Error(res.Response, err, false) 614 if err != nil { 615 return nil, err 616 } 617 618 return &MultipartBlobAbortOutput{}, err 619 } 620 621 func (b *ADLv1) MultipartBlobCommit(param *MultipartBlobCommitInput) (*MultipartBlobCommitOutput, error) { 622 var commitData *ADLv1MultipartBlobCommitInput 623 var ok bool 624 if commitData, ok = param.backendData.(*ADLv1MultipartBlobCommitInput); !ok { 625 panic("Incorrect commit data type") 626 } 627 628 leaseId, err := uuid.FromString(*param.UploadId) 629 if err != nil { 630 return nil, err 631 } 632 res, err := b.client.Append(context.TODO(), b.account, *param.Key, 633 &ReadSeekerCloser{bytes.NewReader([]byte(""))}, PInt64(int64(commitData.Size)), 634 adl.CLOSE, &leaseId, &leaseId) 635 err = mapADLv1Error(res.Response, err, false) 636 if err == fuse.ENOENT { 637 // either the blob was concurrently deleted or we got 638 // another CREATE which broke our lease. Either way 639 // technically we did finish uploading data so swallow 640 // the error 641 err = nil 642 } 643 if err != nil { 644 return nil, err 645 } 646 647 return &MultipartBlobCommitOutput{}, nil 648 } 649 650 func (b *ADLv1) MultipartExpire(param *MultipartExpireInput) (*MultipartExpireOutput, error) { 651 return nil, syscall.ENOTSUP 652 } 653 654 func (b *ADLv1) RemoveBucket(param *RemoveBucketInput) (*RemoveBucketOutput, error) { 655 if b.bucket == "" { 656 return nil, fuse.EINVAL 657 } 658 659 res, err := b.client.Delete(context.TODO(), b.account, b.path(""), PBool(false)) 660 err = mapADLv1Error(res.Response.Response, err, false) 661 if err != nil { 662 return nil, err 663 } 664 if !*res.OperationResult { 665 return nil, fuse.ENOENT 666 } 667 668 return &RemoveBucketOutput{}, nil 669 } 670 671 func (b *ADLv1) MakeBucket(param *MakeBucketInput) (*MakeBucketOutput, error) { 672 if b.bucket == "" { 673 return nil, fuse.EINVAL 674 } 675 676 err := b.mkdir("") 677 if err != nil { 678 return nil, err 679 } 680 681 return &MakeBucketOutput{}, nil 682 } 683 684 func (b *ADLv1) mkdir(dir string) error { 685 res, err := b.client.Mkdirs(context.TODO(), b.account, b.path(dir), 686 PInt32(int32(b.flags.DirMode))) 687 err = mapADLv1Error(res.Response.Response, err, true) 688 if err != nil { 689 return err 690 } 691 if !*res.OperationResult { 692 return fuse.EEXIST 693 } 694 return nil 695 }