github.com/del680202/goofys@v0.19.1-0.20180727070818-6a609fafa266/internal/handles.go (about) 1 // Copyright 2015 - 2017 Ka-Hing Cheung 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package internal 16 17 import ( 18 "fmt" 19 "net/url" 20 "os" 21 "sort" 22 "strings" 23 "sync" 24 "syscall" 25 "time" 26 27 "github.com/aws/aws-sdk-go/aws" 28 "github.com/aws/aws-sdk-go/service/s3" 29 30 "github.com/jacobsa/fuse" 31 "github.com/jacobsa/fuse/fuseops" 32 "github.com/jacobsa/fuse/fuseutil" 33 34 "github.com/sirupsen/logrus" 35 ) 36 37 type InodeAttributes struct { 38 Size uint64 39 Mtime time.Time 40 } 41 42 type Inode struct { 43 Id fuseops.InodeID 44 Name *string 45 fs *Goofys 46 Attributes InodeAttributes 47 KnownSize *uint64 48 AttrTime time.Time 49 50 mu sync.Mutex // everything below is protected by mu 51 52 Parent *Inode 53 54 dir *DirInodeData 55 56 Invalid bool 57 ImplicitDir bool 58 59 fileHandles uint32 60 61 userMetadata map[string][]byte 62 s3Metadata map[string][]byte 63 64 // the refcnt is an exception, it's protected by the global lock 65 // Goofys.mu 66 refcnt uint64 67 } 68 69 func NewInode(fs *Goofys, parent *Inode, name *string, fullName *string) (inode *Inode) { 70 inode = &Inode{ 71 Name: name, 72 fs: fs, 73 AttrTime: time.Now(), 74 Parent: parent, 75 s3Metadata: make(map[string][]byte), 76 refcnt: 1, 77 } 78 79 return 80 } 81 82 func (inode *Inode) FullName() *string { 83 if inode.Parent == nil { 84 return inode.Name 85 } else { 86 s := inode.Parent.getChildName(*inode.Name) 87 return &s 88 } 89 } 90 91 func (inode *Inode) touch() { 92 inode.Attributes.Mtime = time.Now() 93 } 94 95 func (inode *Inode) InflateAttributes() (attr fuseops.InodeAttributes) { 96 mtime := inode.Attributes.Mtime 97 if mtime.IsZero() { 98 mtime = inode.fs.rootAttrs.Mtime 99 } 100 101 attr = fuseops.InodeAttributes{ 102 Size: inode.Attributes.Size, 103 Atime: mtime, 104 Mtime: mtime, 105 Ctime: mtime, 106 Crtime: mtime, 107 Uid: inode.fs.flags.Uid, 108 Gid: inode.fs.flags.Gid, 109 } 110 111 if inode.dir != nil { 112 attr.Nlink = 2 113 attr.Mode = inode.fs.flags.DirMode | os.ModeDir 114 } else { 115 attr.Nlink = 1 116 attr.Mode = inode.fs.flags.FileMode 117 } 118 return 119 } 120 121 func (inode *Inode) logFuse(op string, args ...interface{}) { 122 if fuseLog.Level >= logrus.DebugLevel { 123 fuseLog.Debugln(op, inode.Id, *inode.FullName(), args) 124 } 125 } 126 127 func (inode *Inode) errFuse(op string, args ...interface{}) { 128 fuseLog.Errorln(op, inode.Id, *inode.FullName(), args) 129 } 130 131 func (inode *Inode) ToDir() { 132 inode.Attributes = InodeAttributes{ 133 Size: 4096, 134 // Mtime intentionally not initialized 135 } 136 inode.dir = &DirInodeData{} 137 inode.KnownSize = &inode.fs.rootAttrs.Size 138 } 139 140 func (parent *Inode) findChild(name string) (inode *Inode) { 141 parent.mu.Lock() 142 defer parent.mu.Unlock() 143 144 inode = parent.findChildUnlockedFull(name) 145 return 146 } 147 148 func (parent *Inode) findInodeFunc(name string, isDir bool) func(i int) bool { 149 // sort dirs first, then by name 150 return func(i int) bool { 151 if parent.dir.Children[i].isDir() != isDir { 152 return isDir 153 } 154 return (*parent.dir.Children[i].Name) >= name 155 } 156 } 157 158 func (parent *Inode) findChildUnlockedFull(name string) (inode *Inode) { 159 inode = parent.findChildUnlocked(name, false) 160 if inode == nil { 161 inode = parent.findChildUnlocked(name, true) 162 } 163 return 164 } 165 166 func (parent *Inode) findChildUnlocked(name string, isDir bool) (inode *Inode) { 167 l := len(parent.dir.Children) 168 if l == 0 { 169 return 170 } 171 i := sort.Search(l, parent.findInodeFunc(name, isDir)) 172 if i < l { 173 // found 174 if *parent.dir.Children[i].Name == name { 175 inode = parent.dir.Children[i] 176 } 177 } 178 return 179 } 180 181 func (parent *Inode) findChildIdxUnlocked(name string) int { 182 l := len(parent.dir.Children) 183 if l == 0 { 184 return -1 185 } 186 i := sort.Search(l, parent.findInodeFunc(name, true)) 187 if i < l { 188 // found 189 if *parent.dir.Children[i].Name == name { 190 return i 191 } 192 } 193 return -1 194 } 195 196 func (parent *Inode) removeChildUnlocked(inode *Inode) { 197 l := len(parent.dir.Children) 198 if l == 0 { 199 return 200 } 201 i := sort.Search(l, parent.findInodeFunc(*inode.Name, inode.isDir())) 202 if i >= l || *parent.dir.Children[i].Name != *inode.Name { 203 panic(fmt.Sprintf("%v.removeName(%v) but child not found: %v", 204 *parent.FullName(), *inode.Name, i)) 205 } 206 207 copy(parent.dir.Children[i:], parent.dir.Children[i+1:]) 208 parent.dir.Children[l-1] = nil 209 parent.dir.Children = parent.dir.Children[:l-1] 210 211 if cap(parent.dir.Children)-len(parent.dir.Children) > 20 { 212 tmp := make([]*Inode, len(parent.dir.Children)) 213 copy(tmp, parent.dir.Children) 214 parent.dir.Children = tmp 215 } 216 } 217 218 func (parent *Inode) removeChild(inode *Inode) { 219 parent.mu.Lock() 220 defer parent.mu.Unlock() 221 222 parent.removeChildUnlocked(inode) 223 return 224 } 225 226 func (parent *Inode) insertChild(inode *Inode) { 227 parent.mu.Lock() 228 defer parent.mu.Unlock() 229 230 parent.insertChildUnlocked(inode) 231 } 232 233 func (parent *Inode) insertChildUnlocked(inode *Inode) { 234 l := len(parent.dir.Children) 235 if l == 0 { 236 parent.dir.Children = []*Inode{inode} 237 return 238 } 239 240 i := sort.Search(l, parent.findInodeFunc(*inode.Name, inode.isDir())) 241 if i == l { 242 // not found = new value is the biggest 243 parent.dir.Children = append(parent.dir.Children, inode) 244 } else { 245 if *parent.dir.Children[i].Name == *inode.Name { 246 panic(fmt.Sprintf("double insert of %v", parent.getChildName(*inode.Name))) 247 } 248 249 parent.dir.Children = append(parent.dir.Children, nil) 250 copy(parent.dir.Children[i+1:], parent.dir.Children[i:]) 251 parent.dir.Children[i] = inode 252 } 253 } 254 255 func (parent *Inode) LookUp(name string) (inode *Inode, err error) { 256 parent.logFuse("Inode.LookUp", name) 257 258 inode, err = parent.LookUpInodeMaybeDir(name, parent.getChildName(name)) 259 if err != nil { 260 return nil, err 261 } 262 263 return 264 } 265 266 func (parent *Inode) getChildName(name string) string { 267 if parent.Id == fuseops.RootInodeID { 268 return name 269 } else { 270 return fmt.Sprintf("%v/%v", *parent.FullName(), name) 271 } 272 } 273 274 // LOCKS_REQUIRED(fs.mu) 275 // XXX why did I put lock required? This used to return a resurrect bool 276 // which no long does anything, need to look into that to see if 277 // that was legacy 278 func (inode *Inode) Ref() { 279 inode.logFuse("Ref", inode.refcnt) 280 281 inode.refcnt++ 282 return 283 } 284 285 // LOCKS_REQUIRED(fs.mu) 286 func (inode *Inode) DeRef(n uint64) (stale bool) { 287 inode.logFuse("DeRef", n, inode.refcnt) 288 289 if inode.refcnt < n { 290 panic(fmt.Sprintf("deref %v from %v", n, inode.refcnt)) 291 } 292 293 inode.refcnt -= n 294 295 stale = (inode.refcnt == 0) 296 return 297 } 298 299 func (parent *Inode) Unlink(name string) (err error) { 300 parent.logFuse("Unlink", name) 301 302 fullName := parent.getChildName(name) 303 304 params := &s3.DeleteObjectInput{ 305 Bucket: &parent.fs.bucket, 306 Key: parent.fs.key(fullName), 307 } 308 309 resp, err := parent.fs.s3.DeleteObject(params) 310 if err != nil { 311 return mapAwsError(err) 312 } 313 314 s3Log.Debug(resp) 315 316 parent.mu.Lock() 317 defer parent.mu.Unlock() 318 319 inode := parent.findChildUnlocked(name, false) 320 if inode != nil { 321 parent.removeChildUnlocked(inode) 322 inode.Parent = nil 323 } 324 325 return 326 } 327 328 func (parent *Inode) Create( 329 name string) (inode *Inode, fh *FileHandle) { 330 331 parent.logFuse("Create", name) 332 fullName := parent.getChildName(name) 333 fs := parent.fs 334 335 parent.mu.Lock() 336 defer parent.mu.Unlock() 337 338 now := time.Now() 339 inode = NewInode(fs, parent, &name, &fullName) 340 inode.Attributes = InodeAttributes{ 341 Size: 0, 342 Mtime: now, 343 } 344 345 fh = NewFileHandle(inode) 346 fh.poolHandle = fs.bufferPool 347 fh.dirty = true 348 inode.fileHandles = 1 349 350 parent.touch() 351 352 return 353 } 354 355 func (parent *Inode) MkDir( 356 name string) (inode *Inode, err error) { 357 358 parent.logFuse("MkDir", name) 359 360 fullName := parent.getChildName(name) 361 fs := parent.fs 362 363 params := &s3.PutObjectInput{ 364 Bucket: &fs.bucket, 365 Key: fs.key(fullName + "/"), 366 Body: nil, 367 } 368 369 if fs.flags.UseSSE { 370 params.ServerSideEncryption = &fs.sseType 371 if fs.flags.UseKMS && fs.flags.KMSKeyID != "" { 372 params.SSEKMSKeyId = &fs.flags.KMSKeyID 373 } 374 } 375 376 _, err = fs.s3.PutObject(params) 377 if err != nil { 378 err = mapAwsError(err) 379 return 380 } 381 382 parent.mu.Lock() 383 defer parent.mu.Unlock() 384 385 inode = NewInode(fs, parent, &name, &fullName) 386 inode.ToDir() 387 inode.touch() 388 if parent.Attributes.Mtime.Before(inode.Attributes.Mtime) { 389 parent.Attributes.Mtime = inode.Attributes.Mtime 390 } 391 392 return 393 } 394 395 func isEmptyDir(fs *Goofys, fullName string) (isDir bool, err error) { 396 fullName += "/" 397 398 params := &s3.ListObjectsInput{ 399 Bucket: &fs.bucket, 400 Delimiter: aws.String("/"), 401 MaxKeys: aws.Int64(2), 402 Prefix: fs.key(fullName), 403 } 404 405 resp, err := fs.s3.ListObjects(params) 406 if err != nil { 407 return false, mapAwsError(err) 408 } 409 410 if len(resp.CommonPrefixes) > 0 || len(resp.Contents) > 1 { 411 err = fuse.ENOTEMPTY 412 isDir = true 413 return 414 } 415 416 if len(resp.Contents) == 1 { 417 isDir = true 418 419 if *resp.Contents[0].Key != *fs.key(fullName) { 420 err = fuse.ENOTEMPTY 421 } 422 } 423 424 return 425 } 426 427 func (parent *Inode) RmDir(name string) (err error) { 428 parent.logFuse("Rmdir", name) 429 430 fullName := parent.getChildName(name) 431 fs := parent.fs 432 433 isDir, err := isEmptyDir(fs, fullName) 434 if err != nil { 435 return 436 } 437 if !isDir { 438 return fuse.ENOENT 439 } 440 441 fullName += "/" 442 443 params := &s3.DeleteObjectInput{ 444 Bucket: &fs.bucket, 445 Key: fs.key(fullName), 446 } 447 448 _, err = fs.s3.DeleteObject(params) 449 if err != nil { 450 return mapAwsError(err) 451 } 452 453 // we know this entry is gone 454 parent.mu.Lock() 455 defer parent.mu.Unlock() 456 457 inode := parent.findChildUnlocked(name, true) 458 if inode != nil { 459 parent.removeChildUnlocked(inode) 460 inode.Parent = nil 461 } 462 463 return 464 } 465 466 func (inode *Inode) GetAttributes() (*fuseops.InodeAttributes, error) { 467 // XXX refresh attributes 468 inode.logFuse("GetAttributes") 469 if inode.Invalid { 470 return nil, fuse.ENOENT 471 } 472 attr := inode.InflateAttributes() 473 return &attr, nil 474 } 475 476 func (inode *Inode) isDir() bool { 477 return inode.dir != nil 478 } 479 480 // LOCKS_REQUIRED(inode.mu) 481 func (inode *Inode) fillXattrFromHead(resp *s3.HeadObjectOutput) { 482 inode.userMetadata = make(map[string][]byte) 483 484 if resp.ETag != nil { 485 inode.s3Metadata["etag"] = []byte(*resp.ETag) 486 } 487 if resp.StorageClass != nil { 488 inode.s3Metadata["storage-class"] = []byte(*resp.StorageClass) 489 } else { 490 inode.s3Metadata["storage-class"] = []byte("STANDARD") 491 } 492 493 for k, v := range resp.Metadata { 494 k = strings.ToLower(k) 495 value, err := url.PathUnescape(*v) 496 if err != nil { 497 value = *v 498 } 499 inode.userMetadata[k] = []byte(value) 500 } 501 } 502 503 // LOCKS_REQUIRED(inode.mu) 504 func (inode *Inode) fillXattr() (err error) { 505 if !inode.ImplicitDir && inode.userMetadata == nil { 506 507 fullName := *inode.FullName() 508 if inode.isDir() { 509 fullName += "/" 510 } 511 fs := inode.fs 512 513 params := &s3.HeadObjectInput{Bucket: &fs.bucket, Key: fs.key(fullName)} 514 resp, err := fs.s3.HeadObject(params) 515 if err != nil { 516 err = mapAwsError(err) 517 if err == fuse.ENOENT { 518 err = nil 519 if inode.isDir() { 520 inode.ImplicitDir = true 521 } 522 } 523 return err 524 } else { 525 inode.fillXattrFromHead(resp) 526 } 527 } 528 529 return 530 } 531 532 // LOCKS_REQUIRED(inode.mu) 533 func (inode *Inode) getXattrMap(name string, userOnly bool) ( 534 meta map[string][]byte, newName string, err error) { 535 536 if strings.HasPrefix(name, "s3.") { 537 if userOnly { 538 return nil, "", syscall.EACCES 539 } 540 541 newName = name[3:] 542 meta = inode.s3Metadata 543 } else if strings.HasPrefix(name, "user.") { 544 err = inode.fillXattr() 545 if err != nil { 546 return nil, "", err 547 } 548 549 newName = name[5:] 550 meta = inode.userMetadata 551 } else { 552 if userOnly { 553 return nil, "", syscall.EACCES 554 } else { 555 return nil, "", syscall.ENODATA 556 } 557 } 558 559 if meta == nil { 560 return nil, "", syscall.ENODATA 561 } 562 563 return 564 } 565 566 func convertMetadata(meta map[string][]byte) (metadata map[string]*string) { 567 metadata = make(map[string]*string) 568 for k, v := range meta { 569 metadata[k] = aws.String(xattrEscape(v)) 570 } 571 return 572 } 573 574 // LOCKS_REQUIRED(inode.mu) 575 func (inode *Inode) updateXattr() (err error) { 576 err = copyObjectMaybeMultipart(inode.fs, int64(inode.Attributes.Size), 577 *inode.FullName(), *inode.FullName(), 578 aws.String(string(inode.s3Metadata["etag"])), convertMetadata(inode.userMetadata)) 579 return 580 } 581 582 func (inode *Inode) SetXattr(name string, value []byte, flags uint32) error { 583 inode.logFuse("RemoveXattr", name) 584 585 inode.mu.Lock() 586 defer inode.mu.Unlock() 587 588 meta, name, err := inode.getXattrMap(name, true) 589 if err != nil { 590 return err 591 } 592 593 if flags != 0x0 { 594 _, ok := meta[name] 595 if flags == 0x1 { 596 if ok { 597 return syscall.EEXIST 598 } 599 } else if flags == 0x2 { 600 if !ok { 601 return syscall.ENODATA 602 } 603 } 604 } 605 606 meta[name] = Dup(value) 607 err = inode.updateXattr() 608 return err 609 } 610 611 func (inode *Inode) RemoveXattr(name string) error { 612 inode.logFuse("RemoveXattr", name) 613 614 inode.mu.Lock() 615 defer inode.mu.Unlock() 616 617 meta, name, err := inode.getXattrMap(name, true) 618 if err != nil { 619 return err 620 } 621 622 if _, ok := meta[name]; ok { 623 delete(meta, name) 624 err = inode.updateXattr() 625 return err 626 } else { 627 return syscall.ENODATA 628 } 629 } 630 631 func (inode *Inode) GetXattr(name string) ([]byte, error) { 632 inode.logFuse("GetXattr", name) 633 634 inode.mu.Lock() 635 defer inode.mu.Unlock() 636 637 meta, name, err := inode.getXattrMap(name, false) 638 if err != nil { 639 return nil, err 640 } 641 642 value, ok := meta[name] 643 if ok { 644 return []byte(value), nil 645 } else { 646 return nil, syscall.ENODATA 647 } 648 } 649 650 func (inode *Inode) ListXattr() ([]string, error) { 651 inode.logFuse("ListXattr") 652 653 inode.mu.Lock() 654 defer inode.mu.Unlock() 655 656 var xattrs []string 657 658 err := inode.fillXattr() 659 if err != nil { 660 return nil, err 661 } 662 663 for k, _ := range inode.s3Metadata { 664 xattrs = append(xattrs, "s3."+k) 665 } 666 667 for k, _ := range inode.userMetadata { 668 xattrs = append(xattrs, "user."+k) 669 } 670 671 return xattrs, nil 672 } 673 674 func (inode *Inode) OpenFile() (fh *FileHandle, err error) { 675 inode.logFuse("OpenFile") 676 677 inode.mu.Lock() 678 defer inode.mu.Unlock() 679 680 fh = NewFileHandle(inode) 681 inode.fileHandles += 1 682 return 683 } 684 685 func (parent *Inode) Rename(from string, newParent *Inode, to string) (err error) { 686 parent.logFuse("Rename", from, newParent.getChildName(to)) 687 688 fromFullName := parent.getChildName(from) 689 fs := parent.fs 690 691 var size int64 692 var fromIsDir bool 693 var toIsDir bool 694 695 fromIsDir, err = isEmptyDir(fs, fromFullName) 696 if err != nil { 697 // we don't support renaming a directory that's not empty 698 return 699 } 700 701 toFullName := newParent.getChildName(to) 702 703 toIsDir, err = isEmptyDir(fs, toFullName) 704 if err != nil { 705 return 706 } 707 708 if fromIsDir && !toIsDir { 709 _, err = fs.s3.HeadObject(&s3.HeadObjectInput{ 710 Bucket: &fs.bucket, 711 Key: fs.key(toFullName), 712 }) 713 if err == nil { 714 return fuse.ENOTDIR 715 } else { 716 err = mapAwsError(err) 717 if err != fuse.ENOENT { 718 return 719 } 720 } 721 } else if !fromIsDir && toIsDir { 722 return syscall.EISDIR 723 } 724 725 size = int64(-1) 726 if fromIsDir { 727 fromFullName += "/" 728 toFullName += "/" 729 size = 0 730 } 731 732 err = renameObject(fs, size, fromFullName, toFullName) 733 return 734 } 735 736 func mpuCopyPart(fs *Goofys, from string, to string, mpuId string, bytes string, part int64, 737 wg *sync.WaitGroup, srcEtag *string, etag **string, errout *error) { 738 739 defer func() { 740 wg.Done() 741 }() 742 743 // XXX use CopySourceIfUnmodifiedSince to ensure that 744 // we are copying from the same object 745 params := &s3.UploadPartCopyInput{ 746 Bucket: &fs.bucket, 747 Key: fs.key(to), 748 CopySource: aws.String(pathEscape(from)), 749 UploadId: &mpuId, 750 CopySourceRange: &bytes, 751 CopySourceIfMatch: srcEtag, 752 PartNumber: &part, 753 } 754 755 s3Log.Debug(params) 756 757 resp, err := fs.s3.UploadPartCopy(params) 758 if err != nil { 759 s3Log.Errorf("UploadPartCopy %v = %v", params, err) 760 *errout = mapAwsError(err) 761 return 762 } 763 764 *etag = resp.CopyPartResult.ETag 765 return 766 } 767 768 func sizeToParts(size int64) int { 769 const PART_SIZE = 5 * 1024 * 1024 * 1024 770 771 nParts := int(size / PART_SIZE) 772 if size%PART_SIZE != 0 { 773 nParts++ 774 } 775 return nParts 776 } 777 778 func mpuCopyParts(fs *Goofys, size int64, from string, to string, mpuId string, 779 wg *sync.WaitGroup, srcEtag *string, etags []*string, err *error) { 780 781 const PART_SIZE = 5 * 1024 * 1024 * 1024 782 783 rangeFrom := int64(0) 784 rangeTo := int64(0) 785 786 for i := int64(1); rangeTo < size; i++ { 787 rangeFrom = rangeTo 788 rangeTo = i * PART_SIZE 789 if rangeTo > size { 790 rangeTo = size 791 } 792 bytes := fmt.Sprintf("bytes=%v-%v", rangeFrom, rangeTo-1) 793 794 wg.Add(1) 795 go mpuCopyPart(fs, from, to, mpuId, bytes, i, wg, srcEtag, &etags[i-1], err) 796 } 797 } 798 799 func copyObjectMultipart(fs *Goofys, size int64, from string, to string, mpuId string, 800 srcEtag *string, metadata map[string]*string) (err error) { 801 var wg sync.WaitGroup 802 nParts := sizeToParts(size) 803 etags := make([]*string, nParts) 804 805 if mpuId == "" { 806 params := &s3.CreateMultipartUploadInput{ 807 Bucket: &fs.bucket, 808 Key: fs.key(to), 809 StorageClass: &fs.flags.StorageClass, 810 ContentType: fs.getMimeType(to), 811 Metadata: metadata, 812 } 813 814 if fs.flags.UseSSE { 815 params.ServerSideEncryption = &fs.sseType 816 if fs.flags.UseKMS && fs.flags.KMSKeyID != "" { 817 params.SSEKMSKeyId = &fs.flags.KMSKeyID 818 } 819 } 820 821 if fs.flags.ACL != "" { 822 params.ACL = &fs.flags.ACL 823 } 824 825 resp, err := fs.s3.CreateMultipartUpload(params) 826 if err != nil { 827 return mapAwsError(err) 828 } 829 830 mpuId = *resp.UploadId 831 } 832 833 mpuCopyParts(fs, size, from, to, mpuId, &wg, srcEtag, etags, &err) 834 wg.Wait() 835 836 if err != nil { 837 return 838 } else { 839 parts := make([]*s3.CompletedPart, nParts) 840 for i := 0; i < nParts; i++ { 841 parts[i] = &s3.CompletedPart{ 842 ETag: etags[i], 843 PartNumber: aws.Int64(int64(i + 1)), 844 } 845 } 846 847 params := &s3.CompleteMultipartUploadInput{ 848 Bucket: &fs.bucket, 849 Key: fs.key(to), 850 UploadId: &mpuId, 851 MultipartUpload: &s3.CompletedMultipartUpload{ 852 Parts: parts, 853 }, 854 } 855 856 s3Log.Debug(params) 857 858 _, err := fs.s3.CompleteMultipartUpload(params) 859 if err != nil { 860 s3Log.Errorf("Complete MPU %v = %v", params, err) 861 return mapAwsError(err) 862 } 863 } 864 865 return 866 } 867 868 func copyObjectMaybeMultipart(fs *Goofys, size int64, from string, to string, srcEtag *string, metadata map[string]*string) (err error) { 869 870 if size == -1 || srcEtag == nil || metadata == nil { 871 params := &s3.HeadObjectInput{Bucket: &fs.bucket, Key: fs.key(from)} 872 resp, err := fs.s3.HeadObject(params) 873 if err != nil { 874 return mapAwsError(err) 875 } 876 877 size = *resp.ContentLength 878 metadata = resp.Metadata 879 srcEtag = resp.ETag 880 } 881 882 from = fs.bucket + "/" + *fs.key(from) 883 884 if !fs.gcs && size > 5*1024*1024*1024 { 885 return copyObjectMultipart(fs, size, from, to, "", srcEtag, metadata) 886 } 887 888 storageClass := fs.flags.StorageClass 889 if size < 128*1024 && storageClass == "STANDARD_IA" { 890 storageClass = "STANDARD" 891 } 892 893 params := &s3.CopyObjectInput{ 894 Bucket: &fs.bucket, 895 CopySource: aws.String(pathEscape(from)), 896 Key: fs.key(to), 897 StorageClass: &storageClass, 898 ContentType: fs.getMimeType(to), 899 Metadata: metadata, 900 MetadataDirective: aws.String(s3.MetadataDirectiveReplace), 901 } 902 903 s3Log.Debug(params) 904 905 if fs.flags.UseSSE { 906 params.ServerSideEncryption = &fs.sseType 907 if fs.flags.UseKMS && fs.flags.KMSKeyID != "" { 908 params.SSEKMSKeyId = &fs.flags.KMSKeyID 909 } 910 } 911 912 if fs.flags.ACL != "" { 913 params.ACL = &fs.flags.ACL 914 } 915 916 resp, err := fs.s3.CopyObject(params) 917 if err != nil { 918 s3Log.Errorf("CopyObject %v = %v", params, err) 919 err = mapAwsError(err) 920 } 921 s3Log.Debug(resp) 922 923 return 924 } 925 926 func renameObject(fs *Goofys, size int64, fromFullName string, toFullName string) (err error) { 927 err = copyObjectMaybeMultipart(fs, size, fromFullName, toFullName, nil, nil) 928 if err != nil { 929 return err 930 } 931 932 delParams := &s3.DeleteObjectInput{ 933 Bucket: &fs.bucket, 934 Key: fs.key(fromFullName), 935 } 936 937 _, err = fs.s3.DeleteObject(delParams) 938 if err != nil { 939 return mapAwsError(err) 940 } 941 s3Log.Debugf("Deleted %v", delParams) 942 943 return 944 } 945 946 func (parent *Inode) addDotAndDotDot() { 947 fs := parent.fs 948 en := &DirHandleEntry{ 949 Name: aws.String("."), 950 Type: fuseutil.DT_Directory, 951 Attributes: &parent.Attributes, 952 Offset: 1, 953 } 954 fs.insertInodeFromDirEntry(parent, en) 955 dotDotAttr := &parent.Attributes 956 if parent.Parent != nil { 957 dotDotAttr = &parent.Parent.Attributes 958 } 959 en = &DirHandleEntry{ 960 Name: aws.String(".."), 961 Type: fuseutil.DT_Directory, 962 Attributes: dotDotAttr, 963 Offset: 2, 964 } 965 fs.insertInodeFromDirEntry(parent, en) 966 } 967 968 // if I had seen a/ and a/b, and now I get a/c, that means a/b is 969 // done, but not a/ 970 func (parent *Inode) isParentOf(inode *Inode) bool { 971 return inode.Parent != nil && (parent == inode.Parent || parent.isParentOf(inode.Parent)) 972 } 973 974 func sealPastDirs(dirs map[*Inode]bool, d *Inode) { 975 for p, sealed := range dirs { 976 if p != d && !sealed && !p.isParentOf(d) { 977 dirs[p] = true 978 } 979 } 980 // I just read something in d, obviously it's not done yet 981 dirs[d] = false 982 } 983 984 func (parent *Inode) insertSubTree(path string, obj *s3.Object, dirs map[*Inode]bool) { 985 fs := parent.fs 986 slash := strings.Index(path, "/") 987 if slash == -1 { 988 fs.insertInodeFromDirEntry(parent, objectToDirEntry(fs, obj, path, false)) 989 sealPastDirs(dirs, parent) 990 } else { 991 dir := path[:slash] 992 path = path[slash+1:] 993 994 if len(path) == 0 { 995 inode := fs.insertInodeFromDirEntry(parent, objectToDirEntry(fs, obj, dir, true)) 996 inode.addDotAndDotDot() 997 998 sealPastDirs(dirs, inode) 999 } else { 1000 // ensure that the potentially implicit dir is added 1001 en := &DirHandleEntry{ 1002 Name: &dir, 1003 Type: fuseutil.DT_Directory, 1004 Attributes: &fs.rootAttrs, 1005 Offset: 1, 1006 } 1007 inode := fs.insertInodeFromDirEntry(parent, en) 1008 // mark this dir but don't seal anything else 1009 // until we get to the leaf 1010 dirs[inode] = false 1011 1012 inode.addDotAndDotDot() 1013 inode.insertSubTree(path, obj, dirs) 1014 } 1015 } 1016 } 1017 1018 func (parent *Inode) findChildMaxTime() time.Time { 1019 maxTime := parent.Attributes.Mtime 1020 1021 for i, c := range parent.dir.Children { 1022 if i < 2 { 1023 // skip . and .. 1024 continue 1025 } 1026 if c.Attributes.Mtime.After(maxTime) { 1027 maxTime = c.Attributes.Mtime 1028 } 1029 } 1030 1031 return maxTime 1032 } 1033 1034 func (parent *Inode) readDirFromCache(offset fuseops.DirOffset) (en *DirHandleEntry, ok bool) { 1035 parent.mu.Lock() 1036 defer parent.mu.Unlock() 1037 1038 if !expired(parent.dir.DirTime, parent.fs.flags.TypeCacheTTL) { 1039 ok = true 1040 1041 if int(offset) >= len(parent.dir.Children) { 1042 return 1043 } 1044 child := parent.dir.Children[offset] 1045 1046 en = &DirHandleEntry{ 1047 Name: child.Name, 1048 Inode: child.Id, 1049 Offset: offset + 1, 1050 Attributes: &child.Attributes, 1051 } 1052 if child.isDir() { 1053 en.Type = fuseutil.DT_Directory 1054 } else { 1055 en.Type = fuseutil.DT_File 1056 } 1057 1058 } 1059 return 1060 } 1061 1062 func (parent *Inode) LookUpInodeNotDir(name string, c chan s3.HeadObjectOutput, errc chan error) { 1063 params := &s3.HeadObjectInput{Bucket: &parent.fs.bucket, Key: parent.fs.key(name)} 1064 resp, err := parent.fs.s3.HeadObject(params) 1065 if err != nil { 1066 errc <- mapAwsError(err) 1067 return 1068 } 1069 1070 s3Log.Debug(resp) 1071 c <- *resp 1072 } 1073 1074 func (parent *Inode) LookUpInodeDir(name string, c chan s3.ListObjectsOutput, errc chan error) { 1075 params := &s3.ListObjectsInput{ 1076 Bucket: &parent.fs.bucket, 1077 Delimiter: aws.String("/"), 1078 MaxKeys: aws.Int64(1), 1079 Prefix: parent.fs.key(name + "/"), 1080 } 1081 1082 resp, err := parent.fs.s3.ListObjects(params) 1083 if err != nil { 1084 errc <- mapAwsError(err) 1085 return 1086 } 1087 1088 s3Log.Debug(resp) 1089 c <- *resp 1090 } 1091 1092 // returned inode has nil Id 1093 func (parent *Inode) LookUpInodeMaybeDir(name string, fullName string) (inode *Inode, err error) { 1094 errObjectChan := make(chan error, 1) 1095 objectChan := make(chan s3.HeadObjectOutput, 1) 1096 errDirBlobChan := make(chan error, 1) 1097 dirBlobChan := make(chan s3.HeadObjectOutput, 1) 1098 var errDirChan chan error 1099 var dirChan chan s3.ListObjectsOutput 1100 1101 checking := 3 1102 var checkErr [3]error 1103 1104 if parent.fs.s3 == nil { 1105 panic("s3 disabled") 1106 } 1107 1108 go parent.LookUpInodeNotDir(fullName, objectChan, errObjectChan) 1109 if !parent.fs.flags.Cheap { 1110 go parent.LookUpInodeNotDir(fullName+"/", dirBlobChan, errDirBlobChan) 1111 if !parent.fs.flags.ExplicitDir { 1112 errDirChan = make(chan error, 1) 1113 dirChan = make(chan s3.ListObjectsOutput, 1) 1114 go parent.LookUpInodeDir(fullName, dirChan, errDirChan) 1115 } 1116 } 1117 1118 for { 1119 select { 1120 case resp := <-objectChan: 1121 err = nil 1122 // XXX/TODO if both object and object/ exists, return dir 1123 inode = NewInode(parent.fs, parent, &name, &fullName) 1124 inode.Attributes = InodeAttributes{ 1125 Size: uint64(aws.Int64Value(resp.ContentLength)), 1126 Mtime: *resp.LastModified, 1127 } 1128 1129 // don't want to point to the attribute because that 1130 // can get updated 1131 size := inode.Attributes.Size 1132 inode.KnownSize = &size 1133 1134 inode.fillXattrFromHead(&resp) 1135 return 1136 case err = <-errObjectChan: 1137 checking-- 1138 checkErr[0] = err 1139 s3Log.Debugf("HEAD %v = %v", fullName, err) 1140 case resp := <-dirChan: 1141 err = nil 1142 if len(resp.CommonPrefixes) != 0 || len(resp.Contents) != 0 { 1143 inode = NewInode(parent.fs, parent, &name, &fullName) 1144 inode.ToDir() 1145 if len(resp.Contents) != 0 && *resp.Contents[0].Key == name+"/" { 1146 // it's actually a dir blob 1147 entry := resp.Contents[0] 1148 if entry.ETag != nil { 1149 inode.s3Metadata["etag"] = []byte(*entry.ETag) 1150 } 1151 if entry.StorageClass != nil { 1152 inode.s3Metadata["storage-class"] = []byte(*entry.StorageClass) 1153 } 1154 1155 } 1156 // if cheap is not on, the dir blob 1157 // could exist but this returned first 1158 if inode.fs.flags.Cheap { 1159 inode.ImplicitDir = true 1160 } 1161 return 1162 } else { 1163 checkErr[2] = fuse.ENOENT 1164 checking-- 1165 } 1166 case err = <-errDirChan: 1167 checking-- 1168 checkErr[2] = err 1169 s3Log.Debugf("LIST %v/ = %v", fullName, err) 1170 case resp := <-dirBlobChan: 1171 err = nil 1172 inode = NewInode(parent.fs, parent, &name, &fullName) 1173 inode.ToDir() 1174 inode.Attributes.Mtime = *resp.LastModified 1175 inode.fillXattrFromHead(&resp) 1176 return 1177 case err = <-errDirBlobChan: 1178 checking-- 1179 checkErr[1] = err 1180 s3Log.Debugf("HEAD %v/ = %v", fullName, err) 1181 } 1182 1183 switch checking { 1184 case 2: 1185 if parent.fs.flags.Cheap { 1186 go parent.LookUpInodeNotDir(fullName+"/", dirBlobChan, errDirBlobChan) 1187 } 1188 case 1: 1189 if parent.fs.flags.ExplicitDir { 1190 checkErr[2] = fuse.ENOENT 1191 goto doneCase 1192 } else if parent.fs.flags.Cheap { 1193 errDirChan = make(chan error, 1) 1194 dirChan = make(chan s3.ListObjectsOutput, 1) 1195 go parent.LookUpInodeDir(fullName, dirChan, errDirChan) 1196 } 1197 break 1198 doneCase: 1199 fallthrough 1200 case 0: 1201 for _, e := range checkErr { 1202 if e != fuse.ENOENT { 1203 err = e 1204 return 1205 } 1206 } 1207 1208 err = fuse.ENOENT 1209 return 1210 } 1211 } 1212 }