storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/fs-v1.go (about) 1 /* 2 * MinIO Cloud Storage, (C) 2016, 2017, 2018 MinIO, Inc. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package cmd 18 19 import ( 20 "bytes" 21 "context" 22 "fmt" 23 "io" 24 "io/ioutil" 25 "net/http" 26 "os" 27 "os/user" 28 "path" 29 "sort" 30 "strings" 31 "sync" 32 "sync/atomic" 33 "time" 34 35 jsoniter "github.com/json-iterator/go" 36 "github.com/minio/minio-go/v7/pkg/s3utils" 37 "github.com/minio/minio-go/v7/pkg/tags" 38 39 "storj.io/minio/cmd/config" 40 xhttp "storj.io/minio/cmd/http" 41 "storj.io/minio/cmd/logger" 42 "storj.io/minio/pkg/bucket/policy" 43 "storj.io/minio/pkg/color" 44 xioutil "storj.io/minio/pkg/ioutil" 45 "storj.io/minio/pkg/lock" 46 "storj.io/minio/pkg/madmin" 47 "storj.io/minio/pkg/mimedb" 48 "storj.io/minio/pkg/mountinfo" 49 ) 50 51 // Default etag is used for pre-existing objects. 52 var defaultEtag = "00000000000000000000000000000000-1" 53 54 // FSObjects - Implements fs object layer. 55 type FSObjects struct { 56 GatewayUnsupported 57 58 // The count of concurrent calls on FSObjects API 59 activeIOCount int64 60 61 // Path to be exported over S3 API. 62 fsPath string 63 // meta json filename, varies by fs / cache backend. 64 metaJSONFile string 65 // Unique value to be used for all 66 // temporary transactions. 67 fsUUID string 68 69 // This value shouldn't be touched, once initialized. 70 fsFormatRlk *lock.RLockedFile // Is a read lock on `format.json`. 71 72 // FS rw pool. 73 rwPool *fsIOPool 74 75 // ListObjects pool management. 76 listPool *TreeWalkPool 77 78 diskMount bool 79 80 appendFileMap map[string]*fsAppendFile 81 appendFileMapMu sync.Mutex 82 83 // To manage the appendRoutine go-routines 84 nsMutex *nsLockMap 85 } 86 87 // Represents the background append file. 88 type fsAppendFile struct { 89 sync.Mutex 90 parts []PartInfo // List of parts appended. 91 filePath string // Absolute path of the file in the temp location. 92 } 93 94 // Initializes meta volume on all the fs path. 95 func initMetaVolumeFS(fsPath, fsUUID string) error { 96 // This happens for the first time, but keep this here since this 97 // is the only place where it can be made less expensive 98 // optimizing all other calls. Create minio meta volume, 99 // if it doesn't exist yet. 100 metaBucketPath := pathJoin(fsPath, minioMetaBucket) 101 102 if err := os.MkdirAll(metaBucketPath, 0777); err != nil { 103 return err 104 } 105 106 metaTmpPath := pathJoin(fsPath, minioMetaTmpBucket, fsUUID) 107 if err := os.MkdirAll(metaTmpPath, 0777); err != nil { 108 return err 109 } 110 111 if err := os.MkdirAll(pathJoin(fsPath, dataUsageBucket), 0777); err != nil { 112 return err 113 } 114 115 metaMultipartPath := pathJoin(fsPath, minioMetaMultipartBucket) 116 return os.MkdirAll(metaMultipartPath, 0777) 117 118 } 119 120 // NewFSObjectLayer - initialize new fs object layer. 121 func NewFSObjectLayer(fsPath string) (ObjectLayer, error) { 122 ctx := GlobalContext 123 if fsPath == "" { 124 return nil, errInvalidArgument 125 } 126 127 var err error 128 if fsPath, err = getValidPath(fsPath); err != nil { 129 if err == errMinDiskSize { 130 return nil, config.ErrUnableToWriteInBackend(err).Hint(err.Error()) 131 } 132 133 // Show a descriptive error with a hint about how to fix it. 134 var username string 135 if u, err := user.Current(); err == nil { 136 username = u.Username 137 } else { 138 username = "<your-username>" 139 } 140 hint := fmt.Sprintf("Use 'sudo chown -R %s %s && sudo chmod u+rxw %s' to provide sufficient permissions.", username, fsPath, fsPath) 141 return nil, config.ErrUnableToWriteInBackend(err).Hint(hint) 142 } 143 144 // Assign a new UUID for FS minio mode. Each server instance 145 // gets its own UUID for temporary file transaction. 146 fsUUID := mustGetUUID() 147 148 // Initialize meta volume, if volume already exists ignores it. 149 if err = initMetaVolumeFS(fsPath, fsUUID); err != nil { 150 return nil, err 151 } 152 153 // Initialize `format.json`, this function also returns. 154 rlk, err := initFormatFS(ctx, fsPath) 155 if err != nil { 156 return nil, err 157 } 158 159 // Initialize fs objects. 160 fs := &FSObjects{ 161 fsPath: fsPath, 162 metaJSONFile: fsMetaJSONFile, 163 fsUUID: fsUUID, 164 rwPool: &fsIOPool{ 165 readersMap: make(map[string]*lock.RLockedFile), 166 }, 167 nsMutex: newNSLock(false), 168 listPool: NewTreeWalkPool(globalLookupTimeout), 169 appendFileMap: make(map[string]*fsAppendFile), 170 diskMount: mountinfo.IsLikelyMountPoint(fsPath), 171 } 172 173 // Once the filesystem has initialized hold the read lock for 174 // the life time of the server. This is done to ensure that under 175 // shared backend mode for FS, remote servers do not migrate 176 // or cause changes on backend format. 177 fs.fsFormatRlk = rlk 178 179 go fs.cleanupStaleUploads(ctx, GlobalStaleUploadsCleanupInterval, GlobalStaleUploadsExpiry) 180 go intDataUpdateTracker.start(ctx, fsPath) 181 182 // Return successfully initialized object layer. 183 return fs, nil 184 } 185 186 // NewNSLock - initialize a new namespace RWLocker instance. 187 func (fs *FSObjects) NewNSLock(bucket string, objects ...string) RWLocker { 188 // lockers are explicitly 'nil' for FS mode since there are only local lockers 189 return fs.nsMutex.NewNSLock(nil, bucket, objects...) 190 } 191 192 // SetDriveCounts no-op 193 func (fs *FSObjects) SetDriveCounts() []int { 194 return nil 195 } 196 197 // Shutdown - should be called when process shuts down. 198 func (fs *FSObjects) Shutdown(ctx context.Context) error { 199 fs.fsFormatRlk.Close() 200 201 // Cleanup and delete tmp uuid. 202 return fsRemoveAll(ctx, pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID)) 203 } 204 205 // BackendInfo - returns backend information 206 func (fs *FSObjects) BackendInfo() madmin.BackendInfo { 207 return madmin.BackendInfo{Type: madmin.FS} 208 } 209 210 // LocalStorageInfo - returns underlying storage statistics. 211 func (fs *FSObjects) LocalStorageInfo(ctx context.Context) (StorageInfo, []error) { 212 return fs.StorageInfo(ctx) 213 } 214 215 // StorageInfo - returns underlying storage statistics. 216 func (fs *FSObjects) StorageInfo(ctx context.Context) (StorageInfo, []error) { 217 atomic.AddInt64(&fs.activeIOCount, 1) 218 defer func() { 219 atomic.AddInt64(&fs.activeIOCount, -1) 220 }() 221 222 di, err := getDiskInfo(fs.fsPath) 223 if err != nil { 224 return StorageInfo{}, []error{err} 225 } 226 storageInfo := StorageInfo{ 227 Disks: []madmin.Disk{ 228 { 229 TotalSpace: di.Total, 230 UsedSpace: di.Used, 231 AvailableSpace: di.Free, 232 DrivePath: fs.fsPath, 233 }, 234 }, 235 } 236 storageInfo.Backend.Type = madmin.FS 237 return storageInfo, nil 238 } 239 240 // NSScanner returns data usage stats of the current FS deployment 241 func (fs *FSObjects) NSScanner(ctx context.Context, bf *bloomFilter, updates chan<- madmin.DataUsageInfo) error { 242 // Load bucket totals 243 var totalCache dataUsageCache 244 err := totalCache.load(ctx, fs, dataUsageCacheName) 245 if err != nil { 246 return err 247 } 248 totalCache.Info.Name = dataUsageRoot 249 buckets, err := fs.ListBuckets(ctx) 250 if err != nil { 251 return err 252 } 253 totalCache.Info.BloomFilter = bf.bytes() 254 255 // Clear totals. 256 var root dataUsageEntry 257 if r := totalCache.root(); r != nil { 258 root.Children = r.Children 259 } 260 totalCache.replace(dataUsageRoot, "", root) 261 262 // Delete all buckets that does not exist anymore. 263 totalCache.keepBuckets(buckets) 264 265 for _, b := range buckets { 266 // Load bucket cache. 267 var bCache dataUsageCache 268 err := bCache.load(ctx, fs, path.Join(b.Name, dataUsageCacheName)) 269 if err != nil { 270 return err 271 } 272 if bCache.Info.Name == "" { 273 bCache.Info.Name = b.Name 274 } 275 bCache.Info.BloomFilter = totalCache.Info.BloomFilter 276 277 cache, err := fs.scanBucket(ctx, b.Name, bCache) 278 select { 279 case <-ctx.Done(): 280 return ctx.Err() 281 default: 282 } 283 logger.LogIf(ctx, err) 284 cache.Info.BloomFilter = nil 285 286 if cache.root() == nil { 287 if intDataUpdateTracker.debug { 288 logger.Info(color.Green("NSScanner:") + " No root added. Adding empty") 289 } 290 cache.replace(cache.Info.Name, dataUsageRoot, dataUsageEntry{}) 291 } 292 if cache.Info.LastUpdate.After(bCache.Info.LastUpdate) { 293 if intDataUpdateTracker.debug { 294 logger.Info(color.Green("NSScanner:")+" Saving bucket %q cache with %d entries", b.Name, len(cache.Cache)) 295 } 296 logger.LogIf(ctx, cache.save(ctx, fs, path.Join(b.Name, dataUsageCacheName))) 297 } 298 // Merge, save and send update. 299 // We do it even if unchanged. 300 cl := cache.clone() 301 entry := cl.flatten(*cl.root()) 302 totalCache.replace(cl.Info.Name, dataUsageRoot, entry) 303 if intDataUpdateTracker.debug { 304 logger.Info(color.Green("NSScanner:")+" Saving totals cache with %d entries", len(totalCache.Cache)) 305 } 306 totalCache.Info.LastUpdate = time.Now() 307 logger.LogIf(ctx, totalCache.save(ctx, fs, dataUsageCacheName)) 308 cloned := totalCache.clone() 309 updates <- cloned.dui(dataUsageRoot, buckets) 310 enforceFIFOQuotaBucket(ctx, fs, b.Name, cloned.bucketUsageInfo(b.Name)) 311 } 312 313 return nil 314 } 315 316 // scanBucket scans a single bucket in FS mode. 317 // The updated cache for the bucket is returned. 318 // A partially updated bucket may be returned. 319 func (fs *FSObjects) scanBucket(ctx context.Context, bucket string, cache dataUsageCache) (dataUsageCache, error) { 320 // Get bucket policy 321 // Check if the current bucket has a configured lifecycle policy 322 lc, err := globalLifecycleSys.Get(bucket) 323 if err == nil && lc.HasActiveRules("", true) { 324 if intDataUpdateTracker.debug { 325 logger.Info(color.Green("scanBucket:") + " lifecycle: Active rules found") 326 } 327 cache.Info.lifeCycle = lc 328 } 329 330 // Load bucket info. 331 cache, err = scanDataFolder(ctx, fs.fsPath, cache, func(item scannerItem) (sizeSummary, error) { 332 bucket, object := item.bucket, item.objectPath() 333 fsMetaBytes, err := xioutil.ReadFile(pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fs.metaJSONFile)) 334 if err != nil && !osIsNotExist(err) { 335 if intDataUpdateTracker.debug { 336 logger.Info(color.Green("scanBucket:")+" object return unexpected error: %v/%v: %w", item.bucket, item.objectPath(), err) 337 } 338 return sizeSummary{}, errSkipFile 339 } 340 341 fsMeta := newFSMetaV1() 342 metaOk := false 343 if len(fsMetaBytes) > 0 { 344 var json = jsoniter.ConfigCompatibleWithStandardLibrary 345 if err = json.Unmarshal(fsMetaBytes, &fsMeta); err == nil { 346 metaOk = true 347 } 348 } 349 if !metaOk { 350 fsMeta = fs.defaultFsJSON(object) 351 } 352 353 // Stat the file. 354 fi, fiErr := os.Stat(item.Path) 355 if fiErr != nil { 356 if intDataUpdateTracker.debug { 357 logger.Info(color.Green("scanBucket:")+" object path missing: %v: %w", item.Path, fiErr) 358 } 359 return sizeSummary{}, errSkipFile 360 } 361 362 oi := fsMeta.ToObjectInfo(bucket, object, fi) 363 sz := item.applyActions(ctx, fs, actionMeta{oi: oi}) 364 if sz >= 0 { 365 return sizeSummary{totalSize: sz}, nil 366 } 367 368 return sizeSummary{totalSize: fi.Size()}, nil 369 }) 370 371 return cache, err 372 } 373 374 /// Bucket operations 375 376 // getBucketDir - will convert incoming bucket names to 377 // corresponding valid bucket names on the backend in a platform 378 // compatible way for all operating systems. 379 func (fs *FSObjects) getBucketDir(ctx context.Context, bucket string) (string, error) { 380 if bucket == "" || bucket == "." || bucket == ".." { 381 return "", errVolumeNotFound 382 } 383 bucketDir := pathJoin(fs.fsPath, bucket) 384 return bucketDir, nil 385 } 386 387 func (fs *FSObjects) statBucketDir(ctx context.Context, bucket string) (os.FileInfo, error) { 388 bucketDir, err := fs.getBucketDir(ctx, bucket) 389 if err != nil { 390 return nil, err 391 } 392 st, err := fsStatVolume(ctx, bucketDir) 393 if err != nil { 394 return nil, err 395 } 396 return st, nil 397 } 398 399 // MakeBucketWithLocation - create a new bucket, returns if it already exists. 400 func (fs *FSObjects) MakeBucketWithLocation(ctx context.Context, bucket string, opts BucketOptions) error { 401 if opts.LockEnabled || opts.VersioningEnabled { 402 return NotImplemented{} 403 } 404 405 // Verify if bucket is valid. 406 if s3utils.CheckValidBucketNameStrict(bucket) != nil { 407 return BucketNameInvalid{Bucket: bucket} 408 } 409 410 defer ObjectPathUpdated(bucket + slashSeparator) 411 atomic.AddInt64(&fs.activeIOCount, 1) 412 defer func() { 413 atomic.AddInt64(&fs.activeIOCount, -1) 414 }() 415 416 bucketDir, err := fs.getBucketDir(ctx, bucket) 417 if err != nil { 418 return toObjectErr(err, bucket) 419 } 420 421 if err = fsMkdir(ctx, bucketDir); err != nil { 422 return toObjectErr(err, bucket) 423 } 424 425 meta := newBucketMetadata(bucket) 426 if err := meta.Save(ctx, fs); err != nil { 427 return toObjectErr(err, bucket) 428 } 429 430 globalBucketMetadataSys.Set(bucket, meta) 431 432 return nil 433 } 434 435 // GetBucketPolicy - only needed for FS in NAS mode 436 func (fs *FSObjects) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) { 437 meta, err := loadBucketMetadata(ctx, fs, bucket) 438 if err != nil { 439 return nil, BucketPolicyNotFound{Bucket: bucket} 440 } 441 if meta.policyConfig == nil { 442 return nil, BucketPolicyNotFound{Bucket: bucket} 443 } 444 return meta.policyConfig, nil 445 } 446 447 // SetBucketPolicy - only needed for FS in NAS mode 448 func (fs *FSObjects) SetBucketPolicy(ctx context.Context, bucket string, p *policy.Policy) error { 449 meta, err := loadBucketMetadata(ctx, fs, bucket) 450 if err != nil { 451 return err 452 } 453 454 var json = jsoniter.ConfigCompatibleWithStandardLibrary 455 configData, err := json.Marshal(p) 456 if err != nil { 457 return err 458 } 459 meta.PolicyConfigJSON = configData 460 461 return meta.Save(ctx, fs) 462 } 463 464 // DeleteBucketPolicy - only needed for FS in NAS mode 465 func (fs *FSObjects) DeleteBucketPolicy(ctx context.Context, bucket string) error { 466 meta, err := loadBucketMetadata(ctx, fs, bucket) 467 if err != nil { 468 return err 469 } 470 meta.PolicyConfigJSON = nil 471 return meta.Save(ctx, fs) 472 } 473 474 // GetBucketInfo - fetch bucket metadata info. 475 func (fs *FSObjects) GetBucketInfo(ctx context.Context, bucket string) (bi BucketInfo, e error) { 476 atomic.AddInt64(&fs.activeIOCount, 1) 477 defer func() { 478 atomic.AddInt64(&fs.activeIOCount, -1) 479 }() 480 481 st, err := fs.statBucketDir(ctx, bucket) 482 if err != nil { 483 return bi, toObjectErr(err, bucket) 484 } 485 486 createdTime := st.ModTime() 487 meta, err := globalBucketMetadataSys.Get(bucket) 488 if err == nil { 489 createdTime = meta.Created 490 } 491 492 return BucketInfo{ 493 Name: bucket, 494 Created: createdTime, 495 }, nil 496 } 497 498 // ListBuckets - list all s3 compatible buckets (directories) at fsPath. 499 func (fs *FSObjects) ListBuckets(ctx context.Context) ([]BucketInfo, error) { 500 if err := checkPathLength(fs.fsPath); err != nil { 501 logger.LogIf(ctx, err) 502 return nil, err 503 } 504 505 atomic.AddInt64(&fs.activeIOCount, 1) 506 defer func() { 507 atomic.AddInt64(&fs.activeIOCount, -1) 508 }() 509 510 entries, err := readDir(fs.fsPath) 511 if err != nil { 512 logger.LogIf(ctx, errDiskNotFound) 513 return nil, toObjectErr(errDiskNotFound) 514 } 515 516 bucketInfos := make([]BucketInfo, 0, len(entries)) 517 for _, entry := range entries { 518 // Ignore all reserved bucket names and invalid bucket names. 519 if isReservedOrInvalidBucket(entry, false) { 520 continue 521 } 522 var fi os.FileInfo 523 fi, err = fsStatVolume(ctx, pathJoin(fs.fsPath, entry)) 524 // There seems like no practical reason to check for errors 525 // at this point, if there are indeed errors we can simply 526 // just ignore such buckets and list only those which 527 // return proper Stat information instead. 528 if err != nil { 529 // Ignore any errors returned here. 530 continue 531 } 532 var created = fi.ModTime() 533 meta, err := globalBucketMetadataSys.Get(fi.Name()) 534 if err == nil { 535 created = meta.Created 536 } 537 538 bucketInfos = append(bucketInfos, BucketInfo{ 539 Name: fi.Name(), 540 Created: created, 541 }) 542 } 543 544 // Sort bucket infos by bucket name. 545 sort.Slice(bucketInfos, func(i, j int) bool { 546 return bucketInfos[i].Name < bucketInfos[j].Name 547 }) 548 549 // Succes. 550 return bucketInfos, nil 551 } 552 553 // DeleteBucket - delete a bucket and all the metadata associated 554 // with the bucket including pending multipart, object metadata. 555 func (fs *FSObjects) DeleteBucket(ctx context.Context, bucket string, forceDelete bool) error { 556 atomic.AddInt64(&fs.activeIOCount, 1) 557 defer func() { 558 atomic.AddInt64(&fs.activeIOCount, -1) 559 }() 560 561 bucketDir, err := fs.getBucketDir(ctx, bucket) 562 if err != nil { 563 return toObjectErr(err, bucket) 564 } 565 566 if !forceDelete { 567 // Attempt to delete regular bucket. 568 if err = fsRemoveDir(ctx, bucketDir); err != nil { 569 return toObjectErr(err, bucket) 570 } 571 } else { 572 tmpBucketPath := pathJoin(fs.fsPath, minioMetaTmpBucket, bucket+"."+mustGetUUID()) 573 if err = fsSimpleRenameFile(ctx, bucketDir, tmpBucketPath); err != nil { 574 return toObjectErr(err, bucket) 575 } 576 577 go func() { 578 fsRemoveAll(ctx, tmpBucketPath) // ignore returned error if any. 579 }() 580 } 581 582 // Cleanup all the bucket metadata. 583 minioMetadataBucketDir := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket) 584 if err = fsRemoveAll(ctx, minioMetadataBucketDir); err != nil { 585 return toObjectErr(err, bucket) 586 } 587 588 // Delete all bucket metadata. 589 deleteBucketMetadata(ctx, fs, bucket) 590 591 return nil 592 } 593 594 /// Object Operations 595 596 // CopyObject - copy object source object to destination object. 597 // if source object and destination object are same we only 598 // update metadata. 599 func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (oi ObjectInfo, err error) { 600 if srcOpts.VersionID != "" && srcOpts.VersionID != nullVersionID { 601 return oi, VersionNotFound{ 602 Bucket: srcBucket, 603 Object: srcObject, 604 VersionID: srcOpts.VersionID, 605 } 606 } 607 608 cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject)) 609 defer ObjectPathUpdated(path.Join(dstBucket, dstObject)) 610 611 if !cpSrcDstSame { 612 objectDWLock := fs.NewNSLock(dstBucket, dstObject) 613 ctx, err = objectDWLock.GetLock(ctx, globalOperationTimeout) 614 if err != nil { 615 return oi, err 616 } 617 defer objectDWLock.Unlock() 618 } 619 620 atomic.AddInt64(&fs.activeIOCount, 1) 621 defer func() { 622 atomic.AddInt64(&fs.activeIOCount, -1) 623 }() 624 625 if _, err := fs.statBucketDir(ctx, srcBucket); err != nil { 626 return oi, toObjectErr(err, srcBucket) 627 } 628 629 if cpSrcDstSame && srcInfo.metadataOnly { 630 fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, srcBucket, srcObject, fs.metaJSONFile) 631 wlk, err := fs.rwPool.Write(fsMetaPath) 632 if err != nil { 633 wlk, err = fs.rwPool.Create(fsMetaPath) 634 if err != nil { 635 logger.LogIf(ctx, err) 636 return oi, toObjectErr(err, srcBucket, srcObject) 637 } 638 } 639 // This close will allow for locks to be synchronized on `fs.json`. 640 defer wlk.Close() 641 642 // Save objects' metadata in `fs.json`. 643 fsMeta := newFSMetaV1() 644 if _, err = fsMeta.ReadFrom(ctx, wlk); err != nil { 645 // For any error to read fsMeta, set default ETag and proceed. 646 fsMeta = fs.defaultFsJSON(srcObject) 647 } 648 649 fsMeta.Meta = cloneMSS(srcInfo.UserDefined) 650 fsMeta.Meta["etag"] = srcInfo.ETag 651 if _, err = fsMeta.WriteTo(wlk); err != nil { 652 return oi, toObjectErr(err, srcBucket, srcObject) 653 } 654 655 // Stat the file to get file size. 656 fi, err := fsStatFile(ctx, pathJoin(fs.fsPath, srcBucket, srcObject)) 657 if err != nil { 658 return oi, toObjectErr(err, srcBucket, srcObject) 659 } 660 661 // Return the new object info. 662 return fsMeta.ToObjectInfo(srcBucket, srcObject, fi), nil 663 } 664 665 if err := checkPutObjectArgs(ctx, dstBucket, dstObject, fs); err != nil { 666 return ObjectInfo{}, err 667 } 668 669 objInfo, err := fs.putObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, ObjectOptions{ServerSideEncryption: dstOpts.ServerSideEncryption, UserDefined: srcInfo.UserDefined}) 670 if err != nil { 671 return oi, toObjectErr(err, dstBucket, dstObject) 672 } 673 674 return objInfo, nil 675 } 676 677 // GetObjectNInfo - returns object info and a reader for object 678 // content. 679 func (fs *FSObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) { 680 if opts.VersionID != "" && opts.VersionID != nullVersionID { 681 return nil, VersionNotFound{ 682 Bucket: bucket, 683 Object: object, 684 VersionID: opts.VersionID, 685 } 686 } 687 if err = checkGetObjArgs(ctx, bucket, object); err != nil { 688 return nil, err 689 } 690 691 atomic.AddInt64(&fs.activeIOCount, 1) 692 defer func() { 693 atomic.AddInt64(&fs.activeIOCount, -1) 694 }() 695 696 if _, err = fs.statBucketDir(ctx, bucket); err != nil { 697 return nil, toObjectErr(err, bucket) 698 } 699 700 var nsUnlocker = func() {} 701 702 if lockType != noLock { 703 // Lock the object before reading. 704 lock := fs.NewNSLock(bucket, object) 705 switch lockType { 706 case writeLock: 707 ctx, err = lock.GetLock(ctx, globalOperationTimeout) 708 if err != nil { 709 return nil, err 710 } 711 nsUnlocker = lock.Unlock 712 case readLock: 713 ctx, err = lock.GetRLock(ctx, globalOperationTimeout) 714 if err != nil { 715 return nil, err 716 } 717 nsUnlocker = lock.RUnlock 718 } 719 } 720 721 // Otherwise we get the object info 722 var objInfo ObjectInfo 723 if objInfo, err = fs.getObjectInfo(ctx, bucket, object); err != nil { 724 nsUnlocker() 725 return nil, toObjectErr(err, bucket, object) 726 } 727 // For a directory, we need to return a reader that returns no bytes. 728 if HasSuffix(object, SlashSeparator) { 729 // The lock taken above is released when 730 // objReader.Close() is called by the caller. 731 return NewGetObjectReaderFromReader(bytes.NewBuffer(nil), objInfo, opts, nsUnlocker) 732 } 733 // Take a rwPool lock for NFS gateway type deployment 734 rwPoolUnlocker := func() {} 735 if bucket != minioMetaBucket && lockType != noLock { 736 fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fs.metaJSONFile) 737 _, err = fs.rwPool.Open(fsMetaPath) 738 if err != nil && err != errFileNotFound { 739 logger.LogIf(ctx, err) 740 nsUnlocker() 741 return nil, toObjectErr(err, bucket, object) 742 } 743 // Need to clean up lock after getObject is 744 // completed. 745 rwPoolUnlocker = func() { fs.rwPool.Close(fsMetaPath) } 746 } 747 748 objReaderFn, off, length, err := NewGetObjectReader(rs, objInfo, opts, nsUnlocker, rwPoolUnlocker) 749 if err != nil { 750 return nil, err 751 } 752 753 // Read the object, doesn't exist returns an s3 compatible error. 754 fsObjPath := pathJoin(fs.fsPath, bucket, object) 755 readCloser, size, err := fsOpenFile(ctx, fsObjPath, off) 756 if err != nil { 757 rwPoolUnlocker() 758 nsUnlocker() 759 return nil, toObjectErr(err, bucket, object) 760 } 761 762 closeFn := func() { 763 readCloser.Close() 764 } 765 reader := io.LimitReader(readCloser, length) 766 767 // Check if range is valid 768 if off > size || off+length > size { 769 err = InvalidRange{off, length, size} 770 logger.LogIf(ctx, err, logger.Application) 771 closeFn() 772 rwPoolUnlocker() 773 nsUnlocker() 774 return nil, err 775 } 776 777 return objReaderFn(reader, h, opts.CheckPrecondFn, closeFn) 778 } 779 780 // getObject - wrapper for GetObject 781 func (fs *FSObjects) getObject(ctx context.Context, bucket, object string, offset int64, length int64, writer io.Writer, etag string, lock bool) (err error) { 782 if _, err = fs.statBucketDir(ctx, bucket); err != nil { 783 return toObjectErr(err, bucket) 784 } 785 786 // Offset cannot be negative. 787 if offset < 0 { 788 logger.LogIf(ctx, errUnexpected, logger.Application) 789 return toObjectErr(errUnexpected, bucket, object) 790 } 791 792 // Writer cannot be nil. 793 if writer == nil { 794 logger.LogIf(ctx, errUnexpected, logger.Application) 795 return toObjectErr(errUnexpected, bucket, object) 796 } 797 798 // If its a directory request, we return an empty body. 799 if HasSuffix(object, SlashSeparator) { 800 _, err = writer.Write([]byte("")) 801 logger.LogIf(ctx, err) 802 return toObjectErr(err, bucket, object) 803 } 804 805 if bucket != minioMetaBucket { 806 fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fs.metaJSONFile) 807 if lock { 808 _, err = fs.rwPool.Open(fsMetaPath) 809 if err != nil && err != errFileNotFound { 810 logger.LogIf(ctx, err) 811 return toObjectErr(err, bucket, object) 812 } 813 defer fs.rwPool.Close(fsMetaPath) 814 } 815 } 816 817 if etag != "" && etag != defaultEtag { 818 objEtag, perr := fs.getObjectETag(ctx, bucket, object, lock) 819 if perr != nil { 820 return toObjectErr(perr, bucket, object) 821 } 822 if objEtag != etag { 823 logger.LogIf(ctx, InvalidETag{}, logger.Application) 824 return toObjectErr(InvalidETag{}, bucket, object) 825 } 826 } 827 828 // Read the object, doesn't exist returns an s3 compatible error. 829 fsObjPath := pathJoin(fs.fsPath, bucket, object) 830 reader, size, err := fsOpenFile(ctx, fsObjPath, offset) 831 if err != nil { 832 return toObjectErr(err, bucket, object) 833 } 834 defer reader.Close() 835 836 // For negative length we read everything. 837 if length < 0 { 838 length = size - offset 839 } 840 841 // Reply back invalid range if the input offset and length fall out of range. 842 if offset > size || offset+length > size { 843 err = InvalidRange{offset, length, size} 844 logger.LogIf(ctx, err, logger.Application) 845 return err 846 } 847 848 _, err = io.Copy(writer, io.LimitReader(reader, length)) 849 // The writer will be closed incase of range queries, which will emit ErrClosedPipe. 850 if err == io.ErrClosedPipe { 851 err = nil 852 } 853 return toObjectErr(err, bucket, object) 854 } 855 856 // Create a new fs.json file, if the existing one is corrupt. Should happen very rarely. 857 func (fs *FSObjects) createFsJSON(object, fsMetaPath string) error { 858 fsMeta := newFSMetaV1() 859 fsMeta.Meta = map[string]string{ 860 "etag": GenETag(), 861 "content-type": mimedb.TypeByExtension(path.Ext(object)), 862 } 863 wlk, werr := fs.rwPool.Create(fsMetaPath) 864 if werr == nil { 865 _, err := fsMeta.WriteTo(wlk) 866 wlk.Close() 867 return err 868 } 869 return werr 870 } 871 872 // Used to return default etag values when a pre-existing object's meta data is queried. 873 func (fs *FSObjects) defaultFsJSON(object string) fsMetaV1 { 874 fsMeta := newFSMetaV1() 875 fsMeta.Meta = map[string]string{ 876 "etag": defaultEtag, 877 "content-type": mimedb.TypeByExtension(path.Ext(object)), 878 } 879 return fsMeta 880 } 881 882 func (fs *FSObjects) getObjectInfoNoFSLock(ctx context.Context, bucket, object string) (oi ObjectInfo, e error) { 883 fsMeta := fsMetaV1{} 884 if HasSuffix(object, SlashSeparator) { 885 fi, err := fsStatDir(ctx, pathJoin(fs.fsPath, bucket, object)) 886 if err != nil { 887 return oi, err 888 } 889 return fsMeta.ToObjectInfo(bucket, object, fi), nil 890 } 891 892 fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fs.metaJSONFile) 893 // Read `fs.json` to perhaps contend with 894 // parallel Put() operations. 895 896 rc, _, err := fsOpenFile(ctx, fsMetaPath, 0) 897 if err == nil { 898 fsMetaBuf, rerr := ioutil.ReadAll(rc) 899 rc.Close() 900 if rerr == nil { 901 var json = jsoniter.ConfigCompatibleWithStandardLibrary 902 if rerr = json.Unmarshal(fsMetaBuf, &fsMeta); rerr != nil { 903 // For any error to read fsMeta, set default ETag and proceed. 904 fsMeta = fs.defaultFsJSON(object) 905 } 906 } else { 907 // For any error to read fsMeta, set default ETag and proceed. 908 fsMeta = fs.defaultFsJSON(object) 909 } 910 } 911 912 // Return a default etag and content-type based on the object's extension. 913 if err == errFileNotFound { 914 fsMeta = fs.defaultFsJSON(object) 915 } 916 917 // Ignore if `fs.json` is not available, this is true for pre-existing data. 918 if err != nil && err != errFileNotFound { 919 logger.LogIf(ctx, err) 920 return oi, err 921 } 922 923 // Stat the file to get file size. 924 fi, err := fsStatFile(ctx, pathJoin(fs.fsPath, bucket, object)) 925 if err != nil { 926 return oi, err 927 } 928 929 return fsMeta.ToObjectInfo(bucket, object, fi), nil 930 } 931 932 // getObjectInfo - wrapper for reading object metadata and constructs ObjectInfo. 933 func (fs *FSObjects) getObjectInfo(ctx context.Context, bucket, object string) (oi ObjectInfo, e error) { 934 if strings.HasSuffix(object, SlashSeparator) && !fs.isObjectDir(bucket, object) { 935 return oi, errFileNotFound 936 } 937 938 fsMeta := fsMetaV1{} 939 if HasSuffix(object, SlashSeparator) { 940 fi, err := fsStatDir(ctx, pathJoin(fs.fsPath, bucket, object)) 941 if err != nil { 942 return oi, err 943 } 944 return fsMeta.ToObjectInfo(bucket, object, fi), nil 945 } 946 947 fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fs.metaJSONFile) 948 // Read `fs.json` to perhaps contend with 949 // parallel Put() operations. 950 951 rlk, err := fs.rwPool.Open(fsMetaPath) 952 if err == nil { 953 // Read from fs metadata only if it exists. 954 _, rerr := fsMeta.ReadFrom(ctx, rlk.LockedFile) 955 fs.rwPool.Close(fsMetaPath) 956 if rerr != nil { 957 // For any error to read fsMeta, set default ETag and proceed. 958 fsMeta = fs.defaultFsJSON(object) 959 } 960 } 961 962 // Return a default etag and content-type based on the object's extension. 963 if err == errFileNotFound { 964 fsMeta = fs.defaultFsJSON(object) 965 } 966 967 // Ignore if `fs.json` is not available, this is true for pre-existing data. 968 if err != nil && err != errFileNotFound { 969 logger.LogIf(ctx, err) 970 return oi, err 971 } 972 973 // Stat the file to get file size. 974 fi, err := fsStatFile(ctx, pathJoin(fs.fsPath, bucket, object)) 975 if err != nil { 976 return oi, err 977 } 978 979 return fsMeta.ToObjectInfo(bucket, object, fi), nil 980 } 981 982 // getObjectInfoWithLock - reads object metadata and replies back ObjectInfo. 983 func (fs *FSObjects) getObjectInfoWithLock(ctx context.Context, bucket, object string) (oi ObjectInfo, err error) { 984 // Lock the object before reading. 985 lk := fs.NewNSLock(bucket, object) 986 ctx, err = lk.GetRLock(ctx, globalOperationTimeout) 987 if err != nil { 988 return oi, err 989 } 990 defer lk.RUnlock() 991 992 if err := checkGetObjArgs(ctx, bucket, object); err != nil { 993 return oi, err 994 } 995 996 if _, err := fs.statBucketDir(ctx, bucket); err != nil { 997 return oi, err 998 } 999 1000 if strings.HasSuffix(object, SlashSeparator) && !fs.isObjectDir(bucket, object) { 1001 return oi, errFileNotFound 1002 } 1003 1004 return fs.getObjectInfo(ctx, bucket, object) 1005 } 1006 1007 // GetObjectInfo - reads object metadata and replies back ObjectInfo. 1008 func (fs *FSObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (oi ObjectInfo, e error) { 1009 if opts.VersionID != "" && opts.VersionID != nullVersionID { 1010 return oi, VersionNotFound{ 1011 Bucket: bucket, 1012 Object: object, 1013 VersionID: opts.VersionID, 1014 } 1015 } 1016 1017 atomic.AddInt64(&fs.activeIOCount, 1) 1018 defer func() { 1019 atomic.AddInt64(&fs.activeIOCount, -1) 1020 }() 1021 1022 oi, err := fs.getObjectInfoWithLock(ctx, bucket, object) 1023 if err == errCorruptedFormat || err == io.EOF { 1024 lk := fs.NewNSLock(bucket, object) 1025 ctx, err = lk.GetLock(ctx, globalOperationTimeout) 1026 if err != nil { 1027 return oi, toObjectErr(err, bucket, object) 1028 } 1029 1030 fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fs.metaJSONFile) 1031 err = fs.createFsJSON(object, fsMetaPath) 1032 lk.Unlock() 1033 if err != nil { 1034 return oi, toObjectErr(err, bucket, object) 1035 } 1036 1037 oi, err = fs.getObjectInfoWithLock(ctx, bucket, object) 1038 } 1039 return oi, toObjectErr(err, bucket, object) 1040 } 1041 1042 // This function does the following check, suppose 1043 // object is "a/b/c/d", stat makes sure that objects ""a/b/c"" 1044 // "a/b" and "a" do not exist. 1045 func (fs *FSObjects) parentDirIsObject(ctx context.Context, bucket, parent string) bool { 1046 var isParentDirObject func(string) bool 1047 isParentDirObject = func(p string) bool { 1048 if p == "." || p == SlashSeparator { 1049 return false 1050 } 1051 if fsIsFile(ctx, pathJoin(fs.fsPath, bucket, p)) { 1052 // If there is already a file at prefix "p", return true. 1053 return true 1054 } 1055 1056 // Check if there is a file as one of the parent paths. 1057 return isParentDirObject(path.Dir(p)) 1058 } 1059 return isParentDirObject(parent) 1060 } 1061 1062 // PutObject - creates an object upon reading from the input stream 1063 // until EOF, writes data directly to configured filesystem path. 1064 // Additionally writes `fs.json` which carries the necessary metadata 1065 // for future object operations. 1066 func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) { 1067 if opts.Versioned { 1068 return objInfo, NotImplemented{} 1069 } 1070 1071 if err := checkPutObjectArgs(ctx, bucket, object, fs); err != nil { 1072 return ObjectInfo{}, err 1073 } 1074 1075 // Lock the object. 1076 lk := fs.NewNSLock(bucket, object) 1077 ctx, err = lk.GetLock(ctx, globalOperationTimeout) 1078 if err != nil { 1079 logger.LogIf(ctx, err) 1080 return objInfo, err 1081 } 1082 defer lk.Unlock() 1083 defer ObjectPathUpdated(path.Join(bucket, object)) 1084 1085 atomic.AddInt64(&fs.activeIOCount, 1) 1086 defer func() { 1087 atomic.AddInt64(&fs.activeIOCount, -1) 1088 }() 1089 1090 return fs.putObject(ctx, bucket, object, r, opts) 1091 } 1092 1093 // putObject - wrapper for PutObject 1094 func (fs *FSObjects) putObject(ctx context.Context, bucket string, object string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, retErr error) { 1095 data := r.Reader 1096 1097 // No metadata is set, allocate a new one. 1098 meta := cloneMSS(opts.UserDefined) 1099 var err error 1100 1101 // Validate if bucket name is valid and exists. 1102 if _, err = fs.statBucketDir(ctx, bucket); err != nil { 1103 return ObjectInfo{}, toObjectErr(err, bucket) 1104 } 1105 1106 fsMeta := newFSMetaV1() 1107 fsMeta.Meta = meta 1108 1109 // This is a special case with size as '0' and object ends 1110 // with a slash separator, we treat it like a valid operation 1111 // and return success. 1112 if isObjectDir(object, data.Size()) { 1113 // Check if an object is present as one of the parent dir. 1114 if fs.parentDirIsObject(ctx, bucket, path.Dir(object)) { 1115 return ObjectInfo{}, toObjectErr(errFileParentIsFile, bucket, object) 1116 } 1117 if err = mkdirAll(pathJoin(fs.fsPath, bucket, object), 0777); err != nil { 1118 logger.LogIf(ctx, err) 1119 return ObjectInfo{}, toObjectErr(err, bucket, object) 1120 } 1121 var fi os.FileInfo 1122 if fi, err = fsStatDir(ctx, pathJoin(fs.fsPath, bucket, object)); err != nil { 1123 return ObjectInfo{}, toObjectErr(err, bucket, object) 1124 } 1125 return fsMeta.ToObjectInfo(bucket, object, fi), nil 1126 } 1127 1128 // Check if an object is present as one of the parent dir. 1129 if fs.parentDirIsObject(ctx, bucket, path.Dir(object)) { 1130 return ObjectInfo{}, toObjectErr(errFileParentIsFile, bucket, object) 1131 } 1132 1133 // Validate input data size and it can never be less than zero. 1134 if data.Size() < -1 { 1135 logger.LogIf(ctx, errInvalidArgument, logger.Application) 1136 return ObjectInfo{}, errInvalidArgument 1137 } 1138 1139 var wlk *lock.LockedFile 1140 if bucket != minioMetaBucket { 1141 bucketMetaDir := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix) 1142 fsMetaPath := pathJoin(bucketMetaDir, bucket, object, fs.metaJSONFile) 1143 wlk, err = fs.rwPool.Write(fsMetaPath) 1144 var freshFile bool 1145 if err != nil { 1146 wlk, err = fs.rwPool.Create(fsMetaPath) 1147 if err != nil { 1148 logger.LogIf(ctx, err) 1149 return ObjectInfo{}, toObjectErr(err, bucket, object) 1150 } 1151 freshFile = true 1152 } 1153 // This close will allow for locks to be synchronized on `fs.json`. 1154 defer wlk.Close() 1155 defer func() { 1156 // Remove meta file when PutObject encounters 1157 // any error and it is a fresh file. 1158 // 1159 // We should preserve the `fs.json` of any 1160 // existing object 1161 if retErr != nil && freshFile { 1162 tmpDir := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID) 1163 fsRemoveMeta(ctx, bucketMetaDir, fsMetaPath, tmpDir) 1164 } 1165 }() 1166 } 1167 1168 // Uploaded object will first be written to the temporary location which will eventually 1169 // be renamed to the actual location. It is first written to the temporary location 1170 // so that cleaning it up will be easy if the server goes down. 1171 tempObj := mustGetUUID() 1172 1173 fsTmpObjPath := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, tempObj) 1174 bytesWritten, err := fsCreateFile(ctx, fsTmpObjPath, data, data.Size()) 1175 1176 // Delete the temporary object in the case of a 1177 // failure. If PutObject succeeds, then there would be 1178 // nothing to delete. 1179 defer fsRemoveFile(ctx, fsTmpObjPath) 1180 1181 if err != nil { 1182 return ObjectInfo{}, toObjectErr(err, bucket, object) 1183 } 1184 fsMeta.Meta["etag"] = r.MD5CurrentHexString() 1185 1186 // Should return IncompleteBody{} error when reader has fewer 1187 // bytes than specified in request header. 1188 if bytesWritten < data.Size() { 1189 return ObjectInfo{}, IncompleteBody{Bucket: bucket, Object: object} 1190 } 1191 1192 // Entire object was written to the temp location, now it's safe to rename it to the actual location. 1193 fsNSObjPath := pathJoin(fs.fsPath, bucket, object) 1194 if err = fsRenameFile(ctx, fsTmpObjPath, fsNSObjPath); err != nil { 1195 return ObjectInfo{}, toObjectErr(err, bucket, object) 1196 } 1197 1198 if bucket != minioMetaBucket { 1199 // Write FS metadata after a successful namespace operation. 1200 if _, err = fsMeta.WriteTo(wlk); err != nil { 1201 return ObjectInfo{}, toObjectErr(err, bucket, object) 1202 } 1203 } 1204 1205 // Stat the file to fetch timestamp, size. 1206 fi, err := fsStatFile(ctx, pathJoin(fs.fsPath, bucket, object)) 1207 if err != nil { 1208 return ObjectInfo{}, toObjectErr(err, bucket, object) 1209 } 1210 1211 // Success. 1212 return fsMeta.ToObjectInfo(bucket, object, fi), nil 1213 } 1214 1215 // DeleteObjects - deletes an object from a bucket, this operation is destructive 1216 // and there are no rollbacks supported. 1217 func (fs *FSObjects) DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) { 1218 errs := make([]error, len(objects)) 1219 dobjects := make([]DeletedObject, len(objects)) 1220 for idx, object := range objects { 1221 if object.VersionID != "" { 1222 errs[idx] = VersionNotFound{ 1223 Bucket: bucket, 1224 Object: object.ObjectName, 1225 VersionID: object.VersionID, 1226 } 1227 continue 1228 } 1229 _, errs[idx] = fs.DeleteObject(ctx, bucket, object.ObjectName, opts) 1230 if errs[idx] == nil || isErrObjectNotFound(errs[idx]) { 1231 dobjects[idx] = DeletedObject{ 1232 ObjectName: object.ObjectName, 1233 } 1234 errs[idx] = nil 1235 } 1236 } 1237 return dobjects, errs 1238 } 1239 1240 // DeleteObject - deletes an object from a bucket, this operation is destructive 1241 // and there are no rollbacks supported. 1242 func (fs *FSObjects) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) { 1243 if opts.VersionID != "" && opts.VersionID != nullVersionID { 1244 return objInfo, VersionNotFound{ 1245 Bucket: bucket, 1246 Object: object, 1247 VersionID: opts.VersionID, 1248 } 1249 } 1250 1251 // Acquire a write lock before deleting the object. 1252 lk := fs.NewNSLock(bucket, object) 1253 ctx, err = lk.GetLock(ctx, globalOperationTimeout) 1254 if err != nil { 1255 return objInfo, err 1256 } 1257 defer lk.Unlock() 1258 1259 if err = checkDelObjArgs(ctx, bucket, object); err != nil { 1260 return objInfo, err 1261 } 1262 1263 defer ObjectPathUpdated(path.Join(bucket, object)) 1264 1265 atomic.AddInt64(&fs.activeIOCount, 1) 1266 defer func() { 1267 atomic.AddInt64(&fs.activeIOCount, -1) 1268 }() 1269 1270 if _, err = fs.statBucketDir(ctx, bucket); err != nil { 1271 return objInfo, toObjectErr(err, bucket) 1272 } 1273 1274 var rwlk *lock.LockedFile 1275 1276 minioMetaBucketDir := pathJoin(fs.fsPath, minioMetaBucket) 1277 fsMetaPath := pathJoin(minioMetaBucketDir, bucketMetaPrefix, bucket, object, fs.metaJSONFile) 1278 if bucket != minioMetaBucket { 1279 rwlk, err = fs.rwPool.Write(fsMetaPath) 1280 if err != nil && err != errFileNotFound { 1281 logger.LogIf(ctx, err) 1282 return objInfo, toObjectErr(err, bucket, object) 1283 } 1284 } 1285 1286 // Delete the object. 1287 if err = fsDeleteFile(ctx, pathJoin(fs.fsPath, bucket), pathJoin(fs.fsPath, bucket, object)); err != nil { 1288 if rwlk != nil { 1289 rwlk.Close() 1290 } 1291 return objInfo, toObjectErr(err, bucket, object) 1292 } 1293 1294 // Close fsMetaPath before deletion 1295 if rwlk != nil { 1296 rwlk.Close() 1297 } 1298 1299 if bucket != minioMetaBucket { 1300 // Delete the metadata object. 1301 err = fsDeleteFile(ctx, minioMetaBucketDir, fsMetaPath) 1302 if err != nil && err != errFileNotFound { 1303 return objInfo, toObjectErr(err, bucket, object) 1304 } 1305 } 1306 return ObjectInfo{Bucket: bucket, Name: object}, nil 1307 } 1308 1309 func (fs *FSObjects) isLeafDir(bucket string, leafPath string) bool { 1310 return fs.isObjectDir(bucket, leafPath) 1311 } 1312 1313 func (fs *FSObjects) isLeaf(bucket string, leafPath string) bool { 1314 return !strings.HasSuffix(leafPath, slashSeparator) 1315 } 1316 1317 // Returns function "listDir" of the type listDirFunc. 1318 // isLeaf - is used by listDir function to check if an entry 1319 // is a leaf or non-leaf entry. 1320 func (fs *FSObjects) listDirFactory() ListDirFunc { 1321 // listDir - lists all the entries at a given prefix and given entry in the prefix. 1322 listDir := func(bucket, prefixDir, prefixEntry string) (emptyDir bool, entries []string, delayIsLeaf bool) { 1323 var err error 1324 entries, err = readDir(pathJoin(fs.fsPath, bucket, prefixDir)) 1325 if err != nil && err != errFileNotFound { 1326 logger.LogIf(GlobalContext, err) 1327 return false, nil, false 1328 } 1329 if len(entries) == 0 { 1330 return true, nil, false 1331 } 1332 entries, delayIsLeaf = filterListEntries(bucket, prefixDir, entries, prefixEntry, fs.isLeaf) 1333 return false, entries, delayIsLeaf 1334 } 1335 1336 // Return list factory instance. 1337 return listDir 1338 } 1339 1340 // isObjectDir returns true if the specified bucket & prefix exists 1341 // and the prefix represents an empty directory. An S3 empty directory 1342 // is also an empty directory in the FS backend. 1343 func (fs *FSObjects) isObjectDir(bucket, prefix string) bool { 1344 entries, err := readDirN(pathJoin(fs.fsPath, bucket, prefix), 1) 1345 if err != nil { 1346 return false 1347 } 1348 return len(entries) == 0 1349 } 1350 1351 // getObjectETag is a helper function, which returns only the md5sum 1352 // of the file on the disk. 1353 func (fs *FSObjects) getObjectETag(ctx context.Context, bucket, entry string, lock bool) (string, error) { 1354 fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, entry, fs.metaJSONFile) 1355 1356 var reader io.Reader 1357 var fi os.FileInfo 1358 var size int64 1359 if lock { 1360 // Read `fs.json` to perhaps contend with 1361 // parallel Put() operations. 1362 rlk, err := fs.rwPool.Open(fsMetaPath) 1363 // Ignore if `fs.json` is not available, this is true for pre-existing data. 1364 if err != nil && err != errFileNotFound { 1365 logger.LogIf(ctx, err) 1366 return "", toObjectErr(err, bucket, entry) 1367 } 1368 1369 // If file is not found, we don't need to proceed forward. 1370 if err == errFileNotFound { 1371 return "", nil 1372 } 1373 1374 // Read from fs metadata only if it exists. 1375 defer fs.rwPool.Close(fsMetaPath) 1376 1377 // Fetch the size of the underlying file. 1378 fi, err = rlk.LockedFile.Stat() 1379 if err != nil { 1380 logger.LogIf(ctx, err) 1381 return "", toObjectErr(err, bucket, entry) 1382 } 1383 1384 size = fi.Size() 1385 reader = io.NewSectionReader(rlk.LockedFile, 0, fi.Size()) 1386 } else { 1387 var err error 1388 reader, size, err = fsOpenFile(ctx, fsMetaPath, 0) 1389 if err != nil { 1390 return "", toObjectErr(err, bucket, entry) 1391 } 1392 } 1393 1394 // `fs.json` can be empty due to previously failed 1395 // PutObject() transaction, if we arrive at such 1396 // a situation we just ignore and continue. 1397 if size == 0 { 1398 return "", nil 1399 } 1400 1401 fsMetaBuf, err := ioutil.ReadAll(reader) 1402 if err != nil { 1403 logger.LogIf(ctx, err) 1404 return "", toObjectErr(err, bucket, entry) 1405 } 1406 1407 var fsMeta fsMetaV1 1408 var json = jsoniter.ConfigCompatibleWithStandardLibrary 1409 if err = json.Unmarshal(fsMetaBuf, &fsMeta); err != nil { 1410 return "", err 1411 } 1412 1413 // Check if FS metadata is valid, if not return error. 1414 if !isFSMetaValid(fsMeta.Version) { 1415 logger.LogIf(ctx, errCorruptedFormat) 1416 return "", toObjectErr(errCorruptedFormat, bucket, entry) 1417 } 1418 1419 return extractETag(fsMeta.Meta), nil 1420 } 1421 1422 // ListObjectVersions not implemented for FS mode. 1423 func (fs *FSObjects) ListObjectVersions(ctx context.Context, bucket, prefix, marker, versionMarker, delimiter string, maxKeys int) (loi ListObjectVersionsInfo, e error) { 1424 return loi, NotImplemented{} 1425 } 1426 1427 // ListObjects - list all objects at prefix upto maxKeys., optionally delimited by '/'. Maintains the list pool 1428 // state for future re-entrant list requests. 1429 func (fs *FSObjects) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) { 1430 atomic.AddInt64(&fs.activeIOCount, 1) 1431 defer func() { 1432 atomic.AddInt64(&fs.activeIOCount, -1) 1433 }() 1434 1435 return listObjects(ctx, fs, bucket, prefix, marker, delimiter, maxKeys, fs.listPool, 1436 fs.listDirFactory(), fs.isLeaf, fs.isLeafDir, fs.getObjectInfoNoFSLock, fs.getObjectInfoNoFSLock) 1437 } 1438 1439 // GetObjectTags - get object tags from an existing object 1440 func (fs *FSObjects) GetObjectTags(ctx context.Context, bucket, object string, opts ObjectOptions) (*tags.Tags, error) { 1441 if opts.VersionID != "" && opts.VersionID != nullVersionID { 1442 return nil, VersionNotFound{ 1443 Bucket: bucket, 1444 Object: object, 1445 VersionID: opts.VersionID, 1446 } 1447 } 1448 oi, err := fs.GetObjectInfo(ctx, bucket, object, ObjectOptions{}) 1449 if err != nil { 1450 return nil, err 1451 } 1452 1453 return tags.ParseObjectTags(oi.UserTags) 1454 } 1455 1456 // PutObjectTags - replace or add tags to an existing object 1457 func (fs *FSObjects) PutObjectTags(ctx context.Context, bucket, object string, tags string, opts ObjectOptions) (ObjectInfo, error) { 1458 if opts.VersionID != "" && opts.VersionID != nullVersionID { 1459 return ObjectInfo{}, VersionNotFound{ 1460 Bucket: bucket, 1461 Object: object, 1462 VersionID: opts.VersionID, 1463 } 1464 } 1465 1466 fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fs.metaJSONFile) 1467 fsMeta := fsMetaV1{} 1468 wlk, err := fs.rwPool.Write(fsMetaPath) 1469 if err != nil { 1470 wlk, err = fs.rwPool.Create(fsMetaPath) 1471 if err != nil { 1472 logger.LogIf(ctx, err) 1473 return ObjectInfo{}, toObjectErr(err, bucket, object) 1474 } 1475 } 1476 // This close will allow for locks to be synchronized on `fs.json`. 1477 defer wlk.Close() 1478 1479 // Read objects' metadata in `fs.json`. 1480 if _, err = fsMeta.ReadFrom(ctx, wlk); err != nil { 1481 // For any error to read fsMeta, set default ETag and proceed. 1482 fsMeta = fs.defaultFsJSON(object) 1483 } 1484 1485 // clean fsMeta.Meta of tag key, before updating the new tags 1486 delete(fsMeta.Meta, xhttp.AmzObjectTagging) 1487 1488 // Do not update for empty tags 1489 if tags != "" { 1490 fsMeta.Meta[xhttp.AmzObjectTagging] = tags 1491 } 1492 1493 if _, err = fsMeta.WriteTo(wlk); err != nil { 1494 return ObjectInfo{}, toObjectErr(err, bucket, object) 1495 } 1496 1497 // Stat the file to get file size. 1498 fi, err := fsStatFile(ctx, pathJoin(fs.fsPath, bucket, object)) 1499 if err != nil { 1500 return ObjectInfo{}, err 1501 } 1502 1503 return fsMeta.ToObjectInfo(bucket, object, fi), nil 1504 } 1505 1506 // DeleteObjectTags - delete object tags from an existing object 1507 func (fs *FSObjects) DeleteObjectTags(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) { 1508 return fs.PutObjectTags(ctx, bucket, object, "", opts) 1509 } 1510 1511 // HealFormat - no-op for fs, Valid only for Erasure. 1512 func (fs *FSObjects) HealFormat(ctx context.Context, dryRun bool) (madmin.HealResultItem, error) { 1513 return madmin.HealResultItem{}, NotImplemented{} 1514 } 1515 1516 // HealObject - no-op for fs. Valid only for Erasure. 1517 func (fs *FSObjects) HealObject(ctx context.Context, bucket, object, versionID string, opts madmin.HealOpts) ( 1518 res madmin.HealResultItem, err error) { 1519 return res, NotImplemented{} 1520 } 1521 1522 // HealBucket - no-op for fs, Valid only for Erasure. 1523 func (fs *FSObjects) HealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (madmin.HealResultItem, 1524 error) { 1525 return madmin.HealResultItem{}, NotImplemented{} 1526 } 1527 1528 // Walk a bucket, optionally prefix recursively, until we have returned 1529 // all the content to objectInfo channel, it is callers responsibility 1530 // to allocate a receive channel for ObjectInfo, upon any unhandled 1531 // error walker returns error. Optionally if context.Done() is received 1532 // then Walk() stops the walker. 1533 func (fs *FSObjects) Walk(ctx context.Context, bucket, prefix string, results chan<- ObjectInfo, opts ObjectOptions) error { 1534 return fsWalk(ctx, fs, bucket, prefix, fs.listDirFactory(), fs.isLeaf, fs.isLeafDir, results, fs.getObjectInfoNoFSLock, fs.getObjectInfoNoFSLock) 1535 } 1536 1537 // HealObjects - no-op for fs. Valid only for Erasure. 1538 func (fs *FSObjects) HealObjects(ctx context.Context, bucket, prefix string, opts madmin.HealOpts, fn HealObjectFn) (e error) { 1539 logger.LogIf(ctx, NotImplemented{}) 1540 return NotImplemented{} 1541 } 1542 1543 // GetMetrics - no op 1544 func (fs *FSObjects) GetMetrics(ctx context.Context) (*BackendMetrics, error) { 1545 logger.LogIf(ctx, NotImplemented{}) 1546 return &BackendMetrics{}, NotImplemented{} 1547 } 1548 1549 // ListObjectsV2 lists all blobs in bucket filtered by prefix 1550 func (fs *FSObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) { 1551 marker := continuationToken 1552 if marker == "" { 1553 marker = startAfter 1554 } 1555 1556 loi, err := fs.ListObjects(ctx, bucket, prefix, marker, delimiter, maxKeys) 1557 if err != nil { 1558 return result, err 1559 } 1560 1561 listObjectsV2Info := ListObjectsV2Info{ 1562 IsTruncated: loi.IsTruncated, 1563 ContinuationToken: continuationToken, 1564 NextContinuationToken: loi.NextMarker, 1565 Objects: loi.Objects, 1566 Prefixes: loi.Prefixes, 1567 } 1568 return listObjectsV2Info, err 1569 } 1570 1571 // IsNotificationSupported returns whether bucket notification is applicable for this layer. 1572 func (fs *FSObjects) IsNotificationSupported() bool { 1573 return true 1574 } 1575 1576 // IsListenSupported returns whether listen bucket notification is applicable for this layer. 1577 func (fs *FSObjects) IsListenSupported() bool { 1578 return true 1579 } 1580 1581 // IsEncryptionSupported returns whether server side encryption is implemented for this layer. 1582 func (fs *FSObjects) IsEncryptionSupported() bool { 1583 return true 1584 } 1585 1586 // IsCompressionSupported returns whether compression is applicable for this layer. 1587 func (fs *FSObjects) IsCompressionSupported() bool { 1588 return true 1589 } 1590 1591 // IsTaggingSupported returns true, object tagging is supported in fs object layer. 1592 func (fs *FSObjects) IsTaggingSupported() bool { 1593 return true 1594 } 1595 1596 // Health returns health of the object layer 1597 func (fs *FSObjects) Health(ctx context.Context, opts HealthOptions) HealthResult { 1598 if _, err := os.Stat(fs.fsPath); err != nil { 1599 return HealthResult{} 1600 } 1601 return HealthResult{ 1602 Healthy: newObjectLayerFn() != nil, 1603 } 1604 } 1605 1606 // ReadHealth returns "read" health of the object layer 1607 func (fs *FSObjects) ReadHealth(ctx context.Context) bool { 1608 _, err := os.Stat(fs.fsPath) 1609 return err == nil 1610 }