storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/erasure-object.go (about)

     1  /*
     2   * MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package cmd
    18  
    19  import (
    20  	"bytes"
    21  	"context"
    22  	"errors"
    23  	"fmt"
    24  	"io"
    25  	"net/http"
    26  	"path"
    27  	"strings"
    28  	"sync"
    29  	"time"
    30  
    31  	"github.com/minio/minio-go/v7/pkg/tags"
    32  
    33  	xhttp "storj.io/minio/cmd/http"
    34  	"storj.io/minio/cmd/logger"
    35  	"storj.io/minio/pkg/bucket/lifecycle"
    36  	"storj.io/minio/pkg/bucket/replication"
    37  	"storj.io/minio/pkg/madmin"
    38  	"storj.io/minio/pkg/mimedb"
    39  	"storj.io/minio/pkg/sync/errgroup"
    40  )
    41  
    42  // list all errors which can be ignored in object operations.
    43  var objectOpIgnoredErrs = append(baseIgnoredErrs, errDiskAccessDenied, errUnformattedDisk)
    44  
    45  /// Object Operations
    46  
    47  func countOnlineDisks(onlineDisks []StorageAPI) (online int) {
    48  	for _, onlineDisk := range onlineDisks {
    49  		if onlineDisk != nil && onlineDisk.IsOnline() {
    50  			online++
    51  		}
    52  	}
    53  	return online
    54  }
    55  
    56  // CopyObject - copy object source object to destination object.
    57  // if source object and destination object are same we only
    58  // update metadata.
    59  func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (oi ObjectInfo, err error) {
    60  	// This call shouldn't be used for anything other than metadata updates or adding self referential versions.
    61  	if !srcInfo.metadataOnly {
    62  		return oi, NotImplemented{}
    63  	}
    64  
    65  	defer ObjectPathUpdated(pathJoin(dstBucket, dstObject))
    66  
    67  	lk := er.NewNSLock(dstBucket, dstObject)
    68  	ctx, err = lk.GetLock(ctx, globalOperationTimeout)
    69  	if err != nil {
    70  		return oi, err
    71  	}
    72  	defer lk.Unlock()
    73  
    74  	// Read metadata associated with the object from all disks.
    75  	storageDisks := er.getDisks()
    76  	metaArr, errs := readAllFileInfo(ctx, storageDisks, srcBucket, srcObject, srcOpts.VersionID, true)
    77  
    78  	// get Quorum for this object
    79  	readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
    80  	if err != nil {
    81  		return oi, toObjectErr(err, srcBucket, srcObject)
    82  	}
    83  
    84  	// List all online disks.
    85  	onlineDisks, modTime, dataDir := listOnlineDisks(storageDisks, metaArr, errs)
    86  
    87  	// Pick latest valid metadata.
    88  	fi, err := pickValidFileInfo(ctx, metaArr, modTime, dataDir, readQuorum)
    89  	if err != nil {
    90  		return oi, toObjectErr(err, srcBucket, srcObject)
    91  	}
    92  	if fi.Deleted {
    93  		if srcOpts.VersionID == "" {
    94  			return oi, toObjectErr(errFileNotFound, srcBucket, srcObject)
    95  		}
    96  		return fi.ToObjectInfo(srcBucket, srcObject), toObjectErr(errMethodNotAllowed, srcBucket, srcObject)
    97  	}
    98  
    99  	versionID := srcInfo.VersionID
   100  	if srcInfo.versionOnly {
   101  		versionID = dstOpts.VersionID
   102  		// preserve destination versionId if specified.
   103  		if versionID == "" {
   104  			versionID = mustGetUUID()
   105  		}
   106  		modTime = UTCNow()
   107  	}
   108  	fi.VersionID = versionID // set any new versionID we might have created
   109  	fi.ModTime = modTime     // set modTime for the new versionID
   110  	if !dstOpts.MTime.IsZero() {
   111  		modTime = dstOpts.MTime
   112  		fi.ModTime = dstOpts.MTime
   113  	}
   114  	fi.Metadata = srcInfo.UserDefined
   115  	srcInfo.UserDefined["etag"] = srcInfo.ETag
   116  
   117  	// Update `xl.meta` content on each disks.
   118  	for index := range metaArr {
   119  		if metaArr[index].IsValid() {
   120  			metaArr[index].ModTime = modTime
   121  			metaArr[index].VersionID = versionID
   122  			metaArr[index].Metadata = srcInfo.UserDefined
   123  		}
   124  	}
   125  
   126  	// Write unique `xl.meta` for each disk.
   127  	if _, err = writeUniqueFileInfo(ctx, onlineDisks, srcBucket, srcObject, metaArr, writeQuorum); err != nil {
   128  		return oi, toObjectErr(err, srcBucket, srcObject)
   129  	}
   130  
   131  	return fi.ToObjectInfo(srcBucket, srcObject), nil
   132  }
   133  
   134  // GetObjectNInfo - returns object info and an object
   135  // Read(Closer). When err != nil, the returned reader is always nil.
   136  func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
   137  	var unlockOnDefer bool
   138  	var nsUnlocker = func() {}
   139  	defer func() {
   140  		if unlockOnDefer {
   141  			nsUnlocker()
   142  		}
   143  	}()
   144  
   145  	// Acquire lock
   146  	if lockType != noLock {
   147  		lock := er.NewNSLock(bucket, object)
   148  		switch lockType {
   149  		case writeLock:
   150  			ctx, err = lock.GetLock(ctx, globalOperationTimeout)
   151  			if err != nil {
   152  				return nil, err
   153  			}
   154  			nsUnlocker = lock.Unlock
   155  		case readLock:
   156  			ctx, err = lock.GetRLock(ctx, globalOperationTimeout)
   157  			if err != nil {
   158  				return nil, err
   159  			}
   160  			nsUnlocker = lock.RUnlock
   161  		}
   162  		unlockOnDefer = true
   163  	}
   164  
   165  	fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, bucket, object, opts, true)
   166  	if err != nil {
   167  		return nil, toObjectErr(err, bucket, object)
   168  	}
   169  
   170  	objInfo := fi.ToObjectInfo(bucket, object)
   171  	if objInfo.DeleteMarker {
   172  		if opts.VersionID == "" {
   173  			return &GetObjectReader{
   174  				ObjInfo: objInfo,
   175  			}, toObjectErr(errFileNotFound, bucket, object)
   176  		}
   177  		// Make sure to return object info to provide extra information.
   178  		return &GetObjectReader{
   179  			ObjInfo: objInfo,
   180  		}, toObjectErr(errMethodNotAllowed, bucket, object)
   181  	}
   182  	if objInfo.TransitionStatus == lifecycle.TransitionComplete {
   183  		// If transitioned, stream from transition tier unless object is restored locally or restore date is past.
   184  		restoreHdr, ok := caseInsensitiveMap(objInfo.UserDefined).Lookup(xhttp.AmzRestore)
   185  		if !ok || !strings.HasPrefix(restoreHdr, "ongoing-request=false") || (!objInfo.RestoreExpires.IsZero() && time.Now().After(objInfo.RestoreExpires)) {
   186  			return getTransitionedObjectReader(ctx, bucket, object, rs, h, objInfo, opts)
   187  		}
   188  	}
   189  	unlockOnDefer = false
   190  	fn, off, length, nErr := NewGetObjectReader(rs, objInfo, opts, nsUnlocker)
   191  	if nErr != nil {
   192  		return nil, nErr
   193  	}
   194  	pr, pw := io.Pipe()
   195  	go func() {
   196  		err := er.getObjectWithFileInfo(ctx, bucket, object, off, length, pw, fi, metaArr, onlineDisks)
   197  		pw.CloseWithError(err)
   198  	}()
   199  
   200  	// Cleanup function to cause the go routine above to exit, in
   201  	// case of incomplete read.
   202  	pipeCloser := func() { pr.Close() }
   203  
   204  	return fn(pr, h, opts.CheckPrecondFn, pipeCloser)
   205  }
   206  
   207  // GetObject - reads an object erasured coded across multiple
   208  // disks. Supports additional parameters like offset and length
   209  // which are synonymous with HTTP Range requests.
   210  //
   211  // startOffset indicates the starting read location of the object.
   212  // length indicates the total length of the object.
   213  func (er erasureObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error) {
   214  	// Lock the object before reading.
   215  	lk := er.NewNSLock(bucket, object)
   216  	ctx, err = lk.GetRLock(ctx, globalOperationTimeout)
   217  	if err != nil {
   218  		return err
   219  	}
   220  	defer lk.RUnlock()
   221  
   222  	// Start offset cannot be negative.
   223  	if startOffset < 0 {
   224  		logger.LogIf(ctx, errUnexpected, logger.Application)
   225  		return errUnexpected
   226  	}
   227  
   228  	// Writer cannot be nil.
   229  	if writer == nil {
   230  		logger.LogIf(ctx, errUnexpected)
   231  		return errUnexpected
   232  	}
   233  
   234  	return er.getObject(ctx, bucket, object, startOffset, length, writer, opts)
   235  }
   236  
   237  func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, fi FileInfo, metaArr []FileInfo, onlineDisks []StorageAPI) error {
   238  	// Reorder online disks based on erasure distribution order.
   239  	// Reorder parts metadata based on erasure distribution order.
   240  	onlineDisks, metaArr = shuffleDisksAndPartsMetadataByIndex(onlineDisks, metaArr, fi)
   241  
   242  	// For negative length read everything.
   243  	if length < 0 {
   244  		length = fi.Size - startOffset
   245  	}
   246  
   247  	// Reply back invalid range if the input offset and length fall out of range.
   248  	if startOffset > fi.Size || startOffset+length > fi.Size {
   249  		logger.LogIf(ctx, InvalidRange{startOffset, length, fi.Size}, logger.Application)
   250  		return InvalidRange{startOffset, length, fi.Size}
   251  	}
   252  
   253  	// Get start part index and offset.
   254  	partIndex, partOffset, err := fi.ObjectToPartOffset(ctx, startOffset)
   255  	if err != nil {
   256  		return InvalidRange{startOffset, length, fi.Size}
   257  	}
   258  
   259  	// Calculate endOffset according to length
   260  	endOffset := startOffset
   261  	if length > 0 {
   262  		endOffset += length - 1
   263  	}
   264  
   265  	// Get last part index to read given length.
   266  	lastPartIndex, _, err := fi.ObjectToPartOffset(ctx, endOffset)
   267  	if err != nil {
   268  		return InvalidRange{startOffset, length, fi.Size}
   269  	}
   270  
   271  	var totalBytesRead int64
   272  	erasure, err := NewErasure(ctx, fi.Erasure.DataBlocks, fi.Erasure.ParityBlocks, fi.Erasure.BlockSize)
   273  	if err != nil {
   274  		return toObjectErr(err, bucket, object)
   275  	}
   276  	var healOnce sync.Once
   277  
   278  	for ; partIndex <= lastPartIndex; partIndex++ {
   279  		if length == totalBytesRead {
   280  			break
   281  		}
   282  
   283  		partNumber := fi.Parts[partIndex].Number
   284  
   285  		// Save the current part name and size.
   286  		partSize := fi.Parts[partIndex].Size
   287  
   288  		partLength := partSize - partOffset
   289  		// partLength should be adjusted so that we don't write more data than what was requested.
   290  		if partLength > (length - totalBytesRead) {
   291  			partLength = length - totalBytesRead
   292  		}
   293  
   294  		tillOffset := erasure.ShardFileOffset(partOffset, partLength, partSize)
   295  		// Get the checksums of the current part.
   296  		readers := make([]io.ReaderAt, len(onlineDisks))
   297  		prefer := make([]bool, len(onlineDisks))
   298  		for index, disk := range onlineDisks {
   299  			if disk == OfflineDisk {
   300  				continue
   301  			}
   302  			if !metaArr[index].IsValid() {
   303  				continue
   304  			}
   305  			checksumInfo := metaArr[index].Erasure.GetChecksumInfo(partNumber)
   306  			partPath := pathJoin(object, metaArr[index].DataDir, fmt.Sprintf("part.%d", partNumber))
   307  			data := metaArr[index].Data
   308  			readers[index] = newBitrotReader(disk, data, bucket, partPath, tillOffset,
   309  				checksumInfo.Algorithm, checksumInfo.Hash, erasure.ShardSize())
   310  
   311  			// Prefer local disks
   312  			prefer[index] = disk.Hostname() == ""
   313  		}
   314  
   315  		written, err := erasure.Decode(ctx, writer, readers, partOffset, partLength, partSize, prefer)
   316  		// Note: we should not be defer'ing the following closeBitrotReaders() call as
   317  		// we are inside a for loop i.e if we use defer, we would accumulate a lot of open files by the time
   318  		// we return from this function.
   319  		closeBitrotReaders(readers)
   320  		if err != nil {
   321  			// If we have successfully written all the content that was asked
   322  			// by the client, but we still see an error - this would mean
   323  			// that we have some parts or data blocks missing or corrupted
   324  			// - attempt a heal to successfully heal them for future calls.
   325  			if written == partLength {
   326  				var scan madmin.HealScanMode
   327  				if errors.Is(err, errFileNotFound) {
   328  					scan = madmin.HealNormalScan
   329  				} else if errors.Is(err, errFileCorrupt) {
   330  					scan = madmin.HealDeepScan
   331  				}
   332  				if scan != madmin.HealUnknownScan {
   333  					healOnce.Do(func() {
   334  						if _, healing := er.getOnlineDisksWithHealing(); !healing {
   335  							go healObject(bucket, object, fi.VersionID, scan)
   336  						}
   337  					})
   338  				}
   339  			}
   340  			if err != nil {
   341  				return toObjectErr(err, bucket, object)
   342  			}
   343  		}
   344  		for i, r := range readers {
   345  			if r == nil {
   346  				onlineDisks[i] = OfflineDisk
   347  			}
   348  		}
   349  		// Track total bytes read from disk and written to the client.
   350  		totalBytesRead += partLength
   351  		// partOffset will be valid only for the first part, hence reset it to 0 for
   352  		// the remaining parts.
   353  		partOffset = 0
   354  	} // End of read all parts loop.
   355  	// Return success.
   356  	return nil
   357  }
   358  
   359  // getObject wrapper for erasure GetObject
   360  func (er erasureObjects) getObject(ctx context.Context, bucket, object string, startOffset, length int64, writer io.Writer, opts ObjectOptions) error {
   361  	fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, bucket, object, opts, true)
   362  	if err != nil {
   363  		return toObjectErr(err, bucket, object)
   364  	}
   365  	if fi.Deleted {
   366  		if opts.VersionID == "" {
   367  			return toObjectErr(errFileNotFound, bucket, object)
   368  		}
   369  		// Make sure to return object info to provide extra information.
   370  		return toObjectErr(errMethodNotAllowed, bucket, object)
   371  	}
   372  
   373  	return er.getObjectWithFileInfo(ctx, bucket, object, startOffset, length, writer, fi, metaArr, onlineDisks)
   374  }
   375  
   376  // GetObjectInfo - reads object metadata and replies back ObjectInfo.
   377  func (er erasureObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (info ObjectInfo, err error) {
   378  	if !opts.NoLock {
   379  		// Lock the object before reading.
   380  		lk := er.NewNSLock(bucket, object)
   381  		ctx, err = lk.GetRLock(ctx, globalOperationTimeout)
   382  		if err != nil {
   383  			return ObjectInfo{}, err
   384  		}
   385  		defer lk.RUnlock()
   386  	}
   387  
   388  	return er.getObjectInfo(ctx, bucket, object, opts)
   389  }
   390  
   391  func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object string, opts ObjectOptions, readData bool) (fi FileInfo, metaArr []FileInfo, onlineDisks []StorageAPI, err error) {
   392  	disks := er.getDisks()
   393  
   394  	// Read metadata associated with the object from all disks.
   395  	metaArr, errs := readAllFileInfo(ctx, disks, bucket, object, opts.VersionID, readData)
   396  
   397  	readQuorum, _, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
   398  	if err != nil {
   399  		return fi, nil, nil, err
   400  	}
   401  
   402  	if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil {
   403  		if reducedErr == errErasureReadQuorum && bucket != minioMetaBucket {
   404  			if _, ok := isObjectDangling(metaArr, errs, nil); ok {
   405  				reducedErr = errFileNotFound
   406  				if opts.VersionID != "" {
   407  					reducedErr = errFileVersionNotFound
   408  				}
   409  				// Remove the dangling object only when:
   410  				//  - This is a non versioned bucket
   411  				//  - This is a versioned bucket and the version ID is passed, the reason
   412  				//    is that we cannot fetch the ID of the latest version when we don't trust xl.meta
   413  				if !opts.Versioned || opts.VersionID != "" {
   414  					er.deleteObjectVersion(ctx, bucket, object, 1, FileInfo{
   415  						Name:      object,
   416  						VersionID: opts.VersionID,
   417  					}, false)
   418  				}
   419  			}
   420  		}
   421  		return fi, nil, nil, toObjectErr(reducedErr, bucket, object)
   422  	}
   423  
   424  	// List all online disks.
   425  	onlineDisks, modTime, dataDir := listOnlineDisks(disks, metaArr, errs)
   426  
   427  	// Pick latest valid metadata.
   428  	fi, err = pickValidFileInfo(ctx, metaArr, modTime, dataDir, readQuorum)
   429  	if err != nil {
   430  		return fi, nil, nil, err
   431  	}
   432  
   433  	var missingBlocks int
   434  	for i, err := range errs {
   435  		if err != nil && errors.Is(err, errFileNotFound) {
   436  			missingBlocks++
   437  			continue
   438  		}
   439  		if metaArr[i].IsValid() && metaArr[i].ModTime.Equal(fi.ModTime) && metaArr[i].DataDir == fi.DataDir {
   440  			continue
   441  		}
   442  		missingBlocks++
   443  	}
   444  
   445  	// if missing metadata can be reconstructed, attempt to reconstruct.
   446  	if missingBlocks > 0 && missingBlocks < readQuorum {
   447  		if _, healing := er.getOnlineDisksWithHealing(); !healing {
   448  			go healObject(bucket, object, fi.VersionID, madmin.HealNormalScan)
   449  		}
   450  	}
   451  
   452  	return fi, metaArr, onlineDisks, nil
   453  }
   454  
   455  // getObjectInfo - wrapper for reading object metadata and constructs ObjectInfo.
   456  func (er erasureObjects) getObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
   457  	fi, _, _, err := er.getObjectFileInfo(ctx, bucket, object, opts, false)
   458  	if err != nil {
   459  		return objInfo, toObjectErr(err, bucket, object)
   460  
   461  	}
   462  	objInfo = fi.ToObjectInfo(bucket, object)
   463  	if objInfo.TransitionStatus == lifecycle.TransitionComplete {
   464  		// overlay storage class for transitioned objects with transition tier SC Label
   465  		if sc := transitionSC(ctx, bucket); sc != "" {
   466  			objInfo.StorageClass = sc
   467  		}
   468  	}
   469  	if !fi.VersionPurgeStatus.Empty() && opts.VersionID != "" {
   470  		// Make sure to return object info to provide extra information.
   471  		return objInfo, toObjectErr(errMethodNotAllowed, bucket, object)
   472  	}
   473  
   474  	if fi.Deleted {
   475  		if opts.VersionID == "" || opts.DeleteMarker {
   476  			return objInfo, toObjectErr(errFileNotFound, bucket, object)
   477  		}
   478  		// Make sure to return object info to provide extra information.
   479  		return objInfo, toObjectErr(errMethodNotAllowed, bucket, object)
   480  	}
   481  
   482  	return objInfo, nil
   483  }
   484  
   485  func undoRename(disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, isDir bool, errs []error) {
   486  	// Undo rename object on disks where RenameFile succeeded.
   487  
   488  	// If srcEntry/dstEntry are objects then add a trailing slash to copy
   489  	// over all the parts inside the object directory
   490  	if isDir {
   491  		srcEntry = retainSlash(srcEntry)
   492  		dstEntry = retainSlash(dstEntry)
   493  	}
   494  	g := errgroup.WithNErrs(len(disks))
   495  	for index, disk := range disks {
   496  		if disk == nil {
   497  			continue
   498  		}
   499  		index := index
   500  		g.Go(func() error {
   501  			if errs[index] == nil {
   502  				_ = disks[index].RenameFile(context.TODO(), dstBucket, dstEntry, srcBucket, srcEntry)
   503  			}
   504  			return nil
   505  		}, index)
   506  	}
   507  	g.Wait()
   508  }
   509  
   510  // Similar to rename but renames data from srcEntry to dstEntry at dataDir
   511  func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry string, metadata []FileInfo, dstBucket, dstEntry string, writeQuorum int) ([]StorageAPI, error) {
   512  	defer ObjectPathUpdated(pathJoin(srcBucket, srcEntry))
   513  	defer ObjectPathUpdated(pathJoin(dstBucket, dstEntry))
   514  
   515  	g := errgroup.WithNErrs(len(disks))
   516  
   517  	// Rename file on all underlying storage disks.
   518  	for index := range disks {
   519  		index := index
   520  		g.Go(func() error {
   521  			if disks[index] == nil {
   522  				return errDiskNotFound
   523  			}
   524  			// Pick one FileInfo for a disk at index.
   525  			fi := metadata[index]
   526  			// Assign index when index is initialized
   527  			if fi.Erasure.Index == 0 {
   528  				fi.Erasure.Index = index + 1
   529  			}
   530  			if fi.IsValid() {
   531  				return disks[index].RenameData(ctx, srcBucket, srcEntry, fi, dstBucket, dstEntry)
   532  			}
   533  			return errFileCorrupt
   534  		}, index)
   535  	}
   536  
   537  	// Wait for all renames to finish.
   538  	errs := g.Wait()
   539  
   540  	// We can safely allow RenameFile errors up to len(er.getDisks()) - writeQuorum
   541  	// otherwise return failure. Cleanup successful renames.
   542  	err := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
   543  	return evalDisks(disks, errs), err
   544  }
   545  
   546  // rename - common function that renamePart and renameObject use to rename
   547  // the respective underlying storage layer representations.
   548  func rename(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, isDir bool, writeQuorum int, ignoredErr []error) ([]StorageAPI, error) {
   549  	if isDir {
   550  		dstEntry = retainSlash(dstEntry)
   551  		srcEntry = retainSlash(srcEntry)
   552  	}
   553  	defer ObjectPathUpdated(pathJoin(srcBucket, srcEntry))
   554  	defer ObjectPathUpdated(pathJoin(dstBucket, dstEntry))
   555  
   556  	g := errgroup.WithNErrs(len(disks))
   557  
   558  	// Rename file on all underlying storage disks.
   559  	for index := range disks {
   560  		index := index
   561  		g.Go(func() error {
   562  			if disks[index] == nil {
   563  				return errDiskNotFound
   564  			}
   565  			if err := disks[index].RenameFile(ctx, srcBucket, srcEntry, dstBucket, dstEntry); err != nil {
   566  				if !IsErrIgnored(err, ignoredErr...) {
   567  					return err
   568  				}
   569  			}
   570  			return nil
   571  		}, index)
   572  	}
   573  
   574  	// Wait for all renames to finish.
   575  	errs := g.Wait()
   576  
   577  	// We can safely allow RenameFile errors up to len(er.getDisks()) - writeQuorum
   578  	// otherwise return failure. Cleanup successful renames.
   579  	err := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
   580  	if err == errErasureWriteQuorum {
   581  		// Undo all the partial rename operations.
   582  		undoRename(disks, srcBucket, srcEntry, dstBucket, dstEntry, isDir, errs)
   583  	}
   584  	return evalDisks(disks, errs), err
   585  }
   586  
   587  // PutObject - creates an object upon reading from the input stream
   588  // until EOF, erasure codes the data across all disk and additionally
   589  // writes `xl.meta` which carries the necessary metadata for future
   590  // object operations.
   591  func (er erasureObjects) PutObject(ctx context.Context, bucket string, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
   592  	return er.putObject(ctx, bucket, object, data, opts)
   593  }
   594  
   595  // putObject wrapper for erasureObjects PutObject
   596  func (er erasureObjects) putObject(ctx context.Context, bucket string, object string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
   597  	defer func() {
   598  		ObjectPathUpdated(pathJoin(bucket, object))
   599  	}()
   600  
   601  	data := r.Reader
   602  
   603  	uniqueID := mustGetUUID()
   604  	tempObj := uniqueID
   605  	// No metadata is set, allocate a new one.
   606  	if opts.UserDefined == nil {
   607  		opts.UserDefined = make(map[string]string)
   608  	}
   609  
   610  	storageDisks := er.getDisks()
   611  
   612  	parityDrives := len(storageDisks) / 2
   613  	if !opts.MaxParity {
   614  		// Get parity and data drive count based on storage class metadata
   615  		parityDrives = globalStorageClass.GetParityForSC(opts.UserDefined[xhttp.AmzStorageClass])
   616  		if parityDrives <= 0 {
   617  			parityDrives = er.defaultParityCount
   618  		}
   619  	}
   620  	dataDrives := len(storageDisks) - parityDrives
   621  
   622  	// we now know the number of blocks this object needs for data and parity.
   623  	// writeQuorum is dataBlocks + 1
   624  	writeQuorum := dataDrives
   625  	if dataDrives == parityDrives {
   626  		writeQuorum++
   627  	}
   628  
   629  	// Validate input data size and it can never be less than zero.
   630  	if data.Size() < -1 {
   631  		logger.LogIf(ctx, errInvalidArgument, logger.Application)
   632  		return ObjectInfo{}, toObjectErr(errInvalidArgument)
   633  	}
   634  
   635  	// Check if an object is present as one of the parent dir.
   636  	// -- FIXME. (needs a new kind of lock).
   637  	// -- FIXME (this also causes performance issue when disks are down).
   638  	if opts.ParentIsObject != nil && opts.ParentIsObject(ctx, bucket, path.Dir(object)) {
   639  		return ObjectInfo{}, toObjectErr(errFileParentIsFile, bucket, object)
   640  	}
   641  
   642  	// Initialize parts metadata
   643  	partsMetadata := make([]FileInfo, len(storageDisks))
   644  
   645  	fi := newFileInfo(pathJoin(bucket, object), dataDrives, parityDrives)
   646  
   647  	if opts.Versioned {
   648  		fi.VersionID = opts.VersionID
   649  		if fi.VersionID == "" {
   650  			fi.VersionID = mustGetUUID()
   651  		}
   652  	}
   653  	fi.DataDir = mustGetUUID()
   654  
   655  	// Initialize erasure metadata.
   656  	for index := range partsMetadata {
   657  		partsMetadata[index] = fi
   658  	}
   659  
   660  	// Order disks according to erasure distribution
   661  	var onlineDisks []StorageAPI
   662  	onlineDisks, partsMetadata = shuffleDisksAndPartsMetadata(storageDisks, partsMetadata, fi)
   663  
   664  	erasure, err := NewErasure(ctx, fi.Erasure.DataBlocks, fi.Erasure.ParityBlocks, fi.Erasure.BlockSize)
   665  	if err != nil {
   666  		return ObjectInfo{}, toObjectErr(err, bucket, object)
   667  	}
   668  
   669  	// Fetch buffer for I/O, returns from the pool if not allocates a new one and returns.
   670  	var buffer []byte
   671  	switch size := data.Size(); {
   672  	case size == 0:
   673  		buffer = make([]byte, 1) // Allocate atleast a byte to reach EOF
   674  	case size == -1:
   675  		if size := data.ActualSize(); size > 0 && size < fi.Erasure.BlockSize {
   676  			buffer = make([]byte, data.ActualSize()+256, data.ActualSize()*2+512)
   677  		} else {
   678  			buffer = er.bp.Get()
   679  			defer er.bp.Put(buffer)
   680  		}
   681  	case size >= fi.Erasure.BlockSize:
   682  		buffer = er.bp.Get()
   683  		defer er.bp.Put(buffer)
   684  	case size < fi.Erasure.BlockSize:
   685  		// No need to allocate fully blockSizeV1 buffer if the incoming data is smaller.
   686  		buffer = make([]byte, size, 2*size+int64(fi.Erasure.ParityBlocks+fi.Erasure.DataBlocks-1))
   687  	}
   688  
   689  	if len(buffer) > int(fi.Erasure.BlockSize) {
   690  		buffer = buffer[:fi.Erasure.BlockSize]
   691  	}
   692  
   693  	partName := "part.1"
   694  	tempErasureObj := pathJoin(uniqueID, fi.DataDir, partName)
   695  
   696  	// Delete temporary object in the event of failure.
   697  	// If PutObject succeeded there would be no temporary
   698  	// object to delete.
   699  	var online int
   700  	defer func() {
   701  		if online != len(onlineDisks) {
   702  			er.deleteObject(context.Background(), minioMetaTmpBucket, tempObj, writeQuorum)
   703  		}
   704  	}()
   705  
   706  	shardFileSize := erasure.ShardFileSize(data.Size())
   707  	writers := make([]io.Writer, len(onlineDisks))
   708  	var inlineBuffers []*bytes.Buffer
   709  	if shardFileSize >= 0 {
   710  		if !opts.Versioned && shardFileSize < smallFileThreshold {
   711  			inlineBuffers = make([]*bytes.Buffer, len(onlineDisks))
   712  		} else if shardFileSize < smallFileThreshold/8 {
   713  			inlineBuffers = make([]*bytes.Buffer, len(onlineDisks))
   714  		}
   715  	}
   716  	for i, disk := range onlineDisks {
   717  		if disk == nil {
   718  			continue
   719  		}
   720  
   721  		if len(inlineBuffers) > 0 {
   722  			inlineBuffers[i] = bytes.NewBuffer(make([]byte, 0, shardFileSize))
   723  			writers[i] = newStreamingBitrotWriterBuffer(inlineBuffers[i], DefaultBitrotAlgorithm, erasure.ShardSize())
   724  			continue
   725  		}
   726  		writers[i] = newBitrotWriter(disk, minioMetaTmpBucket, tempErasureObj,
   727  			shardFileSize, DefaultBitrotAlgorithm, erasure.ShardSize(), false)
   728  	}
   729  
   730  	n, erasureErr := erasure.Encode(ctx, data, writers, buffer, writeQuorum)
   731  	closeBitrotWriters(writers)
   732  	if erasureErr != nil {
   733  		return ObjectInfo{}, toObjectErr(erasureErr, minioMetaTmpBucket, tempErasureObj)
   734  	}
   735  
   736  	// Should return IncompleteBody{} error when reader has fewer bytes
   737  	// than specified in request header.
   738  	if n < data.Size() {
   739  		return ObjectInfo{}, IncompleteBody{Bucket: bucket, Object: object}
   740  	}
   741  
   742  	if !opts.NoLock {
   743  		var err error
   744  		lk := er.NewNSLock(bucket, object)
   745  		ctx, err = lk.GetLock(ctx, globalOperationTimeout)
   746  		if err != nil {
   747  			return ObjectInfo{}, err
   748  		}
   749  		defer lk.Unlock()
   750  	}
   751  
   752  	for i, w := range writers {
   753  		if w == nil {
   754  			onlineDisks[i] = nil
   755  			continue
   756  		}
   757  		if len(inlineBuffers) > 0 && inlineBuffers[i] != nil {
   758  			partsMetadata[i].Data = inlineBuffers[i].Bytes()
   759  		} else {
   760  			partsMetadata[i].Data = nil
   761  		}
   762  		partsMetadata[i].AddObjectPart(1, "", n, data.ActualSize())
   763  		partsMetadata[i].Erasure.AddChecksumInfo(ChecksumInfo{
   764  			PartNumber: 1,
   765  			Algorithm:  DefaultBitrotAlgorithm,
   766  			Hash:       bitrotWriterSum(w),
   767  		})
   768  	}
   769  	if opts.UserDefined["etag"] == "" {
   770  		opts.UserDefined["etag"] = r.MD5CurrentHexString()
   771  	}
   772  
   773  	// Guess content-type from the extension if possible.
   774  	if opts.UserDefined["content-type"] == "" {
   775  		opts.UserDefined["content-type"] = mimedb.TypeByExtension(path.Ext(object))
   776  	}
   777  
   778  	modTime := opts.MTime
   779  	if opts.MTime.IsZero() {
   780  		modTime = UTCNow()
   781  	}
   782  
   783  	// Fill all the necessary metadata.
   784  	// Update `xl.meta` content on each disks.
   785  	for index := range partsMetadata {
   786  		partsMetadata[index].Metadata = opts.UserDefined
   787  		partsMetadata[index].Size = n
   788  		partsMetadata[index].ModTime = modTime
   789  	}
   790  
   791  	// Rename the successfully written temporary object to final location.
   792  	if onlineDisks, err = renameData(ctx, onlineDisks, minioMetaTmpBucket, tempObj, partsMetadata, bucket, object, writeQuorum); err != nil {
   793  		logger.LogIf(ctx, err)
   794  		return ObjectInfo{}, toObjectErr(err, bucket, object)
   795  	}
   796  
   797  	// Whether a disk was initially or becomes offline
   798  	// during this upload, send it to the MRF list.
   799  	for i := 0; i < len(onlineDisks); i++ {
   800  		if onlineDisks[i] != nil && onlineDisks[i].IsOnline() {
   801  			continue
   802  		}
   803  		er.addPartial(bucket, object, fi.VersionID)
   804  		break
   805  	}
   806  
   807  	for i := 0; i < len(onlineDisks); i++ {
   808  		if onlineDisks[i] != nil && onlineDisks[i].IsOnline() {
   809  			// Object info is the same in all disks, so we can pick
   810  			// the first meta from online disk
   811  			fi = partsMetadata[i]
   812  			break
   813  		}
   814  	}
   815  	online = countOnlineDisks(onlineDisks)
   816  
   817  	return fi.ToObjectInfo(bucket, object), nil
   818  }
   819  
   820  func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object string, writeQuorum int, fi FileInfo, forceDelMarker bool) error {
   821  	defer ObjectPathUpdated(pathJoin(bucket, object))
   822  	disks := er.getDisks()
   823  	g := errgroup.WithNErrs(len(disks))
   824  	for index := range disks {
   825  		index := index
   826  		g.Go(func() error {
   827  			if disks[index] == nil {
   828  				return errDiskNotFound
   829  			}
   830  			return disks[index].DeleteVersion(ctx, bucket, object, fi, forceDelMarker)
   831  		}, index)
   832  	}
   833  	// return errors if any during deletion
   834  	return reduceWriteQuorumErrs(ctx, g.Wait(), objectOpIgnoredErrs, writeQuorum)
   835  }
   836  
   837  // deleteEmptyDir knows only how to remove an empty directory (not the empty object with a
   838  // trailing slash), this is called for the healing code to remove such directories.
   839  func (er erasureObjects) deleteEmptyDir(ctx context.Context, bucket, object string) error {
   840  	defer ObjectPathUpdated(pathJoin(bucket, object))
   841  
   842  	if bucket == minioMetaTmpBucket {
   843  		return nil
   844  	}
   845  
   846  	disks := er.getDisks()
   847  	g := errgroup.WithNErrs(len(disks))
   848  	for index := range disks {
   849  		index := index
   850  		g.Go(func() error {
   851  			if disks[index] == nil {
   852  				return errDiskNotFound
   853  			}
   854  			return disks[index].Delete(ctx, bucket, object, false)
   855  		}, index)
   856  	}
   857  
   858  	// return errors if any during deletion
   859  	return reduceWriteQuorumErrs(ctx, g.Wait(), objectOpIgnoredErrs, len(disks)/2+1)
   860  }
   861  
   862  // deleteObject - wrapper for delete object, deletes an object from
   863  // all the disks in parallel, including `xl.meta` associated with the
   864  // object.
   865  func (er erasureObjects) deleteObject(ctx context.Context, bucket, object string, writeQuorum int) error {
   866  	var err error
   867  	defer ObjectPathUpdated(pathJoin(bucket, object))
   868  
   869  	disks := er.getDisks()
   870  	tmpObj := mustGetUUID()
   871  	if bucket == minioMetaTmpBucket {
   872  		tmpObj = object
   873  	} else {
   874  		// Rename the current object while requiring write quorum, but also consider
   875  		// that a non found object in a given disk as a success since it already
   876  		// confirms that the object doesn't have a part in that disk (already removed)
   877  		disks, err = rename(ctx, disks, bucket, object, minioMetaTmpBucket, tmpObj, true, writeQuorum,
   878  			[]error{errFileNotFound})
   879  		if err != nil {
   880  			return toObjectErr(err, bucket, object)
   881  		}
   882  	}
   883  
   884  	g := errgroup.WithNErrs(len(disks))
   885  	for index := range disks {
   886  		index := index
   887  		g.Go(func() error {
   888  			if disks[index] == nil {
   889  				return errDiskNotFound
   890  			}
   891  			return disks[index].Delete(ctx, minioMetaTmpBucket, tmpObj, true)
   892  		}, index)
   893  	}
   894  
   895  	// return errors if any during deletion
   896  	return reduceWriteQuorumErrs(ctx, g.Wait(), objectOpIgnoredErrs, writeQuorum)
   897  }
   898  
   899  // DeleteObjects deletes objects/versions in bulk, this function will still automatically split objects list
   900  // into smaller bulks if some object names are found to be duplicated in the delete list, splitting
   901  // into smaller bulks will avoid holding twice the write lock of the duplicated object names.
   902  func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) {
   903  	errs := make([]error, len(objects))
   904  	dobjects := make([]DeletedObject, len(objects))
   905  	writeQuorums := make([]int, len(objects))
   906  
   907  	storageDisks := er.getDisks()
   908  
   909  	for i := range objects {
   910  		// Assume (N/2 + 1) quorums for all objects
   911  		// this is a theoretical assumption such that
   912  		// for delete's we do not need to honor storage
   913  		// class for objects which have reduced quorum
   914  		// storage class only needs to be honored for
   915  		// Read() requests alone which we already do.
   916  		writeQuorums[i] = getWriteQuorum(len(storageDisks))
   917  	}
   918  
   919  	versions := make([]FileInfo, len(objects))
   920  	for i := range objects {
   921  		if objects[i].VersionID == "" {
   922  			modTime := opts.MTime
   923  			if opts.MTime.IsZero() {
   924  				modTime = UTCNow()
   925  			}
   926  			uuid := opts.VersionID
   927  			if uuid == "" {
   928  				uuid = mustGetUUID()
   929  			}
   930  			if opts.Versioned || opts.VersionSuspended {
   931  				versions[i] = FileInfo{
   932  					Name:                          objects[i].ObjectName,
   933  					ModTime:                       modTime,
   934  					Deleted:                       true, // delete marker
   935  					DeleteMarkerReplicationStatus: objects[i].DeleteMarkerReplicationStatus,
   936  					VersionPurgeStatus:            objects[i].VersionPurgeStatus,
   937  				}
   938  				if opts.Versioned {
   939  					versions[i].VersionID = uuid
   940  				}
   941  				continue
   942  			}
   943  		}
   944  		versions[i] = FileInfo{
   945  			Name:                          objects[i].ObjectName,
   946  			VersionID:                     objects[i].VersionID,
   947  			DeleteMarkerReplicationStatus: objects[i].DeleteMarkerReplicationStatus,
   948  			VersionPurgeStatus:            objects[i].VersionPurgeStatus,
   949  		}
   950  	}
   951  
   952  	// Initialize list of errors.
   953  	var delObjErrs = make([][]error, len(storageDisks))
   954  
   955  	var wg sync.WaitGroup
   956  	// Remove versions in bulk for each disk
   957  	for index, disk := range storageDisks {
   958  		wg.Add(1)
   959  		go func(index int, disk StorageAPI) {
   960  			defer wg.Done()
   961  			if disk == nil {
   962  				delObjErrs[index] = make([]error, len(versions))
   963  				for i := range versions {
   964  					delObjErrs[index][i] = errDiskNotFound
   965  				}
   966  				return
   967  			}
   968  			delObjErrs[index] = disk.DeleteVersions(ctx, bucket, versions)
   969  		}(index, disk)
   970  	}
   971  
   972  	wg.Wait()
   973  
   974  	// Reduce errors for each object
   975  	for objIndex := range objects {
   976  		diskErrs := make([]error, len(storageDisks))
   977  		// Iterate over disks to fetch the error
   978  		// of deleting of the current object
   979  		for i := range delObjErrs {
   980  			// delObjErrs[i] is not nil when disks[i] is also not nil
   981  			if delObjErrs[i] != nil {
   982  				diskErrs[i] = delObjErrs[i][objIndex]
   983  			}
   984  		}
   985  		err := reduceWriteQuorumErrs(ctx, diskErrs, objectOpIgnoredErrs, writeQuorums[objIndex])
   986  		if objects[objIndex].VersionID != "" {
   987  			errs[objIndex] = toObjectErr(err, bucket, objects[objIndex].ObjectName, objects[objIndex].VersionID)
   988  		} else {
   989  			errs[objIndex] = toObjectErr(err, bucket, objects[objIndex].ObjectName)
   990  		}
   991  
   992  		if errs[objIndex] == nil {
   993  			ObjectPathUpdated(pathJoin(bucket, objects[objIndex].ObjectName))
   994  		}
   995  
   996  		if versions[objIndex].Deleted {
   997  			dobjects[objIndex] = DeletedObject{
   998  				DeleteMarker:                  versions[objIndex].Deleted,
   999  				DeleteMarkerVersionID:         versions[objIndex].VersionID,
  1000  				DeleteMarkerMTime:             DeleteMarkerMTime{versions[objIndex].ModTime},
  1001  				DeleteMarkerReplicationStatus: versions[objIndex].DeleteMarkerReplicationStatus,
  1002  				ObjectName:                    versions[objIndex].Name,
  1003  				VersionPurgeStatus:            versions[objIndex].VersionPurgeStatus,
  1004  				PurgeTransitioned:             objects[objIndex].PurgeTransitioned,
  1005  			}
  1006  		} else {
  1007  			dobjects[objIndex] = DeletedObject{
  1008  				ObjectName:                    versions[objIndex].Name,
  1009  				VersionID:                     versions[objIndex].VersionID,
  1010  				VersionPurgeStatus:            versions[objIndex].VersionPurgeStatus,
  1011  				DeleteMarkerReplicationStatus: versions[objIndex].DeleteMarkerReplicationStatus,
  1012  				PurgeTransitioned:             objects[objIndex].PurgeTransitioned,
  1013  			}
  1014  		}
  1015  	}
  1016  
  1017  	// Check failed deletes across multiple objects
  1018  	for _, version := range versions {
  1019  		// Check if there is any offline disk and add it to the MRF list
  1020  		for _, disk := range storageDisks {
  1021  			if disk != nil && disk.IsOnline() {
  1022  				// Skip attempted heal on online disks.
  1023  				continue
  1024  			}
  1025  
  1026  			// all other direct versionId references we should
  1027  			// ensure no dangling file is left over.
  1028  			er.addPartial(bucket, version.Name, version.VersionID)
  1029  			break
  1030  		}
  1031  	}
  1032  
  1033  	return dobjects, errs
  1034  }
  1035  
  1036  // DeleteObject - deletes an object, this call doesn't necessary reply
  1037  // any error as it is not necessary for the handler to reply back a
  1038  // response to the client request.
  1039  func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
  1040  	versionFound := true
  1041  	objInfo = ObjectInfo{VersionID: opts.VersionID} // version id needed in Delete API response.
  1042  	goi, gerr := er.GetObjectInfo(ctx, bucket, object, opts)
  1043  	if gerr != nil && goi.Name == "" {
  1044  		switch gerr.(type) {
  1045  		case InsufficientReadQuorum:
  1046  			return objInfo, InsufficientWriteQuorum{}
  1047  		}
  1048  		// For delete marker replication, versionID being replicated will not exist on disk
  1049  		if opts.DeleteMarker {
  1050  			versionFound = false
  1051  		} else {
  1052  			return objInfo, gerr
  1053  		}
  1054  	}
  1055  	// Acquire a write lock before deleting the object.
  1056  	lk := er.NewNSLock(bucket, object)
  1057  	ctx, err = lk.GetLock(ctx, globalDeleteOperationTimeout)
  1058  	if err != nil {
  1059  		return ObjectInfo{}, err
  1060  	}
  1061  	defer lk.Unlock()
  1062  
  1063  	storageDisks := er.getDisks()
  1064  	writeQuorum := len(storageDisks)/2 + 1
  1065  	var markDelete bool
  1066  	// Determine whether to mark object deleted for replication
  1067  	if goi.VersionID != "" {
  1068  		markDelete = true
  1069  	}
  1070  
  1071  	// Default deleteMarker to true if object is under versioning
  1072  	deleteMarker := opts.Versioned
  1073  
  1074  	if opts.VersionID != "" {
  1075  		// case where replica version needs to be deleted on target cluster
  1076  		if versionFound && opts.DeleteMarkerReplicationStatus == replication.Replica.String() {
  1077  			markDelete = false
  1078  		}
  1079  		if opts.VersionPurgeStatus.Empty() && opts.DeleteMarkerReplicationStatus == "" {
  1080  			markDelete = false
  1081  		}
  1082  		if opts.VersionPurgeStatus == Complete {
  1083  			markDelete = false
  1084  		}
  1085  		// determine if the version represents an object delete
  1086  		// deleteMarker = true
  1087  		if versionFound && !goi.DeleteMarker { // implies a versioned delete of object
  1088  			deleteMarker = false
  1089  		}
  1090  	}
  1091  
  1092  	modTime := opts.MTime
  1093  	if opts.MTime.IsZero() {
  1094  		modTime = UTCNow()
  1095  	}
  1096  	if markDelete {
  1097  		if opts.Versioned || opts.VersionSuspended {
  1098  			fi := FileInfo{
  1099  				Name:                          object,
  1100  				Deleted:                       deleteMarker,
  1101  				MarkDeleted:                   markDelete,
  1102  				ModTime:                       modTime,
  1103  				DeleteMarkerReplicationStatus: opts.DeleteMarkerReplicationStatus,
  1104  				VersionPurgeStatus:            opts.VersionPurgeStatus,
  1105  			}
  1106  			if opts.Versioned {
  1107  				fi.VersionID = mustGetUUID()
  1108  				if opts.VersionID != "" {
  1109  					fi.VersionID = opts.VersionID
  1110  				}
  1111  			}
  1112  			fi.TransitionStatus = opts.TransitionStatus
  1113  
  1114  			// versioning suspended means we add `null`
  1115  			// version as delete marker
  1116  			// Add delete marker, since we don't have any version specified explicitly.
  1117  			// Or if a particular version id needs to be replicated.
  1118  			if err = er.deleteObjectVersion(ctx, bucket, object, writeQuorum, fi, opts.DeleteMarker); err != nil {
  1119  				return objInfo, toObjectErr(err, bucket, object)
  1120  			}
  1121  			return fi.ToObjectInfo(bucket, object), nil
  1122  		}
  1123  	}
  1124  
  1125  	// Delete the object version on all disks.
  1126  	if err = er.deleteObjectVersion(ctx, bucket, object, writeQuorum, FileInfo{
  1127  		Name:                          object,
  1128  		VersionID:                     opts.VersionID,
  1129  		MarkDeleted:                   markDelete,
  1130  		Deleted:                       deleteMarker,
  1131  		ModTime:                       modTime,
  1132  		DeleteMarkerReplicationStatus: opts.DeleteMarkerReplicationStatus,
  1133  		VersionPurgeStatus:            opts.VersionPurgeStatus,
  1134  		TransitionStatus:              opts.TransitionStatus,
  1135  	}, opts.DeleteMarker); err != nil {
  1136  		return objInfo, toObjectErr(err, bucket, object)
  1137  	}
  1138  
  1139  	for _, disk := range storageDisks {
  1140  		if disk != nil && disk.IsOnline() {
  1141  			continue
  1142  		}
  1143  		er.addPartial(bucket, object, opts.VersionID)
  1144  		break
  1145  	}
  1146  
  1147  	return ObjectInfo{
  1148  		Bucket:             bucket,
  1149  		Name:               object,
  1150  		VersionID:          opts.VersionID,
  1151  		VersionPurgeStatus: opts.VersionPurgeStatus,
  1152  		ReplicationStatus:  replication.StatusType(opts.DeleteMarkerReplicationStatus),
  1153  	}, nil
  1154  }
  1155  
  1156  // Send the successful but partial upload/delete, however ignore
  1157  // if the channel is blocked by other items.
  1158  func (er erasureObjects) addPartial(bucket, object, versionID string) {
  1159  	select {
  1160  	case er.mrfOpCh <- partialOperation{bucket: bucket, object: object, versionID: versionID}:
  1161  	default:
  1162  	}
  1163  }
  1164  
  1165  func (er erasureObjects) PutObjectMetadata(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
  1166  	var err error
  1167  	// Lock the object before updating tags.
  1168  	lk := er.NewNSLock(bucket, object)
  1169  	ctx, err = lk.GetLock(ctx, globalOperationTimeout)
  1170  	if err != nil {
  1171  		return ObjectInfo{}, err
  1172  	}
  1173  	defer lk.Unlock()
  1174  
  1175  	disks := er.getDisks()
  1176  
  1177  	// Read metadata associated with the object from all disks.
  1178  	metaArr, errs := readAllFileInfo(ctx, disks, bucket, object, opts.VersionID, false)
  1179  
  1180  	readQuorum, _, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
  1181  	if err != nil {
  1182  		return ObjectInfo{}, toObjectErr(err, bucket, object)
  1183  	}
  1184  
  1185  	// List all online disks.
  1186  	_, modTime, dataDir := listOnlineDisks(disks, metaArr, errs)
  1187  
  1188  	// Pick latest valid metadata.
  1189  	fi, err := pickValidFileInfo(ctx, metaArr, modTime, dataDir, readQuorum)
  1190  	if err != nil {
  1191  		return ObjectInfo{}, toObjectErr(err, bucket, object)
  1192  	}
  1193  	if fi.Deleted {
  1194  		if opts.VersionID == "" {
  1195  			return ObjectInfo{}, toObjectErr(errFileNotFound, bucket, object)
  1196  		}
  1197  		return ObjectInfo{}, toObjectErr(errMethodNotAllowed, bucket, object)
  1198  	}
  1199  
  1200  	for k, v := range opts.UserDefined {
  1201  		fi.Metadata[k] = v
  1202  	}
  1203  	fi.ModTime = opts.MTime
  1204  	fi.VersionID = opts.VersionID
  1205  
  1206  	if err = er.updateObjectMeta(ctx, bucket, object, fi); err != nil {
  1207  		return ObjectInfo{}, toObjectErr(err, bucket, object)
  1208  	}
  1209  
  1210  	objInfo := fi.ToObjectInfo(bucket, object)
  1211  	return objInfo, nil
  1212  
  1213  }
  1214  
  1215  // PutObjectTags - replace or add tags to an existing object
  1216  func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object string, tags string, opts ObjectOptions) (ObjectInfo, error) {
  1217  	var err error
  1218  	// Lock the object before updating tags.
  1219  	lk := er.NewNSLock(bucket, object)
  1220  	ctx, err = lk.GetLock(ctx, globalOperationTimeout)
  1221  	if err != nil {
  1222  		return ObjectInfo{}, err
  1223  	}
  1224  	defer lk.Unlock()
  1225  
  1226  	disks := er.getDisks()
  1227  
  1228  	// Read metadata associated with the object from all disks.
  1229  	metaArr, errs := readAllFileInfo(ctx, disks, bucket, object, opts.VersionID, false)
  1230  
  1231  	readQuorum, _, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
  1232  	if err != nil {
  1233  		return ObjectInfo{}, toObjectErr(err, bucket, object)
  1234  	}
  1235  
  1236  	// List all online disks.
  1237  	_, modTime, dataDir := listOnlineDisks(disks, metaArr, errs)
  1238  
  1239  	// Pick latest valid metadata.
  1240  	fi, err := pickValidFileInfo(ctx, metaArr, modTime, dataDir, readQuorum)
  1241  	if err != nil {
  1242  		return ObjectInfo{}, toObjectErr(err, bucket, object)
  1243  	}
  1244  	if fi.Deleted {
  1245  		if opts.VersionID == "" {
  1246  			return ObjectInfo{}, toObjectErr(errFileNotFound, bucket, object)
  1247  		}
  1248  		return ObjectInfo{}, toObjectErr(errMethodNotAllowed, bucket, object)
  1249  	}
  1250  
  1251  	fi.Metadata[xhttp.AmzObjectTagging] = tags
  1252  	for k, v := range opts.UserDefined {
  1253  		fi.Metadata[k] = v
  1254  	}
  1255  
  1256  	if err = er.updateObjectMeta(ctx, bucket, object, fi); err != nil {
  1257  		return ObjectInfo{}, toObjectErr(err, bucket, object)
  1258  	}
  1259  
  1260  	return fi.ToObjectInfo(bucket, object), nil
  1261  }
  1262  
  1263  // updateObjectMeta will update the metadata of a file.
  1264  func (er erasureObjects) updateObjectMeta(ctx context.Context, bucket, object string, fi FileInfo) error {
  1265  	if len(fi.Metadata) == 0 {
  1266  		return nil
  1267  	}
  1268  
  1269  	disks := er.getDisks()
  1270  
  1271  	g := errgroup.WithNErrs(len(disks))
  1272  
  1273  	// Start writing `xl.meta` to all disks in parallel.
  1274  	for index := range disks {
  1275  		index := index
  1276  		g.Go(func() error {
  1277  			if disks[index] == nil {
  1278  				return errDiskNotFound
  1279  			}
  1280  			return disks[index].UpdateMetadata(ctx, bucket, object, fi)
  1281  		}, index)
  1282  	}
  1283  
  1284  	// Wait for all the routines.
  1285  	mErrs := g.Wait()
  1286  
  1287  	return reduceWriteQuorumErrs(ctx, mErrs, objectOpIgnoredErrs, getWriteQuorum(len(disks)))
  1288  }
  1289  
  1290  // DeleteObjectTags - delete object tags from an existing object
  1291  func (er erasureObjects) DeleteObjectTags(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
  1292  	return er.PutObjectTags(ctx, bucket, object, "", opts)
  1293  }
  1294  
  1295  // GetObjectTags - get object tags from an existing object
  1296  func (er erasureObjects) GetObjectTags(ctx context.Context, bucket, object string, opts ObjectOptions) (*tags.Tags, error) {
  1297  	// GetObjectInfo will return tag value as well
  1298  	oi, err := er.GetObjectInfo(ctx, bucket, object, opts)
  1299  	if err != nil {
  1300  		return nil, err
  1301  	}
  1302  
  1303  	return tags.ParseObjectTags(oi.UserTags)
  1304  }