storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/erasure-metadata.go (about)

     1  /*
     2   * MinIO Cloud Storage, (C) 2016-2019 MinIO, Inc.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package cmd
    18  
    19  import (
    20  	"context"
    21  	"crypto/sha256"
    22  	"encoding/hex"
    23  	"fmt"
    24  	"net/http"
    25  	"sort"
    26  	"time"
    27  
    28  	xhttp "storj.io/minio/cmd/http"
    29  	"storj.io/minio/cmd/logger"
    30  	"storj.io/minio/pkg/bucket/replication"
    31  	"storj.io/minio/pkg/sync/errgroup"
    32  )
    33  
    34  const erasureAlgorithm = "rs-vandermonde"
    35  
    36  // byObjectPartNumber is a collection satisfying sort.Interface.
    37  type byObjectPartNumber []ObjectPartInfo
    38  
    39  func (t byObjectPartNumber) Len() int           { return len(t) }
    40  func (t byObjectPartNumber) Swap(i, j int)      { t[i], t[j] = t[j], t[i] }
    41  func (t byObjectPartNumber) Less(i, j int) bool { return t[i].Number < t[j].Number }
    42  
    43  // AddChecksumInfo adds a checksum of a part.
    44  func (e *ErasureInfo) AddChecksumInfo(ckSumInfo ChecksumInfo) {
    45  	for i, sum := range e.Checksums {
    46  		if sum.PartNumber == ckSumInfo.PartNumber {
    47  			e.Checksums[i] = ckSumInfo
    48  			return
    49  		}
    50  	}
    51  	e.Checksums = append(e.Checksums, ckSumInfo)
    52  }
    53  
    54  // GetChecksumInfo - get checksum of a part.
    55  func (e ErasureInfo) GetChecksumInfo(partNumber int) (ckSum ChecksumInfo) {
    56  	for _, sum := range e.Checksums {
    57  		if sum.PartNumber == partNumber {
    58  			// Return the checksum
    59  			return sum
    60  		}
    61  	}
    62  	return ChecksumInfo{}
    63  }
    64  
    65  // ShardFileSize - returns final erasure size from original size.
    66  func (e ErasureInfo) ShardFileSize(totalLength int64) int64 {
    67  	if totalLength == 0 {
    68  		return 0
    69  	}
    70  	if totalLength == -1 {
    71  		return -1
    72  	}
    73  	numShards := totalLength / e.BlockSize
    74  	lastBlockSize := totalLength % e.BlockSize
    75  	lastShardSize := ceilFrac(lastBlockSize, int64(e.DataBlocks))
    76  	return numShards*e.ShardSize() + lastShardSize
    77  }
    78  
    79  // ShardSize - returns actual shared size from erasure blockSize.
    80  func (e ErasureInfo) ShardSize() int64 {
    81  	return ceilFrac(e.BlockSize, int64(e.DataBlocks))
    82  }
    83  
    84  // IsValid - tells if erasure info fields are valid.
    85  func (fi FileInfo) IsValid() bool {
    86  	if fi.Deleted {
    87  		// Delete marker has no data, no need to check
    88  		// for erasure coding information
    89  		return true
    90  	}
    91  	dataBlocks := fi.Erasure.DataBlocks
    92  	parityBlocks := fi.Erasure.ParityBlocks
    93  	correctIndexes := (fi.Erasure.Index > 0 &&
    94  		fi.Erasure.Index <= dataBlocks+parityBlocks &&
    95  		len(fi.Erasure.Distribution) == (dataBlocks+parityBlocks))
    96  	return ((dataBlocks >= parityBlocks) &&
    97  		(dataBlocks != 0) && (parityBlocks != 0) &&
    98  		correctIndexes)
    99  }
   100  
   101  // ToObjectInfo - Converts metadata to object info.
   102  func (fi FileInfo) ToObjectInfo(bucket, object string) ObjectInfo {
   103  	object = decodeDirObject(object)
   104  	versionID := fi.VersionID
   105  	if (globalBucketVersioningSys.Enabled(bucket) || globalBucketVersioningSys.Suspended(bucket)) && versionID == "" {
   106  		versionID = nullVersionID
   107  	}
   108  
   109  	objInfo := ObjectInfo{
   110  		IsDir:            HasSuffix(object, SlashSeparator),
   111  		Bucket:           bucket,
   112  		Name:             object,
   113  		VersionID:        versionID,
   114  		IsLatest:         fi.IsLatest,
   115  		DeleteMarker:     fi.Deleted,
   116  		Size:             fi.Size,
   117  		ModTime:          fi.ModTime,
   118  		Legacy:           fi.XLV1,
   119  		ContentType:      fi.Metadata["content-type"],
   120  		ContentEncoding:  fi.Metadata["content-encoding"],
   121  		NumVersions:      fi.NumVersions,
   122  		SuccessorModTime: fi.SuccessorModTime,
   123  	}
   124  
   125  	// Update expires
   126  	var (
   127  		t time.Time
   128  		e error
   129  	)
   130  	if exp, ok := fi.Metadata["expires"]; ok {
   131  		if t, e = time.Parse(http.TimeFormat, exp); e == nil {
   132  			objInfo.Expires = t.UTC()
   133  		}
   134  	}
   135  	objInfo.backendType = BackendErasure
   136  
   137  	// Extract etag from metadata.
   138  	objInfo.ETag = extractETag(fi.Metadata)
   139  
   140  	// Add user tags to the object info
   141  	tags := fi.Metadata[xhttp.AmzObjectTagging]
   142  	if len(tags) != 0 {
   143  		objInfo.UserTags = tags
   144  	}
   145  
   146  	// Add replication status to the object info
   147  	objInfo.ReplicationStatus = replication.StatusType(fi.Metadata[xhttp.AmzBucketReplicationStatus])
   148  	if fi.Deleted {
   149  		objInfo.ReplicationStatus = replication.StatusType(fi.DeleteMarkerReplicationStatus)
   150  	}
   151  
   152  	objInfo.TransitionStatus = fi.TransitionStatus
   153  
   154  	// etag/md5Sum has already been extracted. We need to
   155  	// remove to avoid it from appearing as part of
   156  	// response headers. e.g, X-Minio-* or X-Amz-*.
   157  	// Tags have also been extracted, we remove that as well.
   158  	objInfo.UserDefined = cleanMetadata(fi.Metadata)
   159  
   160  	// All the parts per object.
   161  	objInfo.Parts = fi.Parts
   162  
   163  	// Update storage class
   164  	if sc, ok := fi.Metadata[xhttp.AmzStorageClass]; ok {
   165  		objInfo.StorageClass = sc
   166  	} else {
   167  		objInfo.StorageClass = globalMinioDefaultStorageClass
   168  	}
   169  	objInfo.VersionPurgeStatus = fi.VersionPurgeStatus
   170  	// set restore status for transitioned object
   171  	if ongoing, exp, err := parseRestoreHeaderFromMeta(fi.Metadata); err == nil {
   172  		objInfo.RestoreOngoing = ongoing
   173  		objInfo.RestoreExpires = exp
   174  	}
   175  	// Success.
   176  	return objInfo
   177  }
   178  
   179  // objectPartIndex - returns the index of matching object part number.
   180  func objectPartIndex(parts []ObjectPartInfo, partNumber int) int {
   181  	for i, part := range parts {
   182  		if partNumber == part.Number {
   183  			return i
   184  		}
   185  	}
   186  	return -1
   187  }
   188  
   189  // AddObjectPart - add a new object part in order.
   190  func (fi *FileInfo) AddObjectPart(partNumber int, partETag string, partSize int64, actualSize int64) {
   191  	partInfo := ObjectPartInfo{
   192  		Number:     partNumber,
   193  		ETag:       partETag,
   194  		Size:       partSize,
   195  		ActualSize: actualSize,
   196  	}
   197  
   198  	// Update part info if it already exists.
   199  	for i, part := range fi.Parts {
   200  		if partNumber == part.Number {
   201  			fi.Parts[i] = partInfo
   202  			return
   203  		}
   204  	}
   205  
   206  	// Proceed to include new part info.
   207  	fi.Parts = append(fi.Parts, partInfo)
   208  
   209  	// Parts in FileInfo should be in sorted order by part number.
   210  	sort.Sort(byObjectPartNumber(fi.Parts))
   211  }
   212  
   213  // ObjectToPartOffset - translate offset of an object to offset of its individual part.
   214  func (fi FileInfo) ObjectToPartOffset(ctx context.Context, offset int64) (partIndex int, partOffset int64, err error) {
   215  	if offset == 0 {
   216  		// Special case - if offset is 0, then partIndex and partOffset are always 0.
   217  		return 0, 0, nil
   218  	}
   219  	partOffset = offset
   220  	// Seek until object offset maps to a particular part offset.
   221  	for i, part := range fi.Parts {
   222  		partIndex = i
   223  		// Offset is smaller than size we have reached the proper part offset.
   224  		if partOffset < part.Size {
   225  			return partIndex, partOffset, nil
   226  		}
   227  		// Continue to towards the next part.
   228  		partOffset -= part.Size
   229  	}
   230  	logger.LogIf(ctx, InvalidRange{})
   231  	// Offset beyond the size of the object return InvalidRange.
   232  	return 0, 0, InvalidRange{}
   233  }
   234  
   235  func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time.Time, dataDir string, quorum int) (xmv FileInfo, e error) {
   236  	metaHashes := make([]string, len(metaArr))
   237  	h := sha256.New()
   238  	for i, meta := range metaArr {
   239  		if meta.IsValid() && meta.ModTime.Equal(modTime) && meta.DataDir == dataDir {
   240  			for _, part := range meta.Parts {
   241  				h.Write([]byte(fmt.Sprintf("part.%d", part.Number)))
   242  			}
   243  			h.Write([]byte(fmt.Sprintf("%v", meta.Erasure.Distribution)))
   244  			// make sure that length of Data is same
   245  			h.Write([]byte(fmt.Sprintf("%v", len(meta.Data))))
   246  			metaHashes[i] = hex.EncodeToString(h.Sum(nil))
   247  			h.Reset()
   248  		}
   249  	}
   250  
   251  	metaHashCountMap := make(map[string]int)
   252  	for _, hash := range metaHashes {
   253  		if hash == "" {
   254  			continue
   255  		}
   256  		metaHashCountMap[hash]++
   257  	}
   258  
   259  	maxHash := ""
   260  	maxCount := 0
   261  	for hash, count := range metaHashCountMap {
   262  		if count > maxCount {
   263  			maxCount = count
   264  			maxHash = hash
   265  		}
   266  	}
   267  
   268  	if maxCount < quorum {
   269  		return FileInfo{}, errErasureReadQuorum
   270  	}
   271  
   272  	for i, hash := range metaHashes {
   273  		if hash == maxHash {
   274  			return metaArr[i], nil
   275  		}
   276  	}
   277  
   278  	return FileInfo{}, errErasureReadQuorum
   279  }
   280  
   281  // pickValidFileInfo - picks one valid FileInfo content and returns from a
   282  // slice of FileInfo.
   283  func pickValidFileInfo(ctx context.Context, metaArr []FileInfo, modTime time.Time, dataDir string, quorum int) (xmv FileInfo, e error) {
   284  	return findFileInfoInQuorum(ctx, metaArr, modTime, dataDir, quorum)
   285  }
   286  
   287  // writeUniqueFileInfo - writes unique `xl.meta` content for each disk concurrently.
   288  func writeUniqueFileInfo(ctx context.Context, disks []StorageAPI, bucket, prefix string, files []FileInfo, quorum int) ([]StorageAPI, error) {
   289  	g := errgroup.WithNErrs(len(disks))
   290  
   291  	// Start writing `xl.meta` to all disks in parallel.
   292  	for index := range disks {
   293  		index := index
   294  		g.Go(func() error {
   295  			if disks[index] == nil {
   296  				return errDiskNotFound
   297  			}
   298  			// Pick one FileInfo for a disk at index.
   299  			fi := files[index]
   300  			fi.Erasure.Index = index + 1
   301  			if fi.IsValid() {
   302  				return disks[index].WriteMetadata(ctx, bucket, prefix, fi)
   303  			}
   304  			return errCorruptedFormat
   305  		}, index)
   306  	}
   307  
   308  	// Wait for all the routines.
   309  	mErrs := g.Wait()
   310  
   311  	err := reduceWriteQuorumErrs(ctx, mErrs, objectOpIgnoredErrs, quorum)
   312  	return evalDisks(disks, mErrs), err
   313  }
   314  
   315  // Returns per object readQuorum and writeQuorum
   316  // readQuorum is the min required disks to read data.
   317  // writeQuorum is the min required disks to write data.
   318  func objectQuorumFromMeta(ctx context.Context, partsMetaData []FileInfo, errs []error, defaultParityCount int) (objectReadQuorum, objectWriteQuorum int, err error) {
   319  	// get the latest updated Metadata and a count of all the latest updated FileInfo(s)
   320  	latestFileInfo, err := getLatestFileInfo(ctx, partsMetaData, errs)
   321  	if err != nil {
   322  		return 0, 0, err
   323  	}
   324  
   325  	dataBlocks := latestFileInfo.Erasure.DataBlocks
   326  	parityBlocks := globalStorageClass.GetParityForSC(latestFileInfo.Metadata[xhttp.AmzStorageClass])
   327  	if parityBlocks <= 0 {
   328  		parityBlocks = defaultParityCount
   329  	}
   330  
   331  	writeQuorum := dataBlocks
   332  	if dataBlocks == parityBlocks {
   333  		writeQuorum++
   334  	}
   335  
   336  	// Since all the valid erasure code meta updated at the same time are equivalent, pass dataBlocks
   337  	// from latestFileInfo to get the quorum
   338  	return dataBlocks, writeQuorum, nil
   339  }