storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/erasure-metadata.go (about) 1 /* 2 * MinIO Cloud Storage, (C) 2016-2019 MinIO, Inc. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package cmd 18 19 import ( 20 "context" 21 "crypto/sha256" 22 "encoding/hex" 23 "fmt" 24 "net/http" 25 "sort" 26 "time" 27 28 xhttp "storj.io/minio/cmd/http" 29 "storj.io/minio/cmd/logger" 30 "storj.io/minio/pkg/bucket/replication" 31 "storj.io/minio/pkg/sync/errgroup" 32 ) 33 34 const erasureAlgorithm = "rs-vandermonde" 35 36 // byObjectPartNumber is a collection satisfying sort.Interface. 37 type byObjectPartNumber []ObjectPartInfo 38 39 func (t byObjectPartNumber) Len() int { return len(t) } 40 func (t byObjectPartNumber) Swap(i, j int) { t[i], t[j] = t[j], t[i] } 41 func (t byObjectPartNumber) Less(i, j int) bool { return t[i].Number < t[j].Number } 42 43 // AddChecksumInfo adds a checksum of a part. 44 func (e *ErasureInfo) AddChecksumInfo(ckSumInfo ChecksumInfo) { 45 for i, sum := range e.Checksums { 46 if sum.PartNumber == ckSumInfo.PartNumber { 47 e.Checksums[i] = ckSumInfo 48 return 49 } 50 } 51 e.Checksums = append(e.Checksums, ckSumInfo) 52 } 53 54 // GetChecksumInfo - get checksum of a part. 55 func (e ErasureInfo) GetChecksumInfo(partNumber int) (ckSum ChecksumInfo) { 56 for _, sum := range e.Checksums { 57 if sum.PartNumber == partNumber { 58 // Return the checksum 59 return sum 60 } 61 } 62 return ChecksumInfo{} 63 } 64 65 // ShardFileSize - returns final erasure size from original size. 66 func (e ErasureInfo) ShardFileSize(totalLength int64) int64 { 67 if totalLength == 0 { 68 return 0 69 } 70 if totalLength == -1 { 71 return -1 72 } 73 numShards := totalLength / e.BlockSize 74 lastBlockSize := totalLength % e.BlockSize 75 lastShardSize := ceilFrac(lastBlockSize, int64(e.DataBlocks)) 76 return numShards*e.ShardSize() + lastShardSize 77 } 78 79 // ShardSize - returns actual shared size from erasure blockSize. 80 func (e ErasureInfo) ShardSize() int64 { 81 return ceilFrac(e.BlockSize, int64(e.DataBlocks)) 82 } 83 84 // IsValid - tells if erasure info fields are valid. 85 func (fi FileInfo) IsValid() bool { 86 if fi.Deleted { 87 // Delete marker has no data, no need to check 88 // for erasure coding information 89 return true 90 } 91 dataBlocks := fi.Erasure.DataBlocks 92 parityBlocks := fi.Erasure.ParityBlocks 93 correctIndexes := (fi.Erasure.Index > 0 && 94 fi.Erasure.Index <= dataBlocks+parityBlocks && 95 len(fi.Erasure.Distribution) == (dataBlocks+parityBlocks)) 96 return ((dataBlocks >= parityBlocks) && 97 (dataBlocks != 0) && (parityBlocks != 0) && 98 correctIndexes) 99 } 100 101 // ToObjectInfo - Converts metadata to object info. 102 func (fi FileInfo) ToObjectInfo(bucket, object string) ObjectInfo { 103 object = decodeDirObject(object) 104 versionID := fi.VersionID 105 if (globalBucketVersioningSys.Enabled(bucket) || globalBucketVersioningSys.Suspended(bucket)) && versionID == "" { 106 versionID = nullVersionID 107 } 108 109 objInfo := ObjectInfo{ 110 IsDir: HasSuffix(object, SlashSeparator), 111 Bucket: bucket, 112 Name: object, 113 VersionID: versionID, 114 IsLatest: fi.IsLatest, 115 DeleteMarker: fi.Deleted, 116 Size: fi.Size, 117 ModTime: fi.ModTime, 118 Legacy: fi.XLV1, 119 ContentType: fi.Metadata["content-type"], 120 ContentEncoding: fi.Metadata["content-encoding"], 121 NumVersions: fi.NumVersions, 122 SuccessorModTime: fi.SuccessorModTime, 123 } 124 125 // Update expires 126 var ( 127 t time.Time 128 e error 129 ) 130 if exp, ok := fi.Metadata["expires"]; ok { 131 if t, e = time.Parse(http.TimeFormat, exp); e == nil { 132 objInfo.Expires = t.UTC() 133 } 134 } 135 objInfo.backendType = BackendErasure 136 137 // Extract etag from metadata. 138 objInfo.ETag = extractETag(fi.Metadata) 139 140 // Add user tags to the object info 141 tags := fi.Metadata[xhttp.AmzObjectTagging] 142 if len(tags) != 0 { 143 objInfo.UserTags = tags 144 } 145 146 // Add replication status to the object info 147 objInfo.ReplicationStatus = replication.StatusType(fi.Metadata[xhttp.AmzBucketReplicationStatus]) 148 if fi.Deleted { 149 objInfo.ReplicationStatus = replication.StatusType(fi.DeleteMarkerReplicationStatus) 150 } 151 152 objInfo.TransitionStatus = fi.TransitionStatus 153 154 // etag/md5Sum has already been extracted. We need to 155 // remove to avoid it from appearing as part of 156 // response headers. e.g, X-Minio-* or X-Amz-*. 157 // Tags have also been extracted, we remove that as well. 158 objInfo.UserDefined = cleanMetadata(fi.Metadata) 159 160 // All the parts per object. 161 objInfo.Parts = fi.Parts 162 163 // Update storage class 164 if sc, ok := fi.Metadata[xhttp.AmzStorageClass]; ok { 165 objInfo.StorageClass = sc 166 } else { 167 objInfo.StorageClass = globalMinioDefaultStorageClass 168 } 169 objInfo.VersionPurgeStatus = fi.VersionPurgeStatus 170 // set restore status for transitioned object 171 if ongoing, exp, err := parseRestoreHeaderFromMeta(fi.Metadata); err == nil { 172 objInfo.RestoreOngoing = ongoing 173 objInfo.RestoreExpires = exp 174 } 175 // Success. 176 return objInfo 177 } 178 179 // objectPartIndex - returns the index of matching object part number. 180 func objectPartIndex(parts []ObjectPartInfo, partNumber int) int { 181 for i, part := range parts { 182 if partNumber == part.Number { 183 return i 184 } 185 } 186 return -1 187 } 188 189 // AddObjectPart - add a new object part in order. 190 func (fi *FileInfo) AddObjectPart(partNumber int, partETag string, partSize int64, actualSize int64) { 191 partInfo := ObjectPartInfo{ 192 Number: partNumber, 193 ETag: partETag, 194 Size: partSize, 195 ActualSize: actualSize, 196 } 197 198 // Update part info if it already exists. 199 for i, part := range fi.Parts { 200 if partNumber == part.Number { 201 fi.Parts[i] = partInfo 202 return 203 } 204 } 205 206 // Proceed to include new part info. 207 fi.Parts = append(fi.Parts, partInfo) 208 209 // Parts in FileInfo should be in sorted order by part number. 210 sort.Sort(byObjectPartNumber(fi.Parts)) 211 } 212 213 // ObjectToPartOffset - translate offset of an object to offset of its individual part. 214 func (fi FileInfo) ObjectToPartOffset(ctx context.Context, offset int64) (partIndex int, partOffset int64, err error) { 215 if offset == 0 { 216 // Special case - if offset is 0, then partIndex and partOffset are always 0. 217 return 0, 0, nil 218 } 219 partOffset = offset 220 // Seek until object offset maps to a particular part offset. 221 for i, part := range fi.Parts { 222 partIndex = i 223 // Offset is smaller than size we have reached the proper part offset. 224 if partOffset < part.Size { 225 return partIndex, partOffset, nil 226 } 227 // Continue to towards the next part. 228 partOffset -= part.Size 229 } 230 logger.LogIf(ctx, InvalidRange{}) 231 // Offset beyond the size of the object return InvalidRange. 232 return 0, 0, InvalidRange{} 233 } 234 235 func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time.Time, dataDir string, quorum int) (xmv FileInfo, e error) { 236 metaHashes := make([]string, len(metaArr)) 237 h := sha256.New() 238 for i, meta := range metaArr { 239 if meta.IsValid() && meta.ModTime.Equal(modTime) && meta.DataDir == dataDir { 240 for _, part := range meta.Parts { 241 h.Write([]byte(fmt.Sprintf("part.%d", part.Number))) 242 } 243 h.Write([]byte(fmt.Sprintf("%v", meta.Erasure.Distribution))) 244 // make sure that length of Data is same 245 h.Write([]byte(fmt.Sprintf("%v", len(meta.Data)))) 246 metaHashes[i] = hex.EncodeToString(h.Sum(nil)) 247 h.Reset() 248 } 249 } 250 251 metaHashCountMap := make(map[string]int) 252 for _, hash := range metaHashes { 253 if hash == "" { 254 continue 255 } 256 metaHashCountMap[hash]++ 257 } 258 259 maxHash := "" 260 maxCount := 0 261 for hash, count := range metaHashCountMap { 262 if count > maxCount { 263 maxCount = count 264 maxHash = hash 265 } 266 } 267 268 if maxCount < quorum { 269 return FileInfo{}, errErasureReadQuorum 270 } 271 272 for i, hash := range metaHashes { 273 if hash == maxHash { 274 return metaArr[i], nil 275 } 276 } 277 278 return FileInfo{}, errErasureReadQuorum 279 } 280 281 // pickValidFileInfo - picks one valid FileInfo content and returns from a 282 // slice of FileInfo. 283 func pickValidFileInfo(ctx context.Context, metaArr []FileInfo, modTime time.Time, dataDir string, quorum int) (xmv FileInfo, e error) { 284 return findFileInfoInQuorum(ctx, metaArr, modTime, dataDir, quorum) 285 } 286 287 // writeUniqueFileInfo - writes unique `xl.meta` content for each disk concurrently. 288 func writeUniqueFileInfo(ctx context.Context, disks []StorageAPI, bucket, prefix string, files []FileInfo, quorum int) ([]StorageAPI, error) { 289 g := errgroup.WithNErrs(len(disks)) 290 291 // Start writing `xl.meta` to all disks in parallel. 292 for index := range disks { 293 index := index 294 g.Go(func() error { 295 if disks[index] == nil { 296 return errDiskNotFound 297 } 298 // Pick one FileInfo for a disk at index. 299 fi := files[index] 300 fi.Erasure.Index = index + 1 301 if fi.IsValid() { 302 return disks[index].WriteMetadata(ctx, bucket, prefix, fi) 303 } 304 return errCorruptedFormat 305 }, index) 306 } 307 308 // Wait for all the routines. 309 mErrs := g.Wait() 310 311 err := reduceWriteQuorumErrs(ctx, mErrs, objectOpIgnoredErrs, quorum) 312 return evalDisks(disks, mErrs), err 313 } 314 315 // Returns per object readQuorum and writeQuorum 316 // readQuorum is the min required disks to read data. 317 // writeQuorum is the min required disks to write data. 318 func objectQuorumFromMeta(ctx context.Context, partsMetaData []FileInfo, errs []error, defaultParityCount int) (objectReadQuorum, objectWriteQuorum int, err error) { 319 // get the latest updated Metadata and a count of all the latest updated FileInfo(s) 320 latestFileInfo, err := getLatestFileInfo(ctx, partsMetaData, errs) 321 if err != nil { 322 return 0, 0, err 323 } 324 325 dataBlocks := latestFileInfo.Erasure.DataBlocks 326 parityBlocks := globalStorageClass.GetParityForSC(latestFileInfo.Metadata[xhttp.AmzStorageClass]) 327 if parityBlocks <= 0 { 328 parityBlocks = defaultParityCount 329 } 330 331 writeQuorum := dataBlocks 332 if dataBlocks == parityBlocks { 333 writeQuorum++ 334 } 335 336 // Since all the valid erasure code meta updated at the same time are equivalent, pass dataBlocks 337 // from latestFileInfo to get the quorum 338 return dataBlocks, writeQuorum, nil 339 }