storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/xl-storage-format-v2.go (about) 1 /* 2 * MinIO Cloud Storage, (C) 2020 MinIO, Inc. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package cmd 18 19 import ( 20 "bytes" 21 "encoding/binary" 22 "errors" 23 "fmt" 24 "sort" 25 "strings" 26 "time" 27 28 "github.com/cespare/xxhash/v2" 29 "github.com/google/uuid" 30 "github.com/tinylib/msgp/msgp" 31 32 xhttp "storj.io/minio/cmd/http" 33 "storj.io/minio/cmd/logger" 34 ) 35 36 var ( 37 // XL header specifies the format 38 xlHeader = [4]byte{'X', 'L', '2', ' '} 39 40 // Current version being written. 41 xlVersionCurrent [4]byte 42 ) 43 44 const ( 45 // Breaking changes. 46 // Newer versions cannot be read by older software. 47 // This will prevent downgrades to incompatible versions. 48 xlVersionMajor = 1 49 50 // Non breaking changes. 51 // Bumping this is informational, but should be done 52 // if any change is made to the data stored, bumping this 53 // will allow to detect the exact version later. 54 xlVersionMinor = 2 55 ) 56 57 func init() { 58 binary.LittleEndian.PutUint16(xlVersionCurrent[0:2], xlVersionMajor) 59 binary.LittleEndian.PutUint16(xlVersionCurrent[2:4], xlVersionMinor) 60 } 61 62 // checkXL2V1 will check if the metadata has correct header and is a known major version. 63 // The remaining payload and versions are returned. 64 func checkXL2V1(buf []byte) (payload []byte, major, minor uint16, err error) { 65 if len(buf) <= 8 { 66 return payload, 0, 0, fmt.Errorf("xlMeta: no data") 67 } 68 69 if !bytes.Equal(buf[:4], xlHeader[:]) { 70 return payload, 0, 0, fmt.Errorf("xlMeta: unknown XLv2 header, expected %v, got %v", xlHeader[:4], buf[:4]) 71 } 72 73 if bytes.Equal(buf[4:8], []byte("1 ")) { 74 // Set as 1,0. 75 major, minor = 1, 0 76 } else { 77 major, minor = binary.LittleEndian.Uint16(buf[4:6]), binary.LittleEndian.Uint16(buf[6:8]) 78 } 79 if major > xlVersionMajor { 80 return buf[8:], major, minor, fmt.Errorf("xlMeta: unknown major version %d found", major) 81 } 82 83 return buf[8:], major, minor, nil 84 } 85 86 func isXL2V1Format(buf []byte) bool { 87 _, _, _, err := checkXL2V1(buf) 88 return err == nil 89 } 90 91 // The []journal contains all the different versions of the object. 92 // 93 // This array can have 3 kinds of objects: 94 // 95 // ``object``: If the object is uploaded the usual way: putobject, multipart-put, copyobject 96 // 97 // ``delete``: This is the delete-marker 98 // 99 // ``legacyObject``: This is the legacy object in xlV1 format, preserved until its overwritten 100 // 101 // The most recently updated element in the array is considered the latest version. 102 103 // Backend directory tree structure: 104 // disk1/ 105 // └── bucket 106 // └── object 107 // ├── a192c1d5-9bd5-41fd-9a90-ab10e165398d 108 // │ └── part.1 109 // ├── c06e0436-f813-447e-ae5e-f2564df9dfd4 110 // │ └── part.1 111 // ├── df433928-2dcf-47b1-a786-43efa0f6b424 112 // │ └── part.1 113 // ├── legacy 114 // │ └── part.1 115 // └── xl.meta 116 117 //go:generate msgp -file=$GOFILE -unexported 118 119 // VersionType defines the type of journal type of the current entry. 120 type VersionType uint8 121 122 // List of different types of journal type 123 const ( 124 invalidVersionType VersionType = 0 125 ObjectType VersionType = 1 126 DeleteType VersionType = 2 127 LegacyType VersionType = 3 128 lastVersionType VersionType = 4 129 ) 130 131 func (e VersionType) valid() bool { 132 return e > invalidVersionType && e < lastVersionType 133 } 134 135 // ErasureAlgo defines common type of different erasure algorithms 136 type ErasureAlgo uint8 137 138 // List of currently supported erasure coding algorithms 139 const ( 140 invalidErasureAlgo ErasureAlgo = 0 141 ReedSolomon ErasureAlgo = 1 142 lastErasureAlgo ErasureAlgo = 2 143 ) 144 145 func (e ErasureAlgo) valid() bool { 146 return e > invalidErasureAlgo && e < lastErasureAlgo 147 } 148 149 func (e ErasureAlgo) String() string { 150 switch e { 151 case ReedSolomon: 152 return "reedsolomon" 153 } 154 return "" 155 } 156 157 // ChecksumAlgo defines common type of different checksum algorithms 158 type ChecksumAlgo uint8 159 160 // List of currently supported checksum algorithms 161 const ( 162 invalidChecksumAlgo ChecksumAlgo = 0 163 HighwayHash ChecksumAlgo = 1 164 lastChecksumAlgo ChecksumAlgo = 2 165 ) 166 167 func (e ChecksumAlgo) valid() bool { 168 return e > invalidChecksumAlgo && e < lastChecksumAlgo 169 } 170 171 // xlMetaV2DeleteMarker defines the data struct for the delete marker journal type 172 type xlMetaV2DeleteMarker struct { 173 VersionID [16]byte `json:"ID" msg:"ID"` // Version ID for delete marker 174 ModTime int64 `json:"MTime" msg:"MTime"` // Object delete marker modified time 175 MetaSys map[string][]byte `json:"MetaSys,omitempty" msg:"MetaSys,omitempty"` // Delete marker internal metadata 176 } 177 178 // xlMetaV2Object defines the data struct for object journal type 179 type xlMetaV2Object struct { 180 VersionID [16]byte `json:"ID" msg:"ID"` // Version ID 181 DataDir [16]byte `json:"DDir" msg:"DDir"` // Data dir ID 182 ErasureAlgorithm ErasureAlgo `json:"EcAlgo" msg:"EcAlgo"` // Erasure coding algorithm 183 ErasureM int `json:"EcM" msg:"EcM"` // Erasure data blocks 184 ErasureN int `json:"EcN" msg:"EcN"` // Erasure parity blocks 185 ErasureBlockSize int64 `json:"EcBSize" msg:"EcBSize"` // Erasure block size 186 ErasureIndex int `json:"EcIndex" msg:"EcIndex"` // Erasure disk index 187 ErasureDist []uint8 `json:"EcDist" msg:"EcDist"` // Erasure distribution 188 BitrotChecksumAlgo ChecksumAlgo `json:"CSumAlgo" msg:"CSumAlgo"` // Bitrot checksum algo 189 PartNumbers []int `json:"PartNums" msg:"PartNums"` // Part Numbers 190 PartETags []string `json:"PartETags" msg:"PartETags"` // Part ETags 191 PartSizes []int64 `json:"PartSizes" msg:"PartSizes"` // Part Sizes 192 PartActualSizes []int64 `json:"PartASizes,omitempty" msg:"PartASizes,omitempty"` // Part ActualSizes (compression) 193 Size int64 `json:"Size" msg:"Size"` // Object version size 194 ModTime int64 `json:"MTime" msg:"MTime"` // Object version modified time 195 MetaSys map[string][]byte `json:"MetaSys,omitempty" msg:"MetaSys,omitempty"` // Object version internal metadata 196 MetaUser map[string]string `json:"MetaUsr,omitempty" msg:"MetaUsr,omitempty"` // Object version metadata set by user 197 } 198 199 // xlMetaV2Version describes the jouranal entry, Type defines 200 // the current journal entry type other types might be nil based 201 // on what Type field carries, it is imperative for the caller 202 // to verify which journal type first before accessing rest of the fields. 203 type xlMetaV2Version struct { 204 Type VersionType `json:"Type" msg:"Type"` 205 ObjectV1 *xlMetaV1Object `json:"V1Obj,omitempty" msg:"V1Obj,omitempty"` 206 ObjectV2 *xlMetaV2Object `json:"V2Obj,omitempty" msg:"V2Obj,omitempty"` 207 DeleteMarker *xlMetaV2DeleteMarker `json:"DelObj,omitempty" msg:"DelObj,omitempty"` 208 } 209 210 // Valid xl meta xlMetaV2Version is valid 211 func (j xlMetaV2Version) Valid() bool { 212 switch j.Type { 213 case LegacyType: 214 return j.ObjectV1 != nil && 215 j.ObjectV1.valid() 216 case ObjectType: 217 return j.ObjectV2 != nil && 218 j.ObjectV2.ErasureAlgorithm.valid() && 219 j.ObjectV2.BitrotChecksumAlgo.valid() && 220 isXLMetaErasureInfoValid(j.ObjectV2.ErasureM, j.ObjectV2.ErasureN) && 221 j.ObjectV2.ModTime > 0 222 case DeleteType: 223 return j.DeleteMarker != nil && 224 j.DeleteMarker.ModTime > 0 225 } 226 return false 227 } 228 229 // xlMetaV2 - object meta structure defines the format and list of 230 // the journals for the object. 231 type xlMetaV2 struct { 232 Versions []xlMetaV2Version `json:"Versions" msg:"Versions"` 233 234 // data will contain raw data if any. 235 // data will be one or more versions indexed by versionID. 236 // To remove all data set to nil. 237 data xlMetaInlineData `msg:"-"` 238 } 239 240 // xlMetaInlineData is serialized data in [string][]byte pairs. 241 // 242 //msgp:ignore xlMetaInlineData 243 type xlMetaInlineData []byte 244 245 // xlMetaInlineDataVer indicates the vesrion of the inline data structure. 246 const xlMetaInlineDataVer = 1 247 248 // versionOK returns whether the version is ok. 249 func (x xlMetaInlineData) versionOK() bool { 250 if len(x) == 0 { 251 return true 252 } 253 return x[0] > 0 && x[0] <= xlMetaInlineDataVer 254 } 255 256 // afterVersion returns the payload after the version, if any. 257 func (x xlMetaInlineData) afterVersion() []byte { 258 if len(x) == 0 { 259 return x 260 } 261 return x[1:] 262 } 263 264 // find the data with key s. 265 // Returns nil if not for or an error occurs. 266 func (x xlMetaInlineData) find(key string) []byte { 267 if len(x) == 0 || !x.versionOK() { 268 return nil 269 } 270 sz, buf, err := msgp.ReadMapHeaderBytes(x.afterVersion()) 271 if err != nil || sz == 0 { 272 return nil 273 } 274 for i := uint32(0); i < sz; i++ { 275 var found []byte 276 found, buf, err = msgp.ReadMapKeyZC(buf) 277 if err != nil || sz == 0 { 278 return nil 279 } 280 if string(found) == key { 281 val, _, _ := msgp.ReadBytesZC(buf) 282 return val 283 } 284 // Skip it 285 _, buf, err = msgp.ReadBytesZC(buf) 286 if err != nil { 287 return nil 288 } 289 } 290 return nil 291 } 292 293 // validate checks if the data is valid. 294 // It does not check integrity of the stored data. 295 func (x xlMetaInlineData) validate() error { 296 if len(x) == 0 { 297 return nil 298 } 299 300 if !x.versionOK() { 301 return fmt.Errorf("xlMetaInlineData: unknown version 0x%x", x[0]) 302 } 303 304 sz, buf, err := msgp.ReadMapHeaderBytes(x.afterVersion()) 305 if err != nil { 306 return fmt.Errorf("xlMetaInlineData: %w", err) 307 } 308 309 for i := uint32(0); i < sz; i++ { 310 var key []byte 311 key, buf, err = msgp.ReadMapKeyZC(buf) 312 if err != nil { 313 return fmt.Errorf("xlMetaInlineData: %w", err) 314 } 315 if len(key) == 0 { 316 return fmt.Errorf("xlMetaInlineData: key %d is length 0", i) 317 } 318 _, buf, err = msgp.ReadBytesZC(buf) 319 if err != nil { 320 return fmt.Errorf("xlMetaInlineData: %w", err) 321 } 322 } 323 324 return nil 325 } 326 327 // repair will copy all seemingly valid data entries from a corrupted set. 328 // This does not ensure that data is correct, but will allow all operations to complete. 329 func (x *xlMetaInlineData) repair() { 330 data := *x 331 if len(data) == 0 { 332 return 333 } 334 335 if !data.versionOK() { 336 *x = nil 337 return 338 } 339 340 sz, buf, err := msgp.ReadMapHeaderBytes(data.afterVersion()) 341 if err != nil { 342 *x = nil 343 return 344 } 345 346 // Remove all current data 347 keys := make([][]byte, 0, sz) 348 vals := make([][]byte, 0, sz) 349 for i := uint32(0); i < sz; i++ { 350 var key, val []byte 351 key, buf, err = msgp.ReadMapKeyZC(buf) 352 if err != nil { 353 break 354 } 355 if len(key) == 0 { 356 break 357 } 358 val, buf, err = msgp.ReadBytesZC(buf) 359 if err != nil { 360 break 361 } 362 keys = append(keys, key) 363 vals = append(vals, val) 364 } 365 x.serialize(-1, keys, vals) 366 } 367 368 // validate checks if the data is valid. 369 // It does not check integrity of the stored data. 370 func (x xlMetaInlineData) list() ([]string, error) { 371 if len(x) == 0 { 372 return nil, nil 373 } 374 if !x.versionOK() { 375 return nil, errors.New("xlMetaInlineData: unknown version") 376 } 377 378 sz, buf, err := msgp.ReadMapHeaderBytes(x.afterVersion()) 379 if err != nil { 380 return nil, err 381 } 382 keys := make([]string, 0, sz) 383 for i := uint32(0); i < sz; i++ { 384 var key []byte 385 key, buf, err = msgp.ReadMapKeyZC(buf) 386 if err != nil { 387 return keys, err 388 } 389 if len(key) == 0 { 390 return keys, fmt.Errorf("xlMetaInlineData: key %d is length 0", i) 391 } 392 keys = append(keys, string(key)) 393 // Skip data... 394 _, buf, err = msgp.ReadBytesZC(buf) 395 if err != nil { 396 return keys, err 397 } 398 } 399 return keys, nil 400 } 401 402 // serialize will serialize the provided keys and values. 403 // The function will panic if keys/value slices aren't of equal length. 404 // Payload size can give an indication of expected payload size. 405 // If plSize is <= 0 it will be calculated. 406 func (x *xlMetaInlineData) serialize(plSize int, keys [][]byte, vals [][]byte) { 407 if len(keys) != len(vals) { 408 panic(fmt.Errorf("xlMetaInlineData.serialize: keys/value number mismatch")) 409 } 410 if len(keys) == 0 { 411 *x = nil 412 return 413 } 414 if plSize <= 0 { 415 plSize = 1 + msgp.MapHeaderSize 416 for i := range keys { 417 plSize += len(keys[i]) + len(vals[i]) + msgp.StringPrefixSize + msgp.ArrayHeaderSize 418 } 419 } 420 payload := make([]byte, 1, plSize) 421 payload[0] = xlMetaInlineDataVer 422 payload = msgp.AppendMapHeader(payload, uint32(len(keys))) 423 for i := range keys { 424 payload = msgp.AppendStringFromBytes(payload, keys[i]) 425 payload = msgp.AppendBytes(payload, vals[i]) 426 } 427 *x = payload 428 } 429 430 // entries returns the number of entries in the data. 431 func (x xlMetaInlineData) entries() int { 432 if len(x) == 0 || !x.versionOK() { 433 return 0 434 } 435 sz, _, _ := msgp.ReadMapHeaderBytes(x.afterVersion()) 436 return int(sz) 437 } 438 439 // replace will add or replace a key/value pair. 440 func (x *xlMetaInlineData) replace(key string, value []byte) { 441 in := x.afterVersion() 442 sz, buf, _ := msgp.ReadMapHeaderBytes(in) 443 keys := make([][]byte, 0, sz+1) 444 vals := make([][]byte, 0, sz+1) 445 446 // Version plus header... 447 plSize := 1 + msgp.MapHeaderSize 448 replaced := false 449 for i := uint32(0); i < sz; i++ { 450 var found, foundVal []byte 451 var err error 452 found, buf, err = msgp.ReadMapKeyZC(buf) 453 if err != nil { 454 break 455 } 456 foundVal, buf, err = msgp.ReadBytesZC(buf) 457 if err != nil { 458 break 459 } 460 plSize += len(found) + msgp.StringPrefixSize + msgp.ArrayHeaderSize 461 keys = append(keys, found) 462 if string(found) == key { 463 vals = append(vals, value) 464 plSize += len(value) 465 replaced = true 466 } else { 467 vals = append(vals, foundVal) 468 plSize += len(foundVal) 469 } 470 } 471 472 // Add one more. 473 if !replaced { 474 keys = append(keys, []byte(key)) 475 vals = append(vals, value) 476 plSize += len(key) + len(value) + msgp.StringPrefixSize + msgp.ArrayHeaderSize 477 } 478 479 // Reserialize... 480 x.serialize(plSize, keys, vals) 481 } 482 483 // rename will rename a key. 484 // Returns whether the key was found. 485 func (x *xlMetaInlineData) rename(oldKey, newKey string) bool { 486 in := x.afterVersion() 487 sz, buf, _ := msgp.ReadMapHeaderBytes(in) 488 keys := make([][]byte, 0, sz) 489 vals := make([][]byte, 0, sz) 490 491 // Version plus header... 492 plSize := 1 + msgp.MapHeaderSize 493 found := false 494 for i := uint32(0); i < sz; i++ { 495 var foundKey, foundVal []byte 496 var err error 497 foundKey, buf, err = msgp.ReadMapKeyZC(buf) 498 if err != nil { 499 break 500 } 501 foundVal, buf, err = msgp.ReadBytesZC(buf) 502 if err != nil { 503 break 504 } 505 plSize += len(foundVal) + msgp.StringPrefixSize + msgp.ArrayHeaderSize 506 vals = append(vals, foundVal) 507 if string(foundKey) != oldKey { 508 keys = append(keys, foundKey) 509 plSize += len(foundKey) 510 } else { 511 keys = append(keys, []byte(newKey)) 512 plSize += len(newKey) 513 found = true 514 } 515 } 516 // If not found, just return. 517 if !found { 518 return false 519 } 520 521 // Reserialize... 522 x.serialize(plSize, keys, vals) 523 return true 524 } 525 526 // remove will remove a key. 527 // Returns whether the key was found. 528 func (x *xlMetaInlineData) remove(key string) bool { 529 in := x.afterVersion() 530 sz, buf, _ := msgp.ReadMapHeaderBytes(in) 531 keys := make([][]byte, 0, sz) 532 vals := make([][]byte, 0, sz) 533 534 // Version plus header... 535 plSize := 1 + msgp.MapHeaderSize 536 found := false 537 for i := uint32(0); i < sz; i++ { 538 var foundKey, foundVal []byte 539 var err error 540 foundKey, buf, err = msgp.ReadMapKeyZC(buf) 541 if err != nil { 542 break 543 } 544 foundVal, buf, err = msgp.ReadBytesZC(buf) 545 if err != nil { 546 break 547 } 548 if string(foundKey) != key { 549 plSize += msgp.StringPrefixSize + msgp.ArrayHeaderSize + len(foundKey) + len(foundVal) 550 keys = append(keys, foundKey) 551 vals = append(vals, foundVal) 552 } else { 553 found = true 554 } 555 } 556 // If not found, just return. 557 if !found { 558 return false 559 } 560 // If none left... 561 if len(keys) == 0 { 562 *x = nil 563 return true 564 } 565 566 // Reserialize... 567 x.serialize(plSize, keys, vals) 568 return true 569 } 570 571 // xlMetaV2TrimData will trim any data from the metadata without unmarshalling it. 572 // If any error occurs the unmodified data is returned. 573 func xlMetaV2TrimData(buf []byte) []byte { 574 metaBuf, min, maj, err := checkXL2V1(buf) 575 if err != nil { 576 return buf 577 } 578 if maj == 1 && min < 1 { 579 // First version to carry data. 580 return buf 581 } 582 // Skip header 583 _, metaBuf, err = msgp.ReadBytesZC(metaBuf) 584 if err != nil { 585 logger.LogIf(GlobalContext, err) 586 return buf 587 } 588 // Skip CRC 589 if maj > 1 || min >= 2 { 590 _, metaBuf, err = msgp.ReadUint32Bytes(metaBuf) 591 logger.LogIf(GlobalContext, err) 592 } 593 // = input - current pos 594 ends := len(buf) - len(metaBuf) 595 if ends > len(buf) { 596 return buf 597 } 598 599 return buf[:ends] 600 } 601 602 // AddLegacy adds a legacy version, is only called when no prior 603 // versions exist, safe to use it by only one function in xl-storage(RenameData) 604 func (z *xlMetaV2) AddLegacy(m *xlMetaV1Object) error { 605 if !m.valid() { 606 return errFileCorrupt 607 } 608 m.VersionID = nullVersionID 609 m.DataDir = legacyDataDir 610 z.Versions = []xlMetaV2Version{ 611 { 612 Type: LegacyType, 613 ObjectV1: m, 614 }, 615 } 616 return nil 617 } 618 619 // Load unmarshal and load the entire message pack. 620 // Note that references to the incoming buffer may be kept as data. 621 func (z *xlMetaV2) Load(buf []byte) error { 622 buf, major, minor, err := checkXL2V1(buf) 623 if err != nil { 624 return fmt.Errorf("xlMetaV2.Load %w", err) 625 } 626 switch major { 627 case 1: 628 switch minor { 629 case 0: 630 _, err = z.UnmarshalMsg(buf) 631 if err != nil { 632 return fmt.Errorf("xlMetaV2.Load %w", err) 633 } 634 return nil 635 case 1, 2: 636 v, buf, err := msgp.ReadBytesZC(buf) 637 if err != nil { 638 return fmt.Errorf("xlMetaV2.Load version(%d), bufLen(%d) %w", minor, len(buf), err) 639 } 640 if minor >= 2 { 641 if crc, nbuf, err := msgp.ReadUint32Bytes(buf); err == nil { 642 // Read metadata CRC (added in v2) 643 buf = nbuf 644 if got := uint32(xxhash.Sum64(v)); got != crc { 645 return fmt.Errorf("xlMetaV2.Load version(%d), CRC mismatch, want 0x%x, got 0x%x", minor, crc, got) 646 } 647 } else { 648 return fmt.Errorf("xlMetaV2.Load version(%d), loading CRC: %w", minor, err) 649 } 650 } 651 652 if _, err = z.UnmarshalMsg(v); err != nil { 653 return fmt.Errorf("xlMetaV2.Load version(%d), vLen(%d), %w", minor, len(v), err) 654 } 655 // Add remaining data. 656 z.data = buf 657 if err = z.data.validate(); err != nil { 658 z.data.repair() 659 logger.Info("xlMetaV2.Load: data validation failed: %v. %d entries after repair", err, z.data.entries()) 660 } 661 default: 662 return errors.New("unknown minor metadata version") 663 } 664 default: 665 return errors.New("unknown major metadata version") 666 } 667 return nil 668 } 669 670 // AppendTo will marshal the data in z and append it to the provided slice. 671 func (z *xlMetaV2) AppendTo(dst []byte) ([]byte, error) { 672 sz := len(xlHeader) + len(xlVersionCurrent) + msgp.ArrayHeaderSize + z.Msgsize() + len(z.data) + len(dst) + msgp.Uint32Size 673 if cap(dst) < sz { 674 buf := make([]byte, len(dst), sz) 675 copy(buf, dst) 676 dst = buf 677 } 678 if err := z.data.validate(); err != nil { 679 return nil, err 680 } 681 682 dst = append(dst, xlHeader[:]...) 683 dst = append(dst, xlVersionCurrent[:]...) 684 // Add "bin 32" type header to always have enough space. 685 // We will fill out the correct size when we know it. 686 dst = append(dst, 0xc6, 0, 0, 0, 0) 687 dataOffset := len(dst) 688 dst, err := z.MarshalMsg(dst) 689 if err != nil { 690 return nil, err 691 } 692 693 // Update size... 694 binary.BigEndian.PutUint32(dst[dataOffset-4:dataOffset], uint32(len(dst)-dataOffset)) 695 696 // Add CRC of metadata. 697 dst = msgp.AppendUint32(dst, uint32(xxhash.Sum64(dst[dataOffset:]))) 698 return append(dst, z.data...), nil 699 } 700 701 // UpdateObjectVersion updates metadata and modTime for a given 702 // versionID, NOTE: versionID must be valid and should exist - 703 // and must not be a DeleteMarker or legacy object, if no 704 // versionID is specified 'null' versionID is updated instead. 705 // 706 // It is callers responsibility to set correct versionID, this 707 // function shouldn't be further extended to update immutable 708 // values such as ErasureInfo, ChecksumInfo. 709 // 710 // Metadata is only updated to new values, existing values 711 // stay as is, if you wish to update all values you should 712 // update all metadata freshly before calling this function 713 // in-case you wish to clear existing metadata. 714 func (z *xlMetaV2) UpdateObjectVersion(fi FileInfo) error { 715 if fi.VersionID == "" { 716 // this means versioning is not yet 717 // enabled or suspend i.e all versions 718 // are basically default value i.e "null" 719 fi.VersionID = nullVersionID 720 } 721 722 var uv uuid.UUID 723 var err error 724 if fi.VersionID != "" && fi.VersionID != nullVersionID { 725 uv, err = uuid.Parse(fi.VersionID) 726 if err != nil { 727 return err 728 } 729 } 730 731 for i, version := range z.Versions { 732 if !version.Valid() { 733 return errFileCorrupt 734 } 735 switch version.Type { 736 case LegacyType: 737 if version.ObjectV1.VersionID == fi.VersionID { 738 return errMethodNotAllowed 739 } 740 case ObjectType: 741 if version.ObjectV2.VersionID == uv { 742 for k, v := range fi.Metadata { 743 if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) { 744 z.Versions[i].ObjectV2.MetaSys[k] = []byte(v) 745 } else { 746 z.Versions[i].ObjectV2.MetaUser[k] = v 747 } 748 } 749 if !fi.ModTime.IsZero() { 750 z.Versions[i].ObjectV2.ModTime = fi.ModTime.UnixNano() 751 } 752 return nil 753 } 754 case DeleteType: 755 return errMethodNotAllowed 756 } 757 } 758 759 return errFileVersionNotFound 760 } 761 762 // AddVersion adds a new version 763 func (z *xlMetaV2) AddVersion(fi FileInfo) error { 764 if fi.VersionID == "" { 765 // this means versioning is not yet 766 // enabled or suspend i.e all versions 767 // are basically default value i.e "null" 768 fi.VersionID = nullVersionID 769 } 770 771 var uv uuid.UUID 772 var err error 773 if fi.VersionID != "" && fi.VersionID != nullVersionID { 774 uv, err = uuid.Parse(fi.VersionID) 775 if err != nil { 776 return err 777 } 778 } 779 780 var dd uuid.UUID 781 if fi.DataDir != "" { 782 dd, err = uuid.Parse(fi.DataDir) 783 if err != nil { 784 return err 785 } 786 } 787 788 ventry := xlMetaV2Version{} 789 790 if fi.Deleted { 791 ventry.Type = DeleteType 792 ventry.DeleteMarker = &xlMetaV2DeleteMarker{ 793 VersionID: uv, 794 ModTime: fi.ModTime.UnixNano(), 795 MetaSys: make(map[string][]byte), 796 } 797 } else { 798 ventry.Type = ObjectType 799 ventry.ObjectV2 = &xlMetaV2Object{ 800 VersionID: uv, 801 DataDir: dd, 802 Size: fi.Size, 803 ModTime: fi.ModTime.UnixNano(), 804 ErasureAlgorithm: ReedSolomon, 805 ErasureM: fi.Erasure.DataBlocks, 806 ErasureN: fi.Erasure.ParityBlocks, 807 ErasureBlockSize: fi.Erasure.BlockSize, 808 ErasureIndex: fi.Erasure.Index, 809 BitrotChecksumAlgo: HighwayHash, 810 ErasureDist: make([]uint8, len(fi.Erasure.Distribution)), 811 PartNumbers: make([]int, len(fi.Parts)), 812 PartETags: make([]string, len(fi.Parts)), 813 PartSizes: make([]int64, len(fi.Parts)), 814 PartActualSizes: make([]int64, len(fi.Parts)), 815 MetaSys: make(map[string][]byte), 816 MetaUser: make(map[string]string, len(fi.Metadata)), 817 } 818 819 for i := range fi.Erasure.Distribution { 820 ventry.ObjectV2.ErasureDist[i] = uint8(fi.Erasure.Distribution[i]) 821 } 822 823 for i := range fi.Parts { 824 ventry.ObjectV2.PartSizes[i] = fi.Parts[i].Size 825 if fi.Parts[i].ETag != "" { 826 ventry.ObjectV2.PartETags[i] = fi.Parts[i].ETag 827 } 828 ventry.ObjectV2.PartNumbers[i] = fi.Parts[i].Number 829 ventry.ObjectV2.PartActualSizes[i] = fi.Parts[i].ActualSize 830 } 831 832 for k, v := range fi.Metadata { 833 if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) { 834 ventry.ObjectV2.MetaSys[k] = []byte(v) 835 } else { 836 ventry.ObjectV2.MetaUser[k] = v 837 } 838 } 839 840 // If asked to save data. 841 if len(fi.Data) > 0 || fi.Size == 0 { 842 z.data.replace(fi.VersionID, fi.Data) 843 } 844 } 845 846 if !ventry.Valid() { 847 return errors.New("internal error: invalid version entry generated") 848 } 849 850 for i, version := range z.Versions { 851 if !version.Valid() { 852 return errFileCorrupt 853 } 854 switch version.Type { 855 case LegacyType: 856 // This would convert legacy type into new ObjectType 857 // this means that we are basically purging the `null` 858 // version of the object. 859 if version.ObjectV1.VersionID == fi.VersionID { 860 z.Versions[i] = ventry 861 return nil 862 } 863 case ObjectType: 864 if version.ObjectV2.VersionID == uv { 865 z.Versions[i] = ventry 866 return nil 867 } 868 case DeleteType: 869 // Allowing delete marker to replaced with an proper 870 // object data type as well, this is not S3 complaint 871 // behavior but kept here for future flexibility. 872 if version.DeleteMarker.VersionID == uv { 873 z.Versions[i] = ventry 874 return nil 875 } 876 } 877 } 878 879 z.Versions = append(z.Versions, ventry) 880 return nil 881 } 882 883 func newXLMetaV2(fi FileInfo) (xlMetaV2, error) { 884 xlMeta := xlMetaV2{} 885 return xlMeta, xlMeta.AddVersion(fi) 886 } 887 888 func (j xlMetaV2DeleteMarker) ToFileInfo(volume, path string) (FileInfo, error) { 889 versionID := "" 890 var uv uuid.UUID 891 // check if the version is not "null" 892 if j.VersionID != uv { 893 versionID = uuid.UUID(j.VersionID).String() 894 } 895 fi := FileInfo{ 896 Volume: volume, 897 Name: path, 898 ModTime: time.Unix(0, j.ModTime).UTC(), 899 VersionID: versionID, 900 Deleted: true, 901 } 902 for k, v := range j.MetaSys { 903 switch { 904 case equals(k, xhttp.AmzBucketReplicationStatus): 905 fi.DeleteMarkerReplicationStatus = string(v) 906 case equals(k, VersionPurgeStatusKey): 907 fi.VersionPurgeStatus = VersionPurgeStatusType(string(v)) 908 } 909 } 910 return fi, nil 911 } 912 913 func (j xlMetaV2Object) ToFileInfo(volume, path string) (FileInfo, error) { 914 versionID := "" 915 var uv uuid.UUID 916 // check if the version is not "null" 917 if !bytes.Equal(j.VersionID[:], uv[:]) { 918 versionID = uuid.UUID(j.VersionID).String() 919 } 920 fi := FileInfo{ 921 Volume: volume, 922 Name: path, 923 Size: j.Size, 924 ModTime: time.Unix(0, j.ModTime).UTC(), 925 VersionID: versionID, 926 } 927 fi.Parts = make([]ObjectPartInfo, len(j.PartNumbers)) 928 for i := range fi.Parts { 929 fi.Parts[i].Number = j.PartNumbers[i] 930 fi.Parts[i].Size = j.PartSizes[i] 931 fi.Parts[i].ETag = j.PartETags[i] 932 fi.Parts[i].ActualSize = j.PartActualSizes[i] 933 } 934 fi.Erasure.Checksums = make([]ChecksumInfo, len(j.PartSizes)) 935 for i := range fi.Parts { 936 fi.Erasure.Checksums[i].PartNumber = fi.Parts[i].Number 937 switch j.BitrotChecksumAlgo { 938 case HighwayHash: 939 fi.Erasure.Checksums[i].Algorithm = HighwayHash256S 940 fi.Erasure.Checksums[i].Hash = []byte{} 941 default: 942 return FileInfo{}, fmt.Errorf("unknown BitrotChecksumAlgo: %v", j.BitrotChecksumAlgo) 943 } 944 } 945 fi.Metadata = make(map[string]string, len(j.MetaUser)+len(j.MetaSys)) 946 for k, v := range j.MetaUser { 947 // https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w 948 if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) { 949 continue 950 } 951 952 fi.Metadata[k] = v 953 } 954 for k, v := range j.MetaSys { 955 switch { 956 case equals(k, ReservedMetadataPrefixLower+"transition-status"): 957 fi.TransitionStatus = string(v) 958 case equals(k, VersionPurgeStatusKey): 959 fi.VersionPurgeStatus = VersionPurgeStatusType(string(v)) 960 case strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower): 961 fi.Metadata[k] = string(v) 962 } 963 } 964 fi.Erasure.Algorithm = j.ErasureAlgorithm.String() 965 fi.Erasure.Index = j.ErasureIndex 966 fi.Erasure.BlockSize = j.ErasureBlockSize 967 fi.Erasure.DataBlocks = j.ErasureM 968 fi.Erasure.ParityBlocks = j.ErasureN 969 fi.Erasure.Distribution = make([]int, len(j.ErasureDist)) 970 for i := range j.ErasureDist { 971 fi.Erasure.Distribution[i] = int(j.ErasureDist[i]) 972 } 973 fi.DataDir = uuid.UUID(j.DataDir).String() 974 975 return fi, nil 976 } 977 978 func (z *xlMetaV2) SharedDataDirCountStr(versionID, dataDir string) int { 979 var ( 980 uv uuid.UUID 981 ddir uuid.UUID 982 err error 983 ) 984 if versionID == nullVersionID { 985 versionID = "" 986 } 987 if versionID != "" { 988 uv, err = uuid.Parse(versionID) 989 if err != nil { 990 return 0 991 } 992 } 993 ddir, err = uuid.Parse(dataDir) 994 if err != nil { 995 return 0 996 } 997 return z.SharedDataDirCount(uv, ddir) 998 } 999 1000 func (z *xlMetaV2) SharedDataDirCount(versionID [16]byte, dataDir [16]byte) int { 1001 // v2 object is inlined, if it is skip dataDir share check. 1002 if z.data.find(uuid.UUID(versionID).String()) != nil { 1003 return 0 1004 } 1005 var sameDataDirCount int 1006 for _, version := range z.Versions { 1007 switch version.Type { 1008 case ObjectType: 1009 if version.ObjectV2.VersionID == versionID { 1010 continue 1011 } 1012 if version.ObjectV2.DataDir == dataDir { 1013 sameDataDirCount++ 1014 } 1015 } 1016 } 1017 return sameDataDirCount 1018 } 1019 1020 // DeleteVersion deletes the version specified by version id. 1021 // returns to the caller which dataDir to delete, also 1022 // indicates if this is the last version. 1023 func (z *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) { 1024 // This is a situation where versionId is explicitly 1025 // specified as "null", as we do not save "null" 1026 // string it is considered empty. But empty also 1027 // means the version which matches will be purged. 1028 if fi.VersionID == nullVersionID { 1029 fi.VersionID = "" 1030 } 1031 1032 var uv uuid.UUID 1033 var err error 1034 if fi.VersionID != "" { 1035 uv, err = uuid.Parse(fi.VersionID) 1036 if err != nil { 1037 return "", false, errFileVersionNotFound 1038 } 1039 } 1040 1041 var ventry xlMetaV2Version 1042 if fi.Deleted { 1043 ventry = xlMetaV2Version{ 1044 Type: DeleteType, 1045 DeleteMarker: &xlMetaV2DeleteMarker{ 1046 VersionID: uv, 1047 ModTime: fi.ModTime.UnixNano(), 1048 MetaSys: make(map[string][]byte), 1049 }, 1050 } 1051 if !ventry.Valid() { 1052 return "", false, errors.New("internal error: invalid version entry generated") 1053 } 1054 } 1055 updateVersion := false 1056 if fi.VersionPurgeStatus.Empty() && (fi.DeleteMarkerReplicationStatus == "REPLICA" || fi.DeleteMarkerReplicationStatus == "") { 1057 updateVersion = fi.MarkDeleted 1058 } else { 1059 // for replication scenario 1060 if fi.Deleted && fi.VersionPurgeStatus != Complete { 1061 if !fi.VersionPurgeStatus.Empty() || fi.DeleteMarkerReplicationStatus != "" { 1062 updateVersion = true 1063 } 1064 } 1065 // object or delete-marker versioned delete is not complete 1066 if !fi.VersionPurgeStatus.Empty() && fi.VersionPurgeStatus != Complete { 1067 updateVersion = true 1068 } 1069 } 1070 if fi.Deleted { 1071 if fi.DeleteMarkerReplicationStatus != "" { 1072 ventry.DeleteMarker.MetaSys[xhttp.AmzBucketReplicationStatus] = []byte(fi.DeleteMarkerReplicationStatus) 1073 } 1074 if !fi.VersionPurgeStatus.Empty() { 1075 ventry.DeleteMarker.MetaSys[VersionPurgeStatusKey] = []byte(fi.VersionPurgeStatus) 1076 } 1077 } 1078 1079 for i, version := range z.Versions { 1080 if !version.Valid() { 1081 return "", false, errFileCorrupt 1082 } 1083 switch version.Type { 1084 case LegacyType: 1085 if version.ObjectV1.VersionID == fi.VersionID { 1086 if fi.TransitionStatus != "" { 1087 z.Versions[i].ObjectV1.Meta[ReservedMetadataPrefixLower+"transition-status"] = fi.TransitionStatus 1088 return uuid.UUID(version.ObjectV2.DataDir).String(), len(z.Versions) == 0, nil 1089 } 1090 1091 z.Versions = append(z.Versions[:i], z.Versions[i+1:]...) 1092 if fi.Deleted { 1093 z.Versions = append(z.Versions, ventry) 1094 } 1095 return version.ObjectV1.DataDir, len(z.Versions) == 0, nil 1096 } 1097 case DeleteType: 1098 if version.DeleteMarker.VersionID == uv { 1099 if updateVersion { 1100 if len(z.Versions[i].DeleteMarker.MetaSys) == 0 { 1101 z.Versions[i].DeleteMarker.MetaSys = make(map[string][]byte) 1102 } 1103 delete(z.Versions[i].DeleteMarker.MetaSys, xhttp.AmzBucketReplicationStatus) 1104 delete(z.Versions[i].DeleteMarker.MetaSys, VersionPurgeStatusKey) 1105 if fi.DeleteMarkerReplicationStatus != "" { 1106 z.Versions[i].DeleteMarker.MetaSys[xhttp.AmzBucketReplicationStatus] = []byte(fi.DeleteMarkerReplicationStatus) 1107 } 1108 if !fi.VersionPurgeStatus.Empty() { 1109 z.Versions[i].DeleteMarker.MetaSys[VersionPurgeStatusKey] = []byte(fi.VersionPurgeStatus) 1110 } 1111 } else { 1112 z.Versions = append(z.Versions[:i], z.Versions[i+1:]...) 1113 if fi.MarkDeleted && (fi.VersionPurgeStatus.Empty() || (fi.VersionPurgeStatus != Complete)) { 1114 z.Versions = append(z.Versions, ventry) 1115 } 1116 } 1117 return "", len(z.Versions) == 0, nil 1118 } 1119 case ObjectType: 1120 if version.ObjectV2.VersionID == uv && updateVersion { 1121 z.Versions[i].ObjectV2.MetaSys[VersionPurgeStatusKey] = []byte(fi.VersionPurgeStatus) 1122 return "", len(z.Versions) == 0, nil 1123 } 1124 } 1125 } 1126 1127 for i, version := range z.Versions { 1128 if !version.Valid() { 1129 return "", false, errFileCorrupt 1130 } 1131 switch version.Type { 1132 case ObjectType: 1133 if version.ObjectV2.VersionID == uv { 1134 if fi.TransitionStatus != "" { 1135 z.Versions[i].ObjectV2.MetaSys[ReservedMetadataPrefixLower+"transition-status"] = []byte(fi.TransitionStatus) 1136 return uuid.UUID(version.ObjectV2.DataDir).String(), len(z.Versions) == 0, nil 1137 } 1138 z.Versions = append(z.Versions[:i], z.Versions[i+1:]...) 1139 if z.SharedDataDirCount(version.ObjectV2.VersionID, version.ObjectV2.DataDir) > 0 { 1140 if fi.Deleted { 1141 z.Versions = append(z.Versions, ventry) 1142 } 1143 // Found that another version references the same dataDir 1144 // we shouldn't remove it, and only remove the version instead 1145 return "", len(z.Versions) == 0, nil 1146 } 1147 if fi.Deleted { 1148 z.Versions = append(z.Versions, ventry) 1149 } 1150 return uuid.UUID(version.ObjectV2.DataDir).String(), len(z.Versions) == 0, nil 1151 } 1152 } 1153 } 1154 1155 if fi.Deleted { 1156 z.Versions = append(z.Versions, ventry) 1157 return "", false, nil 1158 } 1159 return "", false, errFileVersionNotFound 1160 } 1161 1162 // TotalSize returns the total size of all versions. 1163 func (z xlMetaV2) TotalSize() int64 { 1164 var total int64 1165 for i := range z.Versions { 1166 switch z.Versions[i].Type { 1167 case ObjectType: 1168 total += z.Versions[i].ObjectV2.Size 1169 case LegacyType: 1170 total += z.Versions[i].ObjectV1.Stat.Size 1171 } 1172 } 1173 return total 1174 } 1175 1176 // ListVersions lists current versions, and current deleted 1177 // versions returns error for unexpected entries. 1178 // showPendingDeletes is set to true if ListVersions needs to list objects marked deleted 1179 // but waiting to be replicated 1180 func (z xlMetaV2) ListVersions(volume, path string) ([]FileInfo, time.Time, error) { 1181 versions := make([]FileInfo, 0, len(z.Versions)) 1182 var err error 1183 1184 for _, version := range z.Versions { 1185 if !version.Valid() { 1186 return nil, time.Time{}, errFileCorrupt 1187 } 1188 var fi FileInfo 1189 switch version.Type { 1190 case ObjectType: 1191 fi, err = version.ObjectV2.ToFileInfo(volume, path) 1192 case DeleteType: 1193 fi, err = version.DeleteMarker.ToFileInfo(volume, path) 1194 case LegacyType: 1195 fi, err = version.ObjectV1.ToFileInfo(volume, path) 1196 } 1197 if err != nil { 1198 return nil, time.Time{}, err 1199 } 1200 versions = append(versions, fi) 1201 } 1202 1203 sort.Sort(versionsSorter(versions)) 1204 1205 for i := range versions { 1206 versions[i].NumVersions = len(versions) 1207 if i > 0 { 1208 versions[i].SuccessorModTime = versions[i-1].ModTime 1209 } 1210 } 1211 1212 versions[0].IsLatest = true 1213 return versions, versions[0].ModTime, nil 1214 } 1215 1216 func getModTimeFromVersion(v xlMetaV2Version) time.Time { 1217 switch v.Type { 1218 case ObjectType: 1219 return time.Unix(0, v.ObjectV2.ModTime) 1220 case DeleteType: 1221 return time.Unix(0, v.DeleteMarker.ModTime) 1222 case LegacyType: 1223 return v.ObjectV1.Stat.ModTime 1224 } 1225 return time.Time{} 1226 } 1227 1228 // ToFileInfo converts xlMetaV2 into a common FileInfo datastructure 1229 // for consumption across callers. 1230 func (z xlMetaV2) ToFileInfo(volume, path, versionID string) (fi FileInfo, err error) { 1231 var uv uuid.UUID 1232 if versionID != "" && versionID != nullVersionID { 1233 uv, err = uuid.Parse(versionID) 1234 if err != nil { 1235 logger.LogIf(GlobalContext, fmt.Errorf("invalid versionID specified %s", versionID)) 1236 return FileInfo{}, errFileVersionNotFound 1237 } 1238 } 1239 1240 for _, version := range z.Versions { 1241 if !version.Valid() { 1242 logger.LogIf(GlobalContext, fmt.Errorf("invalid version detected %#v", version)) 1243 if versionID == "" { 1244 return FileInfo{}, errFileNotFound 1245 } 1246 return FileInfo{}, errFileVersionNotFound 1247 1248 } 1249 } 1250 1251 orderedVersions := make([]xlMetaV2Version, len(z.Versions)) 1252 copy(orderedVersions, z.Versions) 1253 1254 sort.Slice(orderedVersions, func(i, j int) bool { 1255 mtime1 := getModTimeFromVersion(orderedVersions[i]) 1256 mtime2 := getModTimeFromVersion(orderedVersions[j]) 1257 return mtime1.After(mtime2) 1258 }) 1259 1260 if versionID == "" { 1261 if len(orderedVersions) >= 1 { 1262 switch orderedVersions[0].Type { 1263 case ObjectType: 1264 fi, err = orderedVersions[0].ObjectV2.ToFileInfo(volume, path) 1265 case DeleteType: 1266 fi, err = orderedVersions[0].DeleteMarker.ToFileInfo(volume, path) 1267 case LegacyType: 1268 fi, err = orderedVersions[0].ObjectV1.ToFileInfo(volume, path) 1269 } 1270 fi.IsLatest = true 1271 fi.NumVersions = len(orderedVersions) 1272 return fi, err 1273 } 1274 return FileInfo{}, errFileNotFound 1275 } 1276 1277 var foundIndex = -1 1278 1279 for i := range orderedVersions { 1280 switch orderedVersions[i].Type { 1281 case ObjectType: 1282 if bytes.Equal(orderedVersions[i].ObjectV2.VersionID[:], uv[:]) { 1283 fi, err = orderedVersions[i].ObjectV2.ToFileInfo(volume, path) 1284 foundIndex = i 1285 break 1286 } 1287 case LegacyType: 1288 if orderedVersions[i].ObjectV1.VersionID == versionID { 1289 fi, err = orderedVersions[i].ObjectV1.ToFileInfo(volume, path) 1290 foundIndex = i 1291 break 1292 } 1293 case DeleteType: 1294 if bytes.Equal(orderedVersions[i].DeleteMarker.VersionID[:], uv[:]) { 1295 fi, err = orderedVersions[i].DeleteMarker.ToFileInfo(volume, path) 1296 foundIndex = i 1297 break 1298 } 1299 } 1300 } 1301 if err != nil { 1302 return fi, err 1303 } 1304 1305 if foundIndex >= 0 { 1306 // A version is found, fill dynamic fields 1307 fi.IsLatest = foundIndex == 0 1308 fi.NumVersions = len(z.Versions) 1309 if foundIndex > 0 { 1310 fi.SuccessorModTime = getModTimeFromVersion(orderedVersions[foundIndex-1]) 1311 } 1312 return fi, nil 1313 } 1314 1315 if versionID == "" { 1316 return FileInfo{}, errFileNotFound 1317 } 1318 1319 return FileInfo{}, errFileVersionNotFound 1320 }