github.com/minio/minio@v0.0.0-20240328213742-3f72439b8a27/cmd/metacache-entries.go (about) 1 // Copyright (c) 2015-2021 MinIO, Inc. 2 // 3 // This file is part of MinIO Object Storage stack 4 // 5 // This program is free software: you can redistribute it and/or modify 6 // it under the terms of the GNU Affero General Public License as published by 7 // the Free Software Foundation, either version 3 of the License, or 8 // (at your option) any later version. 9 // 10 // This program is distributed in the hope that it will be useful 11 // but WITHOUT ANY WARRANTY; without even the implied warranty of 12 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 // GNU Affero General Public License for more details. 14 // 15 // You should have received a copy of the GNU Affero General Public License 16 // along with this program. If not, see <http://www.gnu.org/licenses/>. 17 18 package cmd 19 20 import ( 21 "bytes" 22 "context" 23 "errors" 24 "os" 25 "path" 26 "sort" 27 "strings" 28 29 xioutil "github.com/minio/minio/internal/ioutil" 30 "github.com/minio/minio/internal/logger" 31 "github.com/minio/pkg/v2/console" 32 ) 33 34 // metaCacheEntry is an object or a directory within an unknown bucket. 35 type metaCacheEntry struct { 36 // name is the full name of the object including prefixes 37 name string 38 // Metadata. If none is present it is not an object but only a prefix. 39 // Entries without metadata will only be present in non-recursive scans. 40 metadata []byte 41 42 // cached contains the metadata if decoded. 43 cached *xlMetaV2 44 45 // Indicates the entry can be reused and only one reference to metadata is expected. 46 reusable bool 47 } 48 49 // isDir returns if the entry is representing a prefix directory. 50 func (e metaCacheEntry) isDir() bool { 51 return len(e.metadata) == 0 && strings.HasSuffix(e.name, slashSeparator) 52 } 53 54 // isObject returns if the entry is representing an object. 55 func (e metaCacheEntry) isObject() bool { 56 return len(e.metadata) > 0 57 } 58 59 // isObjectDir returns if the entry is representing an object/ 60 func (e metaCacheEntry) isObjectDir() bool { 61 return len(e.metadata) > 0 && strings.HasSuffix(e.name, slashSeparator) 62 } 63 64 // hasPrefix returns whether an entry has a specific prefix 65 func (e metaCacheEntry) hasPrefix(s string) bool { 66 return strings.HasPrefix(e.name, s) 67 } 68 69 // matches returns if the entries have the same versions. 70 // If strict is false we allow signatures to mismatch. 71 func (e *metaCacheEntry) matches(other *metaCacheEntry, strict bool) (prefer *metaCacheEntry, matches bool) { 72 if e == nil && other == nil { 73 return nil, true 74 } 75 if e == nil { 76 return other, false 77 } 78 if other == nil { 79 return e, false 80 } 81 82 // Name should match... 83 if e.name != other.name { 84 if e.name < other.name { 85 return e, false 86 } 87 return other, false 88 } 89 90 if other.isDir() || e.isDir() { 91 if e.isDir() { 92 return e, other.isDir() == e.isDir() 93 } 94 return other, other.isDir() == e.isDir() 95 } 96 eVers, eErr := e.xlmeta() 97 oVers, oErr := other.xlmeta() 98 if eErr != nil || oErr != nil { 99 return nil, false 100 } 101 102 // check both fileInfo's have same number of versions, if not skip 103 // the `other` entry. 104 if len(eVers.versions) != len(oVers.versions) { 105 eTime := eVers.latestModtime() 106 oTime := oVers.latestModtime() 107 if !eTime.Equal(oTime) { 108 if eTime.After(oTime) { 109 return e, false 110 } 111 return other, false 112 } 113 // Tiebreak on version count. 114 if len(eVers.versions) > len(oVers.versions) { 115 return e, false 116 } 117 return other, false 118 } 119 120 // Check if each version matches... 121 for i, eVer := range eVers.versions { 122 oVer := oVers.versions[i] 123 if eVer.header != oVer.header { 124 if !strict && eVer.header.matchesNotStrict(oVer.header) { 125 if prefer == nil { 126 if eVer.header.sortsBefore(oVer.header) { 127 prefer = e 128 } else { 129 prefer = other 130 } 131 } 132 continue 133 } 134 if prefer != nil { 135 return prefer, false 136 } 137 138 if eVer.header.sortsBefore(oVer.header) { 139 return e, false 140 } 141 return other, false 142 } 143 } 144 // If we match, return e 145 if prefer == nil { 146 prefer = e 147 } 148 return prefer, true 149 } 150 151 // isInDir returns whether the entry is in the dir when considering the separator. 152 func (e metaCacheEntry) isInDir(dir, separator string) bool { 153 if len(dir) == 0 { 154 // Root 155 idx := strings.Index(e.name, separator) 156 return idx == -1 || idx == len(e.name)-len(separator) 157 } 158 ext := strings.TrimPrefix(e.name, dir) 159 if len(ext) != len(e.name) { 160 idx := strings.Index(ext, separator) 161 // If separator is not found or is last entry, ok. 162 return idx == -1 || idx == len(ext)-len(separator) 163 } 164 return false 165 } 166 167 // isLatestDeletemarker returns whether the latest version is a delete marker. 168 // If metadata is NOT versioned false will always be returned. 169 // If v2 and UNABLE to load metadata true will be returned. 170 func (e *metaCacheEntry) isLatestDeletemarker() bool { 171 if e.cached != nil { 172 if len(e.cached.versions) == 0 { 173 return true 174 } 175 return e.cached.versions[0].header.Type == DeleteType 176 } 177 if !isXL2V1Format(e.metadata) { 178 return false 179 } 180 if meta, _, err := isIndexedMetaV2(e.metadata); meta != nil { 181 return meta.IsLatestDeleteMarker() 182 } else if err != nil { 183 return true 184 } 185 // Fall back... 186 xlMeta, err := e.xlmeta() 187 if err != nil || len(xlMeta.versions) == 0 { 188 return true 189 } 190 return xlMeta.versions[0].header.Type == DeleteType 191 } 192 193 // isAllFreeVersions returns if all objects are free versions. 194 // If metadata is NOT versioned false will always be returned. 195 // If v2 and UNABLE to load metadata true will be returned. 196 func (e *metaCacheEntry) isAllFreeVersions() bool { 197 if e.cached != nil { 198 if len(e.cached.versions) == 0 { 199 return true 200 } 201 for _, v := range e.cached.versions { 202 if !v.header.FreeVersion() { 203 return false 204 } 205 } 206 return true 207 } 208 if !isXL2V1Format(e.metadata) { 209 return false 210 } 211 if meta, _, err := isIndexedMetaV2(e.metadata); meta != nil { 212 return meta.AllHidden(false) 213 } else if err != nil { 214 return true 215 } 216 // Fall back... 217 xlMeta, err := e.xlmeta() 218 if err != nil || len(xlMeta.versions) == 0 { 219 return true 220 } 221 // Check versions.. 222 for _, v := range e.cached.versions { 223 if !v.header.FreeVersion() { 224 return false 225 } 226 } 227 return true 228 } 229 230 // fileInfo returns the decoded metadata. 231 // If entry is a directory it is returned as that. 232 // If versioned the latest version will be returned. 233 func (e *metaCacheEntry) fileInfo(bucket string) (FileInfo, error) { 234 if e.isDir() { 235 return FileInfo{ 236 Volume: bucket, 237 Name: e.name, 238 Mode: uint32(os.ModeDir), 239 }, nil 240 } 241 if e.cached != nil { 242 if len(e.cached.versions) == 0 { 243 // This special case is needed to handle xlMeta.versions == 0 244 return FileInfo{ 245 Volume: bucket, 246 Name: e.name, 247 Deleted: true, 248 IsLatest: true, 249 ModTime: timeSentinel1970, 250 }, nil 251 } 252 return e.cached.ToFileInfo(bucket, e.name, "", false, false) 253 } 254 return getFileInfo(e.metadata, bucket, e.name, "", false, false) 255 } 256 257 // xlmeta returns the decoded metadata. 258 // This should not be called on directories. 259 func (e *metaCacheEntry) xlmeta() (*xlMetaV2, error) { 260 if e.isDir() { 261 return nil, errFileNotFound 262 } 263 if e.cached == nil { 264 if len(e.metadata) == 0 { 265 // only happens if the entry is not found. 266 return nil, errFileNotFound 267 } 268 var xl xlMetaV2 269 err := xl.LoadOrConvert(e.metadata) 270 if err != nil { 271 return nil, err 272 } 273 e.cached = &xl 274 } 275 return e.cached, nil 276 } 277 278 // fileInfoVersions returns the metadata as FileInfoVersions. 279 // If entry is a directory it is returned as that. 280 func (e *metaCacheEntry) fileInfoVersions(bucket string) (FileInfoVersions, error) { 281 if e.isDir() { 282 return FileInfoVersions{ 283 Volume: bucket, 284 Name: e.name, 285 Versions: []FileInfo{ 286 { 287 Volume: bucket, 288 Name: e.name, 289 Mode: uint32(os.ModeDir), 290 }, 291 }, 292 }, nil 293 } 294 // Too small gains to reuse cache here. 295 return getFileInfoVersions(e.metadata, bucket, e.name, false) 296 } 297 298 // metaCacheEntries is a slice of metacache entries. 299 type metaCacheEntries []metaCacheEntry 300 301 // less function for sorting. 302 func (m metaCacheEntries) less(i, j int) bool { 303 return m[i].name < m[j].name 304 } 305 306 // sort entries by name. 307 // m is sorted and a sorted metadata object is returned. 308 // Changes to m will also be reflected in the returned object. 309 func (m metaCacheEntries) sort() metaCacheEntriesSorted { 310 if m.isSorted() { 311 return metaCacheEntriesSorted{o: m} 312 } 313 sort.Slice(m, m.less) 314 return metaCacheEntriesSorted{o: m} 315 } 316 317 // isSorted returns whether the objects are sorted. 318 // This is usually orders of magnitude faster than actually sorting. 319 func (m metaCacheEntries) isSorted() bool { 320 return sort.SliceIsSorted(m, m.less) 321 } 322 323 // shallowClone will create a shallow clone of the array objects, 324 // but object metadata will not be cloned. 325 func (m metaCacheEntries) shallowClone() metaCacheEntries { 326 dst := make(metaCacheEntries, len(m)) 327 copy(dst, m) 328 return dst 329 } 330 331 type metadataResolutionParams struct { 332 dirQuorum int // Number if disks needed for a directory to 'exist'. 333 objQuorum int // Number of disks needed for an object to 'exist'. 334 335 // An optimization request only an 'n' amount of versions from xl.meta 336 // to avoid resolving all versions to figure out the latest 'version' 337 // for ListObjects, ListObjectsV2 338 requestedVersions int 339 340 bucket string // Name of the bucket. Used for generating cached fileinfo. 341 strict bool // Versions must match exactly, including all metadata. 342 343 // Reusable slice for resolution 344 candidates [][]xlMetaV2ShallowVersion 345 } 346 347 // resolve multiple entries. 348 // entries are resolved by majority, then if tied by mod-time and versions. 349 // Names must match on all entries in m. 350 func (m metaCacheEntries) resolve(r *metadataResolutionParams) (selected *metaCacheEntry, ok bool) { 351 if len(m) == 0 { 352 return nil, false 353 } 354 355 dirExists := 0 356 if cap(r.candidates) < len(m) { 357 r.candidates = make([][]xlMetaV2ShallowVersion, 0, len(m)) 358 } 359 r.candidates = r.candidates[:0] 360 objsAgree := 0 361 objsValid := 0 362 for i := range m { 363 entry := &m[i] 364 // Empty entry 365 if entry.name == "" { 366 continue 367 } 368 369 if entry.isDir() { 370 dirExists++ 371 selected = entry 372 continue 373 } 374 375 // Get new entry metadata, 376 // shallow decode. 377 xl, err := entry.xlmeta() 378 if err != nil { 379 if !errors.Is(err, errFileNotFound) { 380 logger.LogIf(GlobalContext, err) 381 } 382 continue 383 } 384 objsValid++ 385 386 // Add all valid to candidates. 387 r.candidates = append(r.candidates, xl.versions) 388 389 // We select the first object we find as a candidate and see if all match that. 390 // This is to quickly identify if all agree. 391 if selected == nil { 392 selected = entry 393 objsAgree = 1 394 continue 395 } 396 // Names match, check meta... 397 if prefer, ok := entry.matches(selected, r.strict); ok { 398 selected = prefer 399 objsAgree++ 400 continue 401 } 402 } 403 404 // Return dir entries, if enough... 405 if selected != nil && selected.isDir() && dirExists >= r.dirQuorum { 406 return selected, true 407 } 408 409 // If we would never be able to reach read quorum. 410 if objsValid < r.objQuorum { 411 return nil, false 412 } 413 414 // If all objects agree. 415 if selected != nil && objsAgree == objsValid { 416 return selected, true 417 } 418 419 // If cached is nil we shall skip the entry. 420 if selected.cached == nil { 421 return nil, false 422 } 423 424 // Merge if we have disagreement. 425 // Create a new merged result. 426 selected = &metaCacheEntry{ 427 name: selected.name, 428 reusable: true, 429 cached: &xlMetaV2{metaV: selected.cached.metaV}, 430 } 431 selected.cached.versions = mergeXLV2Versions(r.objQuorum, r.strict, r.requestedVersions, r.candidates...) 432 if len(selected.cached.versions) == 0 { 433 return nil, false 434 } 435 436 // Reserialize 437 var err error 438 selected.metadata, err = selected.cached.AppendTo(metaDataPoolGet()) 439 if err != nil { 440 logger.LogIf(context.Background(), err) 441 return nil, false 442 } 443 return selected, true 444 } 445 446 // firstFound returns the first found and the number of set entries. 447 func (m metaCacheEntries) firstFound() (first *metaCacheEntry, n int) { 448 for i, entry := range m { 449 if entry.name != "" { 450 n++ 451 if first == nil { 452 first = &m[i] 453 } 454 } 455 } 456 return first, n 457 } 458 459 // names will return all names in order. 460 // Since this allocates it should not be used in critical functions. 461 func (m metaCacheEntries) names() []string { 462 res := make([]string, 0, len(m)) 463 for _, obj := range m { 464 res = append(res, obj.name) 465 } 466 return res 467 } 468 469 // metaCacheEntriesSorted contains metacache entries that are sorted. 470 type metaCacheEntriesSorted struct { 471 o metaCacheEntries 472 // list id is not serialized 473 listID string 474 // Reuse buffers 475 reuse bool 476 // Contain the last skipped object after an ILM expiry evaluation 477 lastSkippedEntry string 478 } 479 480 // shallowClone will create a shallow clone of the array objects, 481 // but object metadata will not be cloned. 482 func (m metaCacheEntriesSorted) shallowClone() metaCacheEntriesSorted { 483 // We have value receiver so we already have a copy. 484 m.o = m.o.shallowClone() 485 return m 486 } 487 488 // fileInfoVersions converts the metadata to FileInfoVersions where possible. 489 // Metadata that cannot be decoded is skipped. 490 func (m *metaCacheEntriesSorted) fileInfoVersions(bucket, prefix, delimiter, afterV string) (versions []ObjectInfo) { 491 versions = make([]ObjectInfo, 0, m.len()) 492 prevPrefix := "" 493 vcfg, _ := globalBucketVersioningSys.Get(bucket) 494 495 for _, entry := range m.o { 496 if entry.isObject() { 497 if delimiter != "" { 498 idx := strings.Index(strings.TrimPrefix(entry.name, prefix), delimiter) 499 if idx >= 0 { 500 idx = len(prefix) + idx + len(delimiter) 501 currPrefix := entry.name[:idx] 502 if currPrefix == prevPrefix { 503 continue 504 } 505 prevPrefix = currPrefix 506 versions = append(versions, ObjectInfo{ 507 IsDir: true, 508 Bucket: bucket, 509 Name: currPrefix, 510 }) 511 continue 512 } 513 } 514 515 fiv, err := entry.fileInfoVersions(bucket) 516 if err != nil { 517 continue 518 } 519 520 fiVersions := fiv.Versions 521 if afterV != "" { 522 vidMarkerIdx := fiv.findVersionIndex(afterV) 523 if vidMarkerIdx >= 0 { 524 fiVersions = fiVersions[vidMarkerIdx+1:] 525 } 526 afterV = "" 527 } 528 529 for _, version := range fiVersions { 530 versioned := vcfg != nil && vcfg.Versioned(entry.name) 531 versions = append(versions, version.ToObjectInfo(bucket, entry.name, versioned)) 532 } 533 534 continue 535 } 536 537 if entry.isDir() { 538 if delimiter == "" { 539 continue 540 } 541 idx := strings.Index(strings.TrimPrefix(entry.name, prefix), delimiter) 542 if idx < 0 { 543 continue 544 } 545 idx = len(prefix) + idx + len(delimiter) 546 currPrefix := entry.name[:idx] 547 if currPrefix == prevPrefix { 548 continue 549 } 550 prevPrefix = currPrefix 551 versions = append(versions, ObjectInfo{ 552 IsDir: true, 553 Bucket: bucket, 554 Name: currPrefix, 555 }) 556 } 557 } 558 559 return versions 560 } 561 562 // fileInfos converts the metadata to ObjectInfo where possible. 563 // Metadata that cannot be decoded is skipped. 564 func (m *metaCacheEntriesSorted) fileInfos(bucket, prefix, delimiter string) (objects []ObjectInfo) { 565 objects = make([]ObjectInfo, 0, m.len()) 566 prevPrefix := "" 567 568 vcfg, _ := globalBucketVersioningSys.Get(bucket) 569 570 for _, entry := range m.o { 571 if entry.isObject() { 572 if delimiter != "" { 573 idx := strings.Index(strings.TrimPrefix(entry.name, prefix), delimiter) 574 if idx >= 0 { 575 idx = len(prefix) + idx + len(delimiter) 576 currPrefix := entry.name[:idx] 577 if currPrefix == prevPrefix { 578 continue 579 } 580 prevPrefix = currPrefix 581 objects = append(objects, ObjectInfo{ 582 IsDir: true, 583 Bucket: bucket, 584 Name: currPrefix, 585 }) 586 continue 587 } 588 } 589 590 fi, err := entry.fileInfo(bucket) 591 if err == nil { 592 versioned := vcfg != nil && vcfg.Versioned(entry.name) 593 objects = append(objects, fi.ToObjectInfo(bucket, entry.name, versioned)) 594 } 595 continue 596 } 597 if entry.isDir() { 598 if delimiter == "" { 599 continue 600 } 601 idx := strings.Index(strings.TrimPrefix(entry.name, prefix), delimiter) 602 if idx < 0 { 603 continue 604 } 605 idx = len(prefix) + idx + len(delimiter) 606 currPrefix := entry.name[:idx] 607 if currPrefix == prevPrefix { 608 continue 609 } 610 prevPrefix = currPrefix 611 objects = append(objects, ObjectInfo{ 612 IsDir: true, 613 Bucket: bucket, 614 Name: currPrefix, 615 }) 616 } 617 } 618 619 return objects 620 } 621 622 // forwardTo will truncate m so only entries that are s or after is in the list. 623 func (m *metaCacheEntriesSorted) forwardTo(s string) { 624 if s == "" { 625 return 626 } 627 idx := sort.Search(len(m.o), func(i int) bool { 628 return m.o[i].name >= s 629 }) 630 if m.reuse { 631 for i, entry := range m.o[:idx] { 632 metaDataPoolPut(entry.metadata) 633 m.o[i].metadata = nil 634 } 635 } 636 637 m.o = m.o[idx:] 638 } 639 640 // forwardPast will truncate m so only entries that are after s is in the list. 641 func (m *metaCacheEntriesSorted) forwardPast(s string) { 642 if s == "" { 643 return 644 } 645 idx := sort.Search(len(m.o), func(i int) bool { 646 return m.o[i].name > s 647 }) 648 if m.reuse { 649 for i, entry := range m.o[:idx] { 650 metaDataPoolPut(entry.metadata) 651 m.o[i].metadata = nil 652 } 653 } 654 m.o = m.o[idx:] 655 } 656 657 // mergeEntryChannels will merge entries from in and return them sorted on out. 658 // To signify no more results are on an input channel, close it. 659 // The output channel will be closed when all inputs are emptied. 660 // If file names are equal, compareMeta is called to select which one to choose. 661 // The entry not chosen will be discarded. 662 // If the context is canceled the function will return the error, 663 // otherwise the function will return nil. 664 func mergeEntryChannels(ctx context.Context, in []chan metaCacheEntry, out chan<- metaCacheEntry, readQuorum int) error { 665 defer xioutil.SafeClose(out) 666 top := make([]*metaCacheEntry, len(in)) 667 nDone := 0 668 ctxDone := ctx.Done() 669 670 // Use simpler forwarder. 671 if len(in) == 1 { 672 for { 673 select { 674 case <-ctxDone: 675 return ctx.Err() 676 case v, ok := <-in[0]: 677 if !ok { 678 return nil 679 } 680 select { 681 case <-ctxDone: 682 return ctx.Err() 683 case out <- v: 684 } 685 } 686 } 687 } 688 689 selectFrom := func(idx int) error { 690 select { 691 case <-ctxDone: 692 return ctx.Err() 693 case entry, ok := <-in[idx]: 694 if !ok { 695 top[idx] = nil 696 nDone++ 697 } else { 698 top[idx] = &entry 699 } 700 } 701 return nil 702 } 703 // Populate all... 704 for i := range in { 705 if err := selectFrom(i); err != nil { 706 return err 707 } 708 } 709 last := "" 710 var toMerge []int 711 712 // Choose the best to return. 713 for { 714 if nDone == len(in) { 715 return nil 716 } 717 best := top[0] 718 bestIdx := 0 719 toMerge = toMerge[:0] 720 for i, other := range top[1:] { 721 otherIdx := i + 1 722 if other == nil { 723 continue 724 } 725 if best == nil { 726 best = other 727 bestIdx = otherIdx 728 continue 729 } 730 // We should make sure to avoid objects and directories 731 // of this fashion such as 732 // - foo-1 733 // - foo-1/ 734 // we should avoid this situation by making sure that 735 // we compare the `foo-1/` after path.Clean() to 736 // de-dup the entries. 737 if path.Clean(best.name) == path.Clean(other.name) { 738 toMerge = append(toMerge, otherIdx) 739 continue 740 } 741 if best.name > other.name { 742 toMerge = toMerge[:0] 743 best = other 744 bestIdx = otherIdx 745 } 746 } 747 748 // Merge any unmerged 749 if len(toMerge) > 0 { 750 versions := make([][]xlMetaV2ShallowVersion, 0, len(toMerge)+1) 751 xl, err := best.xlmeta() 752 if err == nil { 753 versions = append(versions, xl.versions) 754 } 755 for _, idx := range toMerge { 756 other := top[idx] 757 if other == nil { 758 continue 759 } 760 xl2, err := other.xlmeta() 761 if err != nil { 762 if err := selectFrom(idx); err != nil { 763 return err 764 } 765 continue 766 } 767 if xl == nil { 768 // Discard current "best" 769 if err := selectFrom(bestIdx); err != nil { 770 return err 771 } 772 bestIdx = idx 773 best = other 774 xl = xl2 775 } else { 776 // Mark read, unless we added it as new "best". 777 if err := selectFrom(idx); err != nil { 778 return err 779 } 780 } 781 versions = append(versions, xl2.versions) 782 } 783 784 if xl != nil && len(versions) > 0 { 785 // Merge all versions. 'strict' doesn't matter since we only need one. 786 xl.versions = mergeXLV2Versions(readQuorum, true, 0, versions...) 787 if meta, err := xl.AppendTo(metaDataPoolGet()); err == nil { 788 if best.reusable { 789 metaDataPoolPut(best.metadata) 790 } 791 best.metadata = meta 792 best.cached = xl 793 } 794 } 795 toMerge = toMerge[:0] 796 } 797 if best.name > last { 798 select { 799 case <-ctxDone: 800 return ctx.Err() 801 case out <- *best: 802 last = best.name 803 } 804 } else if serverDebugLog { 805 console.Debugln("mergeEntryChannels: discarding duplicate", best.name, "<=", last) 806 } 807 // Replace entry we just sent. 808 if err := selectFrom(bestIdx); err != nil { 809 return err 810 } 811 } 812 } 813 814 // merge will merge other into m. 815 // If the same entries exists in both and metadata matches only one is added, 816 // otherwise the entry from m will be placed first. 817 // Operation time is expected to be O(n+m). 818 func (m *metaCacheEntriesSorted) merge(other metaCacheEntriesSorted, limit int) { 819 merged := make(metaCacheEntries, 0, m.len()+other.len()) 820 a := m.entries() 821 b := other.entries() 822 for len(a) > 0 && len(b) > 0 { 823 switch { 824 case a[0].name == b[0].name && bytes.Equal(a[0].metadata, b[0].metadata): 825 // Same, discard one. 826 merged = append(merged, a[0]) 827 a = a[1:] 828 b = b[1:] 829 case a[0].name < b[0].name: 830 merged = append(merged, a[0]) 831 a = a[1:] 832 default: 833 merged = append(merged, b[0]) 834 b = b[1:] 835 } 836 if limit > 0 && len(merged) >= limit { 837 break 838 } 839 } 840 // Append anything left. 841 if limit < 0 || len(merged) < limit { 842 merged = append(merged, a...) 843 merged = append(merged, b...) 844 } 845 m.o = merged 846 } 847 848 // filterPrefix will filter m to only contain entries with the specified prefix. 849 func (m *metaCacheEntriesSorted) filterPrefix(s string) { 850 if s == "" { 851 return 852 } 853 m.forwardTo(s) 854 for i, o := range m.o { 855 if !o.hasPrefix(s) { 856 m.o = m.o[:i] 857 break 858 } 859 } 860 } 861 862 // filterObjectsOnly will remove prefix directories. 863 // Order is preserved, but the underlying slice is modified. 864 func (m *metaCacheEntriesSorted) filterObjectsOnly() { 865 dst := m.o[:0] 866 for _, o := range m.o { 867 if !o.isDir() { 868 dst = append(dst, o) 869 } 870 } 871 m.o = dst 872 } 873 874 // filterPrefixesOnly will remove objects. 875 // Order is preserved, but the underlying slice is modified. 876 func (m *metaCacheEntriesSorted) filterPrefixesOnly() { 877 dst := m.o[:0] 878 for _, o := range m.o { 879 if o.isDir() { 880 dst = append(dst, o) 881 } 882 } 883 m.o = dst 884 } 885 886 // filterRecursiveEntries will keep entries only with the prefix that doesn't contain separator. 887 // This can be used to remove recursive listings. 888 // To return root elements only set prefix to an empty string. 889 // Order is preserved, but the underlying slice is modified. 890 func (m *metaCacheEntriesSorted) filterRecursiveEntries(prefix, separator string) { 891 dst := m.o[:0] 892 if prefix != "" { 893 m.forwardTo(prefix) 894 for _, o := range m.o { 895 ext := strings.TrimPrefix(o.name, prefix) 896 if len(ext) != len(o.name) { 897 if !strings.Contains(ext, separator) { 898 dst = append(dst, o) 899 } 900 } 901 } 902 } else { 903 // No prefix, simpler 904 for _, o := range m.o { 905 if !strings.Contains(o.name, separator) { 906 dst = append(dst, o) 907 } 908 } 909 } 910 m.o = dst 911 } 912 913 // truncate the number of entries to maximum n. 914 func (m *metaCacheEntriesSorted) truncate(n int) { 915 if m == nil { 916 return 917 } 918 if len(m.o) > n { 919 if m.reuse { 920 for i, entry := range m.o[n:] { 921 metaDataPoolPut(entry.metadata) 922 m.o[n+i].metadata = nil 923 } 924 } 925 m.o = m.o[:n] 926 } 927 } 928 929 // len returns the number of objects and prefix dirs in m. 930 func (m *metaCacheEntriesSorted) len() int { 931 if m == nil { 932 return 0 933 } 934 return len(m.o) 935 } 936 937 // entries returns the underlying objects as is currently represented. 938 func (m *metaCacheEntriesSorted) entries() metaCacheEntries { 939 if m == nil { 940 return nil 941 } 942 return m.o 943 }