storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/metacache-stream.go (about) 1 /* 2 * MinIO Cloud Storage, (C) 2020 MinIO, Inc. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package cmd 18 19 import ( 20 "bytes" 21 "context" 22 "errors" 23 "fmt" 24 "io" 25 "strings" 26 "sync" 27 28 jsoniter "github.com/json-iterator/go" 29 "github.com/klauspost/compress/s2" 30 "github.com/tinylib/msgp/msgp" 31 32 "storj.io/minio/cmd/logger" 33 ) 34 35 // metadata stream format: 36 // 37 // The stream is s2 compressed. 38 // https://github.com/klauspost/compress/tree/master/s2#s2-compression 39 // This ensures integrity and reduces the size typically by at least 50%. 40 // 41 // All stream elements are msgpack encoded. 42 // 43 // 1 Integer, metacacheStreamVersion of the writer. 44 // This can be used for managing breaking changes. 45 // 46 // For each element: 47 // 1. Bool. If false at end of stream. 48 // 2. String. Name of object. Directories contains a trailing slash. 49 // 3. Binary. Blob of metadata. Length 0 on directories. 50 // ... Next element. 51 // 52 // Streams can be assumed to be sorted in ascending order. 53 // If the stream ends before a false boolean it can be assumed it was truncated. 54 55 const metacacheStreamVersion = 1 56 57 // metacacheWriter provides a serializer of metacache objects. 58 type metacacheWriter struct { 59 mw *msgp.Writer 60 creator func() error 61 closer func() error 62 blockSize int 63 64 streamErr error 65 streamWg sync.WaitGroup 66 } 67 68 // newMetacacheWriter will create a serializer that will write objects in given order to the output. 69 // Provide a block size that affects latency. If 0 a default of 128KiB will be used. 70 // Block size can be up to 4MiB. 71 func newMetacacheWriter(out io.Writer, blockSize int) *metacacheWriter { 72 if blockSize < 8<<10 { 73 blockSize = 128 << 10 74 } 75 w := metacacheWriter{ 76 mw: nil, 77 blockSize: blockSize, 78 } 79 w.creator = func() error { 80 s2w := s2.NewWriter(out, s2.WriterBlockSize(blockSize), s2.WriterConcurrency(2)) 81 w.mw = msgp.NewWriter(s2w) 82 w.creator = nil 83 if err := w.mw.WriteByte(metacacheStreamVersion); err != nil { 84 return err 85 } 86 87 w.closer = func() (err error) { 88 defer func() { 89 cerr := s2w.Close() 90 if err == nil && cerr != nil { 91 err = cerr 92 } 93 }() 94 if w.streamErr != nil { 95 return w.streamErr 96 } 97 if err = w.mw.WriteBool(false); err != nil { 98 return err 99 } 100 return w.mw.Flush() 101 } 102 return nil 103 } 104 return &w 105 } 106 107 // write one or more objects to the stream in order. 108 // It is favorable to send as many objects as possible in a single write, 109 // but no more than math.MaxUint32 110 func (w *metacacheWriter) write(objs ...metaCacheEntry) error { 111 if w == nil { 112 return errors.New("metacacheWriter: nil writer") 113 } 114 if len(objs) == 0 { 115 return nil 116 } 117 if w.creator != nil { 118 err := w.creator() 119 w.creator = nil 120 if err != nil { 121 return fmt.Errorf("metacacheWriter: unable to create writer: %w", err) 122 } 123 if w.mw == nil { 124 return errors.New("metacacheWriter: writer not initialized") 125 } 126 } 127 for _, o := range objs { 128 if len(o.name) == 0 { 129 return errors.New("metacacheWriter: no name provided") 130 } 131 // Indicate EOS 132 err := w.mw.WriteBool(true) 133 if err != nil { 134 return err 135 } 136 err = w.mw.WriteString(o.name) 137 if err != nil { 138 return err 139 } 140 err = w.mw.WriteBytes(o.metadata) 141 if err != nil { 142 return err 143 } 144 } 145 146 return nil 147 } 148 149 // stream entries to the output. 150 // The returned channel should be closed when done. 151 // Any error is reported when closing the metacacheWriter. 152 func (w *metacacheWriter) stream() (chan<- metaCacheEntry, error) { 153 if w.creator != nil { 154 err := w.creator() 155 w.creator = nil 156 if err != nil { 157 return nil, fmt.Errorf("metacacheWriter: unable to create writer: %w", err) 158 } 159 if w.mw == nil { 160 return nil, errors.New("metacacheWriter: writer not initialized") 161 } 162 } 163 var objs = make(chan metaCacheEntry, 100) 164 w.streamErr = nil 165 w.streamWg.Add(1) 166 go func() { 167 defer w.streamWg.Done() 168 for o := range objs { 169 if len(o.name) == 0 || w.streamErr != nil { 170 continue 171 } 172 // Indicate EOS 173 err := w.mw.WriteBool(true) 174 if err != nil { 175 w.streamErr = err 176 continue 177 } 178 err = w.mw.WriteString(o.name) 179 if err != nil { 180 w.streamErr = err 181 continue 182 } 183 err = w.mw.WriteBytes(o.metadata) 184 if err != nil { 185 w.streamErr = err 186 continue 187 } 188 } 189 }() 190 191 return objs, nil 192 } 193 194 // Close and release resources. 195 func (w *metacacheWriter) Close() error { 196 if w == nil || w.closer == nil { 197 return nil 198 } 199 w.streamWg.Wait() 200 err := w.closer() 201 w.closer = nil 202 return err 203 } 204 205 // Reset and start writing to new writer. 206 // Close must have been called before this. 207 func (w *metacacheWriter) Reset(out io.Writer) { 208 w.streamErr = nil 209 w.creator = func() error { 210 s2w := s2.NewWriter(out, s2.WriterBlockSize(w.blockSize), s2.WriterConcurrency(2)) 211 w.mw = msgp.NewWriter(s2w) 212 w.creator = nil 213 if err := w.mw.WriteByte(metacacheStreamVersion); err != nil { 214 return err 215 } 216 217 w.closer = func() error { 218 if w.streamErr != nil { 219 return w.streamErr 220 } 221 if err := w.mw.WriteBool(false); err != nil { 222 return err 223 } 224 if err := w.mw.Flush(); err != nil { 225 return err 226 } 227 return s2w.Close() 228 } 229 return nil 230 } 231 } 232 233 var s2DecPool = sync.Pool{New: func() interface{} { 234 // Default alloc block for network transfer. 235 return s2.NewReader(nil, s2.ReaderAllocBlock(16<<10)) 236 }} 237 238 // metacacheReader allows reading a cache stream. 239 type metacacheReader struct { 240 mr *msgp.Reader 241 current metaCacheEntry 242 err error // stateful error 243 closer func() 244 creator func() error 245 } 246 247 // newMetacacheReader creates a new cache reader. 248 // Nothing will be read from the stream yet. 249 func newMetacacheReader(r io.Reader) (*metacacheReader, error) { 250 dec := s2DecPool.Get().(*s2.Reader) 251 dec.Reset(r) 252 mr := msgp.NewReader(dec) 253 m := metacacheReader{ 254 mr: mr, 255 closer: func() { 256 dec.Reset(nil) 257 s2DecPool.Put(dec) 258 }, 259 creator: func() error { 260 v, err := mr.ReadByte() 261 if err != nil { 262 return err 263 } 264 switch v { 265 case metacacheStreamVersion: 266 default: 267 return fmt.Errorf("metacacheReader: Unknown version: %d", v) 268 } 269 return nil 270 }, 271 } 272 return &m, nil 273 } 274 275 func (r *metacacheReader) checkInit() { 276 if r.creator == nil || r.err != nil { 277 return 278 } 279 r.err = r.creator() 280 r.creator = nil 281 } 282 283 // peek will return the name of the next object. 284 // Will return io.EOF if there are no more objects. 285 // Should be used sparingly. 286 func (r *metacacheReader) peek() (metaCacheEntry, error) { 287 r.checkInit() 288 if r.err != nil { 289 return metaCacheEntry{}, r.err 290 } 291 if r.current.name != "" { 292 return r.current, nil 293 } 294 if more, err := r.mr.ReadBool(); !more { 295 switch err { 296 case nil: 297 r.err = io.EOF 298 return metaCacheEntry{}, io.EOF 299 case io.EOF: 300 r.err = io.ErrUnexpectedEOF 301 return metaCacheEntry{}, io.ErrUnexpectedEOF 302 } 303 r.err = err 304 return metaCacheEntry{}, err 305 } 306 307 var err error 308 if r.current.name, err = r.mr.ReadString(); err != nil { 309 if err == io.EOF { 310 err = io.ErrUnexpectedEOF 311 } 312 r.err = err 313 return metaCacheEntry{}, err 314 } 315 r.current.metadata, err = r.mr.ReadBytes(r.current.metadata[:0]) 316 if err == io.EOF { 317 err = io.ErrUnexpectedEOF 318 } 319 r.err = err 320 return r.current, err 321 } 322 323 // next will read one entry from the stream. 324 // Generally not recommended for fast operation. 325 func (r *metacacheReader) next() (metaCacheEntry, error) { 326 r.checkInit() 327 if r.err != nil { 328 return metaCacheEntry{}, r.err 329 } 330 var m metaCacheEntry 331 var err error 332 if r.current.name != "" { 333 m.name = r.current.name 334 m.metadata = r.current.metadata 335 r.current.name = "" 336 r.current.metadata = nil 337 return m, nil 338 } 339 if more, err := r.mr.ReadBool(); !more { 340 switch err { 341 case nil: 342 r.err = io.EOF 343 return m, io.EOF 344 case io.EOF: 345 r.err = io.ErrUnexpectedEOF 346 return m, io.ErrUnexpectedEOF 347 } 348 r.err = err 349 return m, err 350 } 351 if m.name, err = r.mr.ReadString(); err != nil { 352 if err == io.EOF { 353 err = io.ErrUnexpectedEOF 354 } 355 r.err = err 356 return m, err 357 } 358 m.metadata, err = r.mr.ReadBytes(nil) 359 if err == io.EOF { 360 err = io.ErrUnexpectedEOF 361 } 362 r.err = err 363 return m, err 364 } 365 366 // next will read one entry from the stream. 367 // Generally not recommended for fast operation. 368 func (r *metacacheReader) nextEOF() bool { 369 r.checkInit() 370 if r.err != nil { 371 return r.err == io.EOF 372 } 373 if r.current.name != "" { 374 return false 375 } 376 _, err := r.peek() 377 if err != nil { 378 r.err = err 379 return r.err == io.EOF 380 } 381 return false 382 } 383 384 // forwardTo will forward to the first entry that is >= s. 385 // Will return io.EOF if end of stream is reached without finding any. 386 func (r *metacacheReader) forwardTo(s string) error { 387 r.checkInit() 388 if r.err != nil { 389 return r.err 390 } 391 392 if s == "" { 393 return nil 394 } 395 if r.current.name != "" { 396 if r.current.name >= s { 397 return nil 398 } 399 r.current.name = "" 400 r.current.metadata = nil 401 } 402 // temporary name buffer. 403 var tmp = make([]byte, 0, 256) 404 for { 405 if more, err := r.mr.ReadBool(); !more { 406 switch err { 407 case nil: 408 r.err = io.EOF 409 return io.EOF 410 case io.EOF: 411 r.err = io.ErrUnexpectedEOF 412 return io.ErrUnexpectedEOF 413 } 414 r.err = err 415 return err 416 } 417 // Read name without allocating more than 1 buffer. 418 sz, err := r.mr.ReadStringHeader() 419 if err != nil { 420 r.err = err 421 return err 422 } 423 if cap(tmp) < int(sz) { 424 tmp = make([]byte, 0, sz+256) 425 } 426 tmp = tmp[:sz] 427 _, err = r.mr.R.ReadFull(tmp) 428 if err != nil { 429 r.err = err 430 return err 431 } 432 if string(tmp) >= s { 433 r.current.name = string(tmp) 434 r.current.metadata, r.err = r.mr.ReadBytes(nil) 435 return r.err 436 } 437 // Skip metadata 438 err = r.mr.Skip() 439 if err != nil { 440 if err == io.EOF { 441 err = io.ErrUnexpectedEOF 442 } 443 r.err = err 444 return err 445 } 446 } 447 } 448 449 // readN will return all the requested number of entries in order 450 // or all if n < 0. 451 // Will return io.EOF if end of stream is reached. 452 // If requesting 0 objects nil error will always be returned regardless of at end of stream. 453 // Use peek to determine if at end of stream. 454 func (r *metacacheReader) readN(n int, inclDeleted, inclDirs bool, prefix string) (metaCacheEntriesSorted, error) { 455 r.checkInit() 456 if n == 0 { 457 return metaCacheEntriesSorted{}, nil 458 } 459 if r.err != nil { 460 return metaCacheEntriesSorted{}, r.err 461 } 462 463 var res metaCacheEntries 464 if n > 0 { 465 res = make(metaCacheEntries, 0, n) 466 } 467 if prefix != "" { 468 if err := r.forwardTo(prefix); err != nil { 469 return metaCacheEntriesSorted{}, err 470 } 471 } 472 next, err := r.peek() 473 if err != nil { 474 return metaCacheEntriesSorted{}, err 475 } 476 if !next.hasPrefix(prefix) { 477 return metaCacheEntriesSorted{}, io.EOF 478 } 479 480 if r.current.name != "" { 481 if (inclDeleted || !r.current.isLatestDeletemarker()) && r.current.hasPrefix(prefix) && (inclDirs || r.current.isObject()) { 482 res = append(res, r.current) 483 } 484 r.current.name = "" 485 r.current.metadata = nil 486 } 487 488 for n < 0 || len(res) < n { 489 if more, err := r.mr.ReadBool(); !more { 490 switch err { 491 case nil: 492 r.err = io.EOF 493 return metaCacheEntriesSorted{o: res}, io.EOF 494 case io.EOF: 495 r.err = io.ErrUnexpectedEOF 496 return metaCacheEntriesSorted{o: res}, io.ErrUnexpectedEOF 497 } 498 r.err = err 499 return metaCacheEntriesSorted{o: res}, err 500 } 501 var err error 502 var meta metaCacheEntry 503 if meta.name, err = r.mr.ReadString(); err != nil { 504 if err == io.EOF { 505 err = io.ErrUnexpectedEOF 506 } 507 r.err = err 508 return metaCacheEntriesSorted{o: res}, err 509 } 510 if !meta.hasPrefix(prefix) { 511 r.mr.R.Skip(1) 512 return metaCacheEntriesSorted{o: res}, io.EOF 513 } 514 if meta.metadata, err = r.mr.ReadBytes(nil); err != nil { 515 if err == io.EOF { 516 err = io.ErrUnexpectedEOF 517 } 518 r.err = err 519 return metaCacheEntriesSorted{o: res}, err 520 } 521 if !inclDirs && meta.isDir() { 522 continue 523 } 524 if !inclDeleted && meta.isLatestDeletemarker() { 525 continue 526 } 527 res = append(res, meta) 528 } 529 return metaCacheEntriesSorted{o: res}, nil 530 } 531 532 // readAll will return all remaining objects on the dst channel and close it when done. 533 // The context allows the operation to be canceled. 534 func (r *metacacheReader) readAll(ctx context.Context, dst chan<- metaCacheEntry) error { 535 r.checkInit() 536 if r.err != nil { 537 return r.err 538 } 539 defer close(dst) 540 if r.current.name != "" { 541 select { 542 case <-ctx.Done(): 543 r.err = ctx.Err() 544 return ctx.Err() 545 case dst <- r.current: 546 } 547 r.current.name = "" 548 r.current.metadata = nil 549 } 550 for { 551 if more, err := r.mr.ReadBool(); !more { 552 switch err { 553 case io.EOF: 554 err = io.ErrUnexpectedEOF 555 } 556 r.err = err 557 return err 558 } 559 560 var err error 561 var meta metaCacheEntry 562 if meta.name, err = r.mr.ReadString(); err != nil { 563 if err == io.EOF { 564 err = io.ErrUnexpectedEOF 565 } 566 r.err = err 567 return err 568 } 569 if meta.metadata, err = r.mr.ReadBytes(nil); err != nil { 570 if err == io.EOF { 571 err = io.ErrUnexpectedEOF 572 } 573 r.err = err 574 return err 575 } 576 select { 577 case <-ctx.Done(): 578 r.err = ctx.Err() 579 return ctx.Err() 580 case dst <- meta: 581 } 582 } 583 } 584 585 // readFn will return all remaining objects 586 // and provide a callback for each entry read in order 587 // as long as true is returned on the callback. 588 func (r *metacacheReader) readFn(fn func(entry metaCacheEntry) bool) error { 589 r.checkInit() 590 if r.err != nil { 591 return r.err 592 } 593 if r.current.name != "" { 594 fn(r.current) 595 r.current.name = "" 596 r.current.metadata = nil 597 } 598 for { 599 if more, err := r.mr.ReadBool(); !more { 600 switch err { 601 case io.EOF: 602 r.err = io.ErrUnexpectedEOF 603 return io.ErrUnexpectedEOF 604 case nil: 605 r.err = io.EOF 606 return io.EOF 607 } 608 return err 609 } 610 611 var err error 612 var meta metaCacheEntry 613 if meta.name, err = r.mr.ReadString(); err != nil { 614 if err == io.EOF { 615 err = io.ErrUnexpectedEOF 616 } 617 r.err = err 618 return err 619 } 620 if meta.metadata, err = r.mr.ReadBytes(nil); err != nil { 621 if err == io.EOF { 622 err = io.ErrUnexpectedEOF 623 } 624 r.err = err 625 return err 626 } 627 // Send it! 628 if !fn(meta) { 629 return nil 630 } 631 } 632 } 633 634 // readNames will return all the requested number of names in order 635 // or all if n < 0. 636 // Will return io.EOF if end of stream is reached. 637 func (r *metacacheReader) readNames(n int) ([]string, error) { 638 r.checkInit() 639 if r.err != nil { 640 return nil, r.err 641 } 642 if n == 0 { 643 return nil, nil 644 } 645 var res []string 646 if n > 0 { 647 res = make([]string, 0, n) 648 } 649 if r.current.name != "" { 650 res = append(res, r.current.name) 651 r.current.name = "" 652 r.current.metadata = nil 653 } 654 for n < 0 || len(res) < n { 655 if more, err := r.mr.ReadBool(); !more { 656 switch err { 657 case nil: 658 r.err = io.EOF 659 return res, io.EOF 660 case io.EOF: 661 r.err = io.ErrUnexpectedEOF 662 return res, io.ErrUnexpectedEOF 663 } 664 return res, err 665 } 666 667 var err error 668 var name string 669 if name, err = r.mr.ReadString(); err != nil { 670 r.err = err 671 return res, err 672 } 673 if err = r.mr.Skip(); err != nil { 674 if err == io.EOF { 675 err = io.ErrUnexpectedEOF 676 } 677 r.err = err 678 return res, err 679 } 680 res = append(res, name) 681 } 682 return res, nil 683 } 684 685 // skip n entries on the input stream. 686 // If there are less entries left io.EOF is returned. 687 func (r *metacacheReader) skip(n int) error { 688 r.checkInit() 689 if r.err != nil { 690 return r.err 691 } 692 if n <= 0 { 693 return nil 694 } 695 if r.current.name != "" { 696 n-- 697 r.current.name = "" 698 r.current.metadata = nil 699 } 700 for n > 0 { 701 if more, err := r.mr.ReadBool(); !more { 702 switch err { 703 case nil: 704 r.err = io.EOF 705 return io.EOF 706 case io.EOF: 707 r.err = io.ErrUnexpectedEOF 708 return io.ErrUnexpectedEOF 709 } 710 return err 711 } 712 713 if err := r.mr.Skip(); err != nil { 714 if err == io.EOF { 715 err = io.ErrUnexpectedEOF 716 } 717 r.err = err 718 return err 719 } 720 if err := r.mr.Skip(); err != nil { 721 if err == io.EOF { 722 err = io.ErrUnexpectedEOF 723 } 724 r.err = err 725 return err 726 } 727 n-- 728 } 729 return nil 730 } 731 732 // Close and release resources. 733 func (r *metacacheReader) Close() error { 734 if r == nil || r.closer == nil { 735 return nil 736 } 737 r.closer() 738 r.closer = nil 739 r.creator = nil 740 return nil 741 } 742 743 // metacacheBlockWriter collects blocks and provides a callaback to store them. 744 type metacacheBlockWriter struct { 745 wg sync.WaitGroup 746 streamErr error 747 blockEntries int 748 } 749 750 var bufferPool = sync.Pool{ 751 New: func() interface{} { 752 return new(bytes.Buffer) 753 }, 754 } 755 756 // newMetacacheBlockWriter provides a streaming block writer. 757 // Each block is the size of the capacity of the input channel. 758 // The caller should close to indicate the stream has ended. 759 func newMetacacheBlockWriter(in <-chan metaCacheEntry, nextBlock func(b *metacacheBlock) error) *metacacheBlockWriter { 760 w := metacacheBlockWriter{blockEntries: cap(in)} 761 w.wg.Add(1) 762 go func() { 763 defer w.wg.Done() 764 var current metacacheBlock 765 var n int 766 buf := bufferPool.Get().(*bytes.Buffer) 767 defer func() { 768 buf.Reset() 769 bufferPool.Put(buf) 770 }() 771 block := newMetacacheWriter(buf, 1<<20) 772 defer block.Close() 773 finishBlock := func() { 774 if err := block.Close(); err != nil { 775 w.streamErr = err 776 return 777 } 778 current.data = buf.Bytes() 779 w.streamErr = nextBlock(¤t) 780 // Prepare for next 781 current.n++ 782 buf.Reset() 783 block.Reset(buf) 784 current.First = "" 785 } 786 for o := range in { 787 if len(o.name) == 0 || w.streamErr != nil { 788 continue 789 } 790 if current.First == "" { 791 current.First = o.name 792 } 793 794 if n >= w.blockEntries-1 { 795 finishBlock() 796 n = 0 797 } 798 n++ 799 800 w.streamErr = block.write(o) 801 if w.streamErr != nil { 802 continue 803 } 804 current.Last = o.name 805 } 806 if n > 0 || current.n == 0 { 807 current.EOS = true 808 finishBlock() 809 } 810 }() 811 return &w 812 } 813 814 // Close the stream. 815 // The incoming channel must be closed before calling this. 816 // Returns the first error the occurred during the writing if any. 817 func (w *metacacheBlockWriter) Close() error { 818 w.wg.Wait() 819 return w.streamErr 820 } 821 822 type metacacheBlock struct { 823 data []byte 824 n int 825 First string `json:"f"` 826 Last string `json:"l"` 827 EOS bool `json:"eos,omitempty"` 828 } 829 830 func (b metacacheBlock) headerKV() map[string]string { 831 var json = jsoniter.ConfigCompatibleWithStandardLibrary 832 v, err := json.Marshal(b) 833 if err != nil { 834 logger.LogIf(context.Background(), err) // Unlikely 835 return nil 836 } 837 return map[string]string{fmt.Sprintf("%s-metacache-part-%d", ReservedMetadataPrefixLower, b.n): string(v)} 838 } 839 840 // pastPrefix returns true if the given prefix is before start of the block. 841 func (b metacacheBlock) pastPrefix(prefix string) bool { 842 if prefix == "" || strings.HasPrefix(b.First, prefix) { 843 return false 844 } 845 // We have checked if prefix matches, so we can do direct compare. 846 return b.First > prefix 847 } 848 849 // endedPrefix returns true if the given prefix ends within the block. 850 func (b metacacheBlock) endedPrefix(prefix string) bool { 851 if prefix == "" || strings.HasPrefix(b.Last, prefix) { 852 return false 853 } 854 855 // We have checked if prefix matches, so we can do direct compare. 856 return b.Last > prefix 857 }