storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/metacache-stream.go (about)

     1  /*
     2   * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package cmd
    18  
    19  import (
    20  	"bytes"
    21  	"context"
    22  	"errors"
    23  	"fmt"
    24  	"io"
    25  	"strings"
    26  	"sync"
    27  
    28  	jsoniter "github.com/json-iterator/go"
    29  	"github.com/klauspost/compress/s2"
    30  	"github.com/tinylib/msgp/msgp"
    31  
    32  	"storj.io/minio/cmd/logger"
    33  )
    34  
    35  // metadata stream format:
    36  //
    37  // The stream is s2 compressed.
    38  // https://github.com/klauspost/compress/tree/master/s2#s2-compression
    39  // This ensures integrity and reduces the size typically by at least 50%.
    40  //
    41  // All stream elements are msgpack encoded.
    42  //
    43  // 1 Integer, metacacheStreamVersion of the writer.
    44  // This can be used for managing breaking changes.
    45  //
    46  // For each element:
    47  // 1. Bool. If false at end of stream.
    48  // 2. String. Name of object. Directories contains a trailing slash.
    49  // 3. Binary. Blob of metadata. Length 0 on directories.
    50  // ... Next element.
    51  //
    52  // Streams can be assumed to be sorted in ascending order.
    53  // If the stream ends before a false boolean it can be assumed it was truncated.
    54  
    55  const metacacheStreamVersion = 1
    56  
    57  // metacacheWriter provides a serializer of metacache objects.
    58  type metacacheWriter struct {
    59  	mw        *msgp.Writer
    60  	creator   func() error
    61  	closer    func() error
    62  	blockSize int
    63  
    64  	streamErr error
    65  	streamWg  sync.WaitGroup
    66  }
    67  
    68  // newMetacacheWriter will create a serializer that will write objects in given order to the output.
    69  // Provide a block size that affects latency. If 0 a default of 128KiB will be used.
    70  // Block size can be up to 4MiB.
    71  func newMetacacheWriter(out io.Writer, blockSize int) *metacacheWriter {
    72  	if blockSize < 8<<10 {
    73  		blockSize = 128 << 10
    74  	}
    75  	w := metacacheWriter{
    76  		mw:        nil,
    77  		blockSize: blockSize,
    78  	}
    79  	w.creator = func() error {
    80  		s2w := s2.NewWriter(out, s2.WriterBlockSize(blockSize), s2.WriterConcurrency(2))
    81  		w.mw = msgp.NewWriter(s2w)
    82  		w.creator = nil
    83  		if err := w.mw.WriteByte(metacacheStreamVersion); err != nil {
    84  			return err
    85  		}
    86  
    87  		w.closer = func() (err error) {
    88  			defer func() {
    89  				cerr := s2w.Close()
    90  				if err == nil && cerr != nil {
    91  					err = cerr
    92  				}
    93  			}()
    94  			if w.streamErr != nil {
    95  				return w.streamErr
    96  			}
    97  			if err = w.mw.WriteBool(false); err != nil {
    98  				return err
    99  			}
   100  			return w.mw.Flush()
   101  		}
   102  		return nil
   103  	}
   104  	return &w
   105  }
   106  
   107  // write one or more objects to the stream in order.
   108  // It is favorable to send as many objects as possible in a single write,
   109  // but no more than math.MaxUint32
   110  func (w *metacacheWriter) write(objs ...metaCacheEntry) error {
   111  	if w == nil {
   112  		return errors.New("metacacheWriter: nil writer")
   113  	}
   114  	if len(objs) == 0 {
   115  		return nil
   116  	}
   117  	if w.creator != nil {
   118  		err := w.creator()
   119  		w.creator = nil
   120  		if err != nil {
   121  			return fmt.Errorf("metacacheWriter: unable to create writer: %w", err)
   122  		}
   123  		if w.mw == nil {
   124  			return errors.New("metacacheWriter: writer not initialized")
   125  		}
   126  	}
   127  	for _, o := range objs {
   128  		if len(o.name) == 0 {
   129  			return errors.New("metacacheWriter: no name provided")
   130  		}
   131  		// Indicate EOS
   132  		err := w.mw.WriteBool(true)
   133  		if err != nil {
   134  			return err
   135  		}
   136  		err = w.mw.WriteString(o.name)
   137  		if err != nil {
   138  			return err
   139  		}
   140  		err = w.mw.WriteBytes(o.metadata)
   141  		if err != nil {
   142  			return err
   143  		}
   144  	}
   145  
   146  	return nil
   147  }
   148  
   149  // stream entries to the output.
   150  // The returned channel should be closed when done.
   151  // Any error is reported when closing the metacacheWriter.
   152  func (w *metacacheWriter) stream() (chan<- metaCacheEntry, error) {
   153  	if w.creator != nil {
   154  		err := w.creator()
   155  		w.creator = nil
   156  		if err != nil {
   157  			return nil, fmt.Errorf("metacacheWriter: unable to create writer: %w", err)
   158  		}
   159  		if w.mw == nil {
   160  			return nil, errors.New("metacacheWriter: writer not initialized")
   161  		}
   162  	}
   163  	var objs = make(chan metaCacheEntry, 100)
   164  	w.streamErr = nil
   165  	w.streamWg.Add(1)
   166  	go func() {
   167  		defer w.streamWg.Done()
   168  		for o := range objs {
   169  			if len(o.name) == 0 || w.streamErr != nil {
   170  				continue
   171  			}
   172  			// Indicate EOS
   173  			err := w.mw.WriteBool(true)
   174  			if err != nil {
   175  				w.streamErr = err
   176  				continue
   177  			}
   178  			err = w.mw.WriteString(o.name)
   179  			if err != nil {
   180  				w.streamErr = err
   181  				continue
   182  			}
   183  			err = w.mw.WriteBytes(o.metadata)
   184  			if err != nil {
   185  				w.streamErr = err
   186  				continue
   187  			}
   188  		}
   189  	}()
   190  
   191  	return objs, nil
   192  }
   193  
   194  // Close and release resources.
   195  func (w *metacacheWriter) Close() error {
   196  	if w == nil || w.closer == nil {
   197  		return nil
   198  	}
   199  	w.streamWg.Wait()
   200  	err := w.closer()
   201  	w.closer = nil
   202  	return err
   203  }
   204  
   205  // Reset and start writing to new writer.
   206  // Close must have been called before this.
   207  func (w *metacacheWriter) Reset(out io.Writer) {
   208  	w.streamErr = nil
   209  	w.creator = func() error {
   210  		s2w := s2.NewWriter(out, s2.WriterBlockSize(w.blockSize), s2.WriterConcurrency(2))
   211  		w.mw = msgp.NewWriter(s2w)
   212  		w.creator = nil
   213  		if err := w.mw.WriteByte(metacacheStreamVersion); err != nil {
   214  			return err
   215  		}
   216  
   217  		w.closer = func() error {
   218  			if w.streamErr != nil {
   219  				return w.streamErr
   220  			}
   221  			if err := w.mw.WriteBool(false); err != nil {
   222  				return err
   223  			}
   224  			if err := w.mw.Flush(); err != nil {
   225  				return err
   226  			}
   227  			return s2w.Close()
   228  		}
   229  		return nil
   230  	}
   231  }
   232  
   233  var s2DecPool = sync.Pool{New: func() interface{} {
   234  	// Default alloc block for network transfer.
   235  	return s2.NewReader(nil, s2.ReaderAllocBlock(16<<10))
   236  }}
   237  
   238  // metacacheReader allows reading a cache stream.
   239  type metacacheReader struct {
   240  	mr      *msgp.Reader
   241  	current metaCacheEntry
   242  	err     error // stateful error
   243  	closer  func()
   244  	creator func() error
   245  }
   246  
   247  // newMetacacheReader creates a new cache reader.
   248  // Nothing will be read from the stream yet.
   249  func newMetacacheReader(r io.Reader) (*metacacheReader, error) {
   250  	dec := s2DecPool.Get().(*s2.Reader)
   251  	dec.Reset(r)
   252  	mr := msgp.NewReader(dec)
   253  	m := metacacheReader{
   254  		mr: mr,
   255  		closer: func() {
   256  			dec.Reset(nil)
   257  			s2DecPool.Put(dec)
   258  		},
   259  		creator: func() error {
   260  			v, err := mr.ReadByte()
   261  			if err != nil {
   262  				return err
   263  			}
   264  			switch v {
   265  			case metacacheStreamVersion:
   266  			default:
   267  				return fmt.Errorf("metacacheReader: Unknown version: %d", v)
   268  			}
   269  			return nil
   270  		},
   271  	}
   272  	return &m, nil
   273  }
   274  
   275  func (r *metacacheReader) checkInit() {
   276  	if r.creator == nil || r.err != nil {
   277  		return
   278  	}
   279  	r.err = r.creator()
   280  	r.creator = nil
   281  }
   282  
   283  // peek will return the name of the next object.
   284  // Will return io.EOF if there are no more objects.
   285  // Should be used sparingly.
   286  func (r *metacacheReader) peek() (metaCacheEntry, error) {
   287  	r.checkInit()
   288  	if r.err != nil {
   289  		return metaCacheEntry{}, r.err
   290  	}
   291  	if r.current.name != "" {
   292  		return r.current, nil
   293  	}
   294  	if more, err := r.mr.ReadBool(); !more {
   295  		switch err {
   296  		case nil:
   297  			r.err = io.EOF
   298  			return metaCacheEntry{}, io.EOF
   299  		case io.EOF:
   300  			r.err = io.ErrUnexpectedEOF
   301  			return metaCacheEntry{}, io.ErrUnexpectedEOF
   302  		}
   303  		r.err = err
   304  		return metaCacheEntry{}, err
   305  	}
   306  
   307  	var err error
   308  	if r.current.name, err = r.mr.ReadString(); err != nil {
   309  		if err == io.EOF {
   310  			err = io.ErrUnexpectedEOF
   311  		}
   312  		r.err = err
   313  		return metaCacheEntry{}, err
   314  	}
   315  	r.current.metadata, err = r.mr.ReadBytes(r.current.metadata[:0])
   316  	if err == io.EOF {
   317  		err = io.ErrUnexpectedEOF
   318  	}
   319  	r.err = err
   320  	return r.current, err
   321  }
   322  
   323  // next will read one entry from the stream.
   324  // Generally not recommended for fast operation.
   325  func (r *metacacheReader) next() (metaCacheEntry, error) {
   326  	r.checkInit()
   327  	if r.err != nil {
   328  		return metaCacheEntry{}, r.err
   329  	}
   330  	var m metaCacheEntry
   331  	var err error
   332  	if r.current.name != "" {
   333  		m.name = r.current.name
   334  		m.metadata = r.current.metadata
   335  		r.current.name = ""
   336  		r.current.metadata = nil
   337  		return m, nil
   338  	}
   339  	if more, err := r.mr.ReadBool(); !more {
   340  		switch err {
   341  		case nil:
   342  			r.err = io.EOF
   343  			return m, io.EOF
   344  		case io.EOF:
   345  			r.err = io.ErrUnexpectedEOF
   346  			return m, io.ErrUnexpectedEOF
   347  		}
   348  		r.err = err
   349  		return m, err
   350  	}
   351  	if m.name, err = r.mr.ReadString(); err != nil {
   352  		if err == io.EOF {
   353  			err = io.ErrUnexpectedEOF
   354  		}
   355  		r.err = err
   356  		return m, err
   357  	}
   358  	m.metadata, err = r.mr.ReadBytes(nil)
   359  	if err == io.EOF {
   360  		err = io.ErrUnexpectedEOF
   361  	}
   362  	r.err = err
   363  	return m, err
   364  }
   365  
   366  // next will read one entry from the stream.
   367  // Generally not recommended for fast operation.
   368  func (r *metacacheReader) nextEOF() bool {
   369  	r.checkInit()
   370  	if r.err != nil {
   371  		return r.err == io.EOF
   372  	}
   373  	if r.current.name != "" {
   374  		return false
   375  	}
   376  	_, err := r.peek()
   377  	if err != nil {
   378  		r.err = err
   379  		return r.err == io.EOF
   380  	}
   381  	return false
   382  }
   383  
   384  // forwardTo will forward to the first entry that is >= s.
   385  // Will return io.EOF if end of stream is reached without finding any.
   386  func (r *metacacheReader) forwardTo(s string) error {
   387  	r.checkInit()
   388  	if r.err != nil {
   389  		return r.err
   390  	}
   391  
   392  	if s == "" {
   393  		return nil
   394  	}
   395  	if r.current.name != "" {
   396  		if r.current.name >= s {
   397  			return nil
   398  		}
   399  		r.current.name = ""
   400  		r.current.metadata = nil
   401  	}
   402  	// temporary name buffer.
   403  	var tmp = make([]byte, 0, 256)
   404  	for {
   405  		if more, err := r.mr.ReadBool(); !more {
   406  			switch err {
   407  			case nil:
   408  				r.err = io.EOF
   409  				return io.EOF
   410  			case io.EOF:
   411  				r.err = io.ErrUnexpectedEOF
   412  				return io.ErrUnexpectedEOF
   413  			}
   414  			r.err = err
   415  			return err
   416  		}
   417  		// Read name without allocating more than 1 buffer.
   418  		sz, err := r.mr.ReadStringHeader()
   419  		if err != nil {
   420  			r.err = err
   421  			return err
   422  		}
   423  		if cap(tmp) < int(sz) {
   424  			tmp = make([]byte, 0, sz+256)
   425  		}
   426  		tmp = tmp[:sz]
   427  		_, err = r.mr.R.ReadFull(tmp)
   428  		if err != nil {
   429  			r.err = err
   430  			return err
   431  		}
   432  		if string(tmp) >= s {
   433  			r.current.name = string(tmp)
   434  			r.current.metadata, r.err = r.mr.ReadBytes(nil)
   435  			return r.err
   436  		}
   437  		// Skip metadata
   438  		err = r.mr.Skip()
   439  		if err != nil {
   440  			if err == io.EOF {
   441  				err = io.ErrUnexpectedEOF
   442  			}
   443  			r.err = err
   444  			return err
   445  		}
   446  	}
   447  }
   448  
   449  // readN will return all the requested number of entries in order
   450  // or all if n < 0.
   451  // Will return io.EOF if end of stream is reached.
   452  // If requesting 0 objects nil error will always be returned regardless of at end of stream.
   453  // Use peek to determine if at end of stream.
   454  func (r *metacacheReader) readN(n int, inclDeleted, inclDirs bool, prefix string) (metaCacheEntriesSorted, error) {
   455  	r.checkInit()
   456  	if n == 0 {
   457  		return metaCacheEntriesSorted{}, nil
   458  	}
   459  	if r.err != nil {
   460  		return metaCacheEntriesSorted{}, r.err
   461  	}
   462  
   463  	var res metaCacheEntries
   464  	if n > 0 {
   465  		res = make(metaCacheEntries, 0, n)
   466  	}
   467  	if prefix != "" {
   468  		if err := r.forwardTo(prefix); err != nil {
   469  			return metaCacheEntriesSorted{}, err
   470  		}
   471  	}
   472  	next, err := r.peek()
   473  	if err != nil {
   474  		return metaCacheEntriesSorted{}, err
   475  	}
   476  	if !next.hasPrefix(prefix) {
   477  		return metaCacheEntriesSorted{}, io.EOF
   478  	}
   479  
   480  	if r.current.name != "" {
   481  		if (inclDeleted || !r.current.isLatestDeletemarker()) && r.current.hasPrefix(prefix) && (inclDirs || r.current.isObject()) {
   482  			res = append(res, r.current)
   483  		}
   484  		r.current.name = ""
   485  		r.current.metadata = nil
   486  	}
   487  
   488  	for n < 0 || len(res) < n {
   489  		if more, err := r.mr.ReadBool(); !more {
   490  			switch err {
   491  			case nil:
   492  				r.err = io.EOF
   493  				return metaCacheEntriesSorted{o: res}, io.EOF
   494  			case io.EOF:
   495  				r.err = io.ErrUnexpectedEOF
   496  				return metaCacheEntriesSorted{o: res}, io.ErrUnexpectedEOF
   497  			}
   498  			r.err = err
   499  			return metaCacheEntriesSorted{o: res}, err
   500  		}
   501  		var err error
   502  		var meta metaCacheEntry
   503  		if meta.name, err = r.mr.ReadString(); err != nil {
   504  			if err == io.EOF {
   505  				err = io.ErrUnexpectedEOF
   506  			}
   507  			r.err = err
   508  			return metaCacheEntriesSorted{o: res}, err
   509  		}
   510  		if !meta.hasPrefix(prefix) {
   511  			r.mr.R.Skip(1)
   512  			return metaCacheEntriesSorted{o: res}, io.EOF
   513  		}
   514  		if meta.metadata, err = r.mr.ReadBytes(nil); err != nil {
   515  			if err == io.EOF {
   516  				err = io.ErrUnexpectedEOF
   517  			}
   518  			r.err = err
   519  			return metaCacheEntriesSorted{o: res}, err
   520  		}
   521  		if !inclDirs && meta.isDir() {
   522  			continue
   523  		}
   524  		if !inclDeleted && meta.isLatestDeletemarker() {
   525  			continue
   526  		}
   527  		res = append(res, meta)
   528  	}
   529  	return metaCacheEntriesSorted{o: res}, nil
   530  }
   531  
   532  // readAll will return all remaining objects on the dst channel and close it when done.
   533  // The context allows the operation to be canceled.
   534  func (r *metacacheReader) readAll(ctx context.Context, dst chan<- metaCacheEntry) error {
   535  	r.checkInit()
   536  	if r.err != nil {
   537  		return r.err
   538  	}
   539  	defer close(dst)
   540  	if r.current.name != "" {
   541  		select {
   542  		case <-ctx.Done():
   543  			r.err = ctx.Err()
   544  			return ctx.Err()
   545  		case dst <- r.current:
   546  		}
   547  		r.current.name = ""
   548  		r.current.metadata = nil
   549  	}
   550  	for {
   551  		if more, err := r.mr.ReadBool(); !more {
   552  			switch err {
   553  			case io.EOF:
   554  				err = io.ErrUnexpectedEOF
   555  			}
   556  			r.err = err
   557  			return err
   558  		}
   559  
   560  		var err error
   561  		var meta metaCacheEntry
   562  		if meta.name, err = r.mr.ReadString(); err != nil {
   563  			if err == io.EOF {
   564  				err = io.ErrUnexpectedEOF
   565  			}
   566  			r.err = err
   567  			return err
   568  		}
   569  		if meta.metadata, err = r.mr.ReadBytes(nil); err != nil {
   570  			if err == io.EOF {
   571  				err = io.ErrUnexpectedEOF
   572  			}
   573  			r.err = err
   574  			return err
   575  		}
   576  		select {
   577  		case <-ctx.Done():
   578  			r.err = ctx.Err()
   579  			return ctx.Err()
   580  		case dst <- meta:
   581  		}
   582  	}
   583  }
   584  
   585  // readFn will return all remaining objects
   586  // and provide a callback for each entry read in order
   587  // as long as true is returned on the callback.
   588  func (r *metacacheReader) readFn(fn func(entry metaCacheEntry) bool) error {
   589  	r.checkInit()
   590  	if r.err != nil {
   591  		return r.err
   592  	}
   593  	if r.current.name != "" {
   594  		fn(r.current)
   595  		r.current.name = ""
   596  		r.current.metadata = nil
   597  	}
   598  	for {
   599  		if more, err := r.mr.ReadBool(); !more {
   600  			switch err {
   601  			case io.EOF:
   602  				r.err = io.ErrUnexpectedEOF
   603  				return io.ErrUnexpectedEOF
   604  			case nil:
   605  				r.err = io.EOF
   606  				return io.EOF
   607  			}
   608  			return err
   609  		}
   610  
   611  		var err error
   612  		var meta metaCacheEntry
   613  		if meta.name, err = r.mr.ReadString(); err != nil {
   614  			if err == io.EOF {
   615  				err = io.ErrUnexpectedEOF
   616  			}
   617  			r.err = err
   618  			return err
   619  		}
   620  		if meta.metadata, err = r.mr.ReadBytes(nil); err != nil {
   621  			if err == io.EOF {
   622  				err = io.ErrUnexpectedEOF
   623  			}
   624  			r.err = err
   625  			return err
   626  		}
   627  		// Send it!
   628  		if !fn(meta) {
   629  			return nil
   630  		}
   631  	}
   632  }
   633  
   634  // readNames will return all the requested number of names in order
   635  // or all if n < 0.
   636  // Will return io.EOF if end of stream is reached.
   637  func (r *metacacheReader) readNames(n int) ([]string, error) {
   638  	r.checkInit()
   639  	if r.err != nil {
   640  		return nil, r.err
   641  	}
   642  	if n == 0 {
   643  		return nil, nil
   644  	}
   645  	var res []string
   646  	if n > 0 {
   647  		res = make([]string, 0, n)
   648  	}
   649  	if r.current.name != "" {
   650  		res = append(res, r.current.name)
   651  		r.current.name = ""
   652  		r.current.metadata = nil
   653  	}
   654  	for n < 0 || len(res) < n {
   655  		if more, err := r.mr.ReadBool(); !more {
   656  			switch err {
   657  			case nil:
   658  				r.err = io.EOF
   659  				return res, io.EOF
   660  			case io.EOF:
   661  				r.err = io.ErrUnexpectedEOF
   662  				return res, io.ErrUnexpectedEOF
   663  			}
   664  			return res, err
   665  		}
   666  
   667  		var err error
   668  		var name string
   669  		if name, err = r.mr.ReadString(); err != nil {
   670  			r.err = err
   671  			return res, err
   672  		}
   673  		if err = r.mr.Skip(); err != nil {
   674  			if err == io.EOF {
   675  				err = io.ErrUnexpectedEOF
   676  			}
   677  			r.err = err
   678  			return res, err
   679  		}
   680  		res = append(res, name)
   681  	}
   682  	return res, nil
   683  }
   684  
   685  // skip n entries on the input stream.
   686  // If there are less entries left io.EOF is returned.
   687  func (r *metacacheReader) skip(n int) error {
   688  	r.checkInit()
   689  	if r.err != nil {
   690  		return r.err
   691  	}
   692  	if n <= 0 {
   693  		return nil
   694  	}
   695  	if r.current.name != "" {
   696  		n--
   697  		r.current.name = ""
   698  		r.current.metadata = nil
   699  	}
   700  	for n > 0 {
   701  		if more, err := r.mr.ReadBool(); !more {
   702  			switch err {
   703  			case nil:
   704  				r.err = io.EOF
   705  				return io.EOF
   706  			case io.EOF:
   707  				r.err = io.ErrUnexpectedEOF
   708  				return io.ErrUnexpectedEOF
   709  			}
   710  			return err
   711  		}
   712  
   713  		if err := r.mr.Skip(); err != nil {
   714  			if err == io.EOF {
   715  				err = io.ErrUnexpectedEOF
   716  			}
   717  			r.err = err
   718  			return err
   719  		}
   720  		if err := r.mr.Skip(); err != nil {
   721  			if err == io.EOF {
   722  				err = io.ErrUnexpectedEOF
   723  			}
   724  			r.err = err
   725  			return err
   726  		}
   727  		n--
   728  	}
   729  	return nil
   730  }
   731  
   732  // Close and release resources.
   733  func (r *metacacheReader) Close() error {
   734  	if r == nil || r.closer == nil {
   735  		return nil
   736  	}
   737  	r.closer()
   738  	r.closer = nil
   739  	r.creator = nil
   740  	return nil
   741  }
   742  
   743  // metacacheBlockWriter collects blocks and provides a callaback to store them.
   744  type metacacheBlockWriter struct {
   745  	wg           sync.WaitGroup
   746  	streamErr    error
   747  	blockEntries int
   748  }
   749  
   750  var bufferPool = sync.Pool{
   751  	New: func() interface{} {
   752  		return new(bytes.Buffer)
   753  	},
   754  }
   755  
   756  // newMetacacheBlockWriter provides a streaming block writer.
   757  // Each block is the size of the capacity of the input channel.
   758  // The caller should close to indicate the stream has ended.
   759  func newMetacacheBlockWriter(in <-chan metaCacheEntry, nextBlock func(b *metacacheBlock) error) *metacacheBlockWriter {
   760  	w := metacacheBlockWriter{blockEntries: cap(in)}
   761  	w.wg.Add(1)
   762  	go func() {
   763  		defer w.wg.Done()
   764  		var current metacacheBlock
   765  		var n int
   766  		buf := bufferPool.Get().(*bytes.Buffer)
   767  		defer func() {
   768  			buf.Reset()
   769  			bufferPool.Put(buf)
   770  		}()
   771  		block := newMetacacheWriter(buf, 1<<20)
   772  		defer block.Close()
   773  		finishBlock := func() {
   774  			if err := block.Close(); err != nil {
   775  				w.streamErr = err
   776  				return
   777  			}
   778  			current.data = buf.Bytes()
   779  			w.streamErr = nextBlock(&current)
   780  			// Prepare for next
   781  			current.n++
   782  			buf.Reset()
   783  			block.Reset(buf)
   784  			current.First = ""
   785  		}
   786  		for o := range in {
   787  			if len(o.name) == 0 || w.streamErr != nil {
   788  				continue
   789  			}
   790  			if current.First == "" {
   791  				current.First = o.name
   792  			}
   793  
   794  			if n >= w.blockEntries-1 {
   795  				finishBlock()
   796  				n = 0
   797  			}
   798  			n++
   799  
   800  			w.streamErr = block.write(o)
   801  			if w.streamErr != nil {
   802  				continue
   803  			}
   804  			current.Last = o.name
   805  		}
   806  		if n > 0 || current.n == 0 {
   807  			current.EOS = true
   808  			finishBlock()
   809  		}
   810  	}()
   811  	return &w
   812  }
   813  
   814  // Close the stream.
   815  // The incoming channel must be closed before calling this.
   816  // Returns the first error the occurred during the writing if any.
   817  func (w *metacacheBlockWriter) Close() error {
   818  	w.wg.Wait()
   819  	return w.streamErr
   820  }
   821  
   822  type metacacheBlock struct {
   823  	data  []byte
   824  	n     int
   825  	First string `json:"f"`
   826  	Last  string `json:"l"`
   827  	EOS   bool   `json:"eos,omitempty"`
   828  }
   829  
   830  func (b metacacheBlock) headerKV() map[string]string {
   831  	var json = jsoniter.ConfigCompatibleWithStandardLibrary
   832  	v, err := json.Marshal(b)
   833  	if err != nil {
   834  		logger.LogIf(context.Background(), err) // Unlikely
   835  		return nil
   836  	}
   837  	return map[string]string{fmt.Sprintf("%s-metacache-part-%d", ReservedMetadataPrefixLower, b.n): string(v)}
   838  }
   839  
   840  // pastPrefix returns true if the given prefix is before start of the block.
   841  func (b metacacheBlock) pastPrefix(prefix string) bool {
   842  	if prefix == "" || strings.HasPrefix(b.First, prefix) {
   843  		return false
   844  	}
   845  	// We have checked if prefix matches, so we can do direct compare.
   846  	return b.First > prefix
   847  }
   848  
   849  // endedPrefix returns true if the given prefix ends within the block.
   850  func (b metacacheBlock) endedPrefix(prefix string) bool {
   851  	if prefix == "" || strings.HasPrefix(b.Last, prefix) {
   852  		return false
   853  	}
   854  
   855  	// We have checked if prefix matches, so we can do direct compare.
   856  	return b.Last > prefix
   857  }