github.com/grailbio/base@v0.0.11/recordio/recordio.go (about)

     1  package recordio
     2  
     3  import (
     4  	"github.com/grailbio/base/recordio/internal"
     5  )
     6  
     7  // TransformFunc is called to (un)compress or (un)encrypt data. Parameter
     8  // scratch is passed as an performance hint. If the result of the transformation
     9  // fits in scratch, the function should store the result in scratch and return
    10  // it as the first return value. Else, it should allocate a new []byte and
    11  // return it.
    12  type TransformFunc func(scratch []byte, in [][]byte) (out []byte, err error)
    13  
    14  // FormatVersion defines the file-format version. Not for general use.  It may
    15  // be removed without notice.
    16  type FormatVersion int
    17  
    18  const (
    19  	// V1 is pre 2018-02 format
    20  	V1 FormatVersion = 1
    21  	// V2 is post 2018-02 format
    22  	V2 FormatVersion = 2
    23  )
    24  
    25  // MaxReadRecordSize defines a max size for a record when reading to avoid
    26  // crashes for unreasonable requests.
    27  var MaxReadRecordSize = internal.MaxReadRecordSize
    28  
    29  // MarshalFunc is called to serialize data.  Parameter scratch is passed as an
    30  // performance hint. If the result of the transformation fits in scratch, the
    31  // function should store the result in scratch and return it as the first return
    32  // value. Else, it should allocate a new []byte and return it.
    33  type MarshalFunc func(scratch []byte, v interface{}) ([]byte, error)
    34  
    35  // MagicPacked is the chunk header for legacy and v2 data chunks. Not for
    36  // general use.
    37  var MagicPacked = internal.MagicPacked