github.com/streamdal/segmentio-kafka-go@v0.4.47-streamdal/protocol/record_batch.go (about)

     1  package protocol
     2  
     3  import (
     4  	"errors"
     5  	"io"
     6  	"time"
     7  )
     8  
     9  // RecordReader is an interface representing a sequence of records. Record sets
    10  // are used in both produce and fetch requests to represent the sequence of
    11  // records that are sent to or receive from kafka brokers.
    12  //
    13  // RecordSet values are not safe to use concurrently from multiple goroutines.
    14  type RecordReader interface {
    15  	// Returns the next record in the set, or io.EOF if the end of the sequence
    16  	// has been reached.
    17  	//
    18  	// The returned Record is guaranteed to be valid until the next call to
    19  	// ReadRecord. If the program needs to retain the Record value it must make
    20  	// a copy.
    21  	ReadRecord() (*Record, error)
    22  }
    23  
    24  // NewRecordReader constructs a reader exposing the records passed as arguments.
    25  func NewRecordReader(records ...Record) RecordReader {
    26  	switch len(records) {
    27  	case 0:
    28  		return emptyRecordReader{}
    29  	default:
    30  		r := &recordReader{records: make([]Record, len(records))}
    31  		copy(r.records, records)
    32  		return r
    33  	}
    34  }
    35  
    36  // MultiRecordReader merges multiple record batches into one.
    37  func MultiRecordReader(batches ...RecordReader) RecordReader {
    38  	switch len(batches) {
    39  	case 0:
    40  		return emptyRecordReader{}
    41  	case 1:
    42  		return batches[0]
    43  	default:
    44  		m := &multiRecordReader{batches: make([]RecordReader, len(batches))}
    45  		copy(m.batches, batches)
    46  		return m
    47  	}
    48  }
    49  
    50  func forEachRecord(r RecordReader, f func(int, *Record) error) error {
    51  	for i := 0; ; i++ {
    52  		rec, err := r.ReadRecord()
    53  
    54  		if err != nil {
    55  			if errors.Is(err, io.EOF) {
    56  				err = nil
    57  			}
    58  			return err
    59  		}
    60  
    61  		if err := handleRecord(i, rec, f); err != nil {
    62  			return err
    63  		}
    64  	}
    65  }
    66  
    67  func handleRecord(i int, r *Record, f func(int, *Record) error) error {
    68  	if r.Key != nil {
    69  		defer r.Key.Close()
    70  	}
    71  	if r.Value != nil {
    72  		defer r.Value.Close()
    73  	}
    74  	return f(i, r)
    75  }
    76  
    77  type recordReader struct {
    78  	records []Record
    79  	index   int
    80  }
    81  
    82  func (r *recordReader) ReadRecord() (*Record, error) {
    83  	if i := r.index; i >= 0 && i < len(r.records) {
    84  		r.index++
    85  		return &r.records[i], nil
    86  	}
    87  	return nil, io.EOF
    88  }
    89  
    90  type multiRecordReader struct {
    91  	batches []RecordReader
    92  	index   int
    93  }
    94  
    95  func (m *multiRecordReader) ReadRecord() (*Record, error) {
    96  	for {
    97  		if m.index == len(m.batches) {
    98  			return nil, io.EOF
    99  		}
   100  		r, err := m.batches[m.index].ReadRecord()
   101  		if err == nil {
   102  			return r, nil
   103  		}
   104  		if !errors.Is(err, io.EOF) {
   105  			return nil, err
   106  		}
   107  		m.index++
   108  	}
   109  }
   110  
   111  // optimizedRecordReader is an implementation of a RecordReader which exposes a
   112  // sequence.
   113  type optimizedRecordReader struct {
   114  	records []optimizedRecord
   115  	index   int
   116  	buffer  Record
   117  	headers [][]Header
   118  }
   119  
   120  func (r *optimizedRecordReader) ReadRecord() (*Record, error) {
   121  	if i := r.index; i >= 0 && i < len(r.records) {
   122  		rec := &r.records[i]
   123  		r.index++
   124  		r.buffer = Record{
   125  			Offset: rec.offset,
   126  			Time:   rec.time(),
   127  			Key:    rec.key(),
   128  			Value:  rec.value(),
   129  		}
   130  		if i < len(r.headers) {
   131  			r.buffer.Headers = r.headers[i]
   132  		}
   133  		return &r.buffer, nil
   134  	}
   135  	return nil, io.EOF
   136  }
   137  
   138  type optimizedRecord struct {
   139  	offset    int64
   140  	timestamp int64
   141  	keyRef    *pageRef
   142  	valueRef  *pageRef
   143  }
   144  
   145  func (r *optimizedRecord) time() time.Time {
   146  	return makeTime(r.timestamp)
   147  }
   148  
   149  func (r *optimizedRecord) key() Bytes {
   150  	return makeBytes(r.keyRef)
   151  }
   152  
   153  func (r *optimizedRecord) value() Bytes {
   154  	return makeBytes(r.valueRef)
   155  }
   156  
   157  func makeBytes(ref *pageRef) Bytes {
   158  	if ref == nil {
   159  		return nil
   160  	}
   161  	return ref
   162  }
   163  
   164  type emptyRecordReader struct{}
   165  
   166  func (emptyRecordReader) ReadRecord() (*Record, error) { return nil, io.EOF }
   167  
   168  // ControlRecord represents a record read from a control batch.
   169  type ControlRecord struct {
   170  	Offset  int64
   171  	Time    time.Time
   172  	Version int16
   173  	Type    int16
   174  	Data    []byte
   175  	Headers []Header
   176  }
   177  
   178  func ReadControlRecord(r *Record) (*ControlRecord, error) {
   179  	if r.Key != nil {
   180  		defer r.Key.Close()
   181  	}
   182  	if r.Value != nil {
   183  		defer r.Value.Close()
   184  	}
   185  
   186  	k, err := ReadAll(r.Key)
   187  	if err != nil {
   188  		return nil, err
   189  	}
   190  	if k == nil {
   191  		return nil, Error("invalid control record with nil key")
   192  	}
   193  	if len(k) != 4 {
   194  		return nil, Errorf("invalid control record with key of size %d", len(k))
   195  	}
   196  
   197  	v, err := ReadAll(r.Value)
   198  	if err != nil {
   199  		return nil, err
   200  	}
   201  
   202  	c := &ControlRecord{
   203  		Offset:  r.Offset,
   204  		Time:    r.Time,
   205  		Version: readInt16(k[:2]),
   206  		Type:    readInt16(k[2:]),
   207  		Data:    v,
   208  		Headers: r.Headers,
   209  	}
   210  
   211  	return c, nil
   212  }
   213  
   214  func (cr *ControlRecord) Key() Bytes {
   215  	k := make([]byte, 4)
   216  	writeInt16(k[:2], cr.Version)
   217  	writeInt16(k[2:], cr.Type)
   218  	return NewBytes(k)
   219  }
   220  
   221  func (cr *ControlRecord) Value() Bytes {
   222  	return NewBytes(cr.Data)
   223  }
   224  
   225  func (cr *ControlRecord) Record() Record {
   226  	return Record{
   227  		Offset:  cr.Offset,
   228  		Time:    cr.Time,
   229  		Key:     cr.Key(),
   230  		Value:   cr.Value(),
   231  		Headers: cr.Headers,
   232  	}
   233  }
   234  
   235  // ControlBatch is an implementation of the RecordReader interface representing
   236  // control batches returned by kafka brokers.
   237  type ControlBatch struct {
   238  	Attributes           Attributes
   239  	PartitionLeaderEpoch int32
   240  	BaseOffset           int64
   241  	ProducerID           int64
   242  	ProducerEpoch        int16
   243  	BaseSequence         int32
   244  	Records              RecordReader
   245  }
   246  
   247  // NewControlBatch constructs a control batch from the list of records passed as
   248  // arguments.
   249  func NewControlBatch(records ...ControlRecord) *ControlBatch {
   250  	rawRecords := make([]Record, len(records))
   251  	for i, cr := range records {
   252  		rawRecords[i] = cr.Record()
   253  	}
   254  	return &ControlBatch{
   255  		Records: NewRecordReader(rawRecords...),
   256  	}
   257  }
   258  
   259  func (c *ControlBatch) ReadRecord() (*Record, error) {
   260  	return c.Records.ReadRecord()
   261  }
   262  
   263  func (c *ControlBatch) ReadControlRecord() (*ControlRecord, error) {
   264  	r, err := c.ReadRecord()
   265  	if err != nil {
   266  		return nil, err
   267  	}
   268  	if r.Key != nil {
   269  		defer r.Key.Close()
   270  	}
   271  	if r.Value != nil {
   272  		defer r.Value.Close()
   273  	}
   274  	return ReadControlRecord(r)
   275  }
   276  
   277  func (c *ControlBatch) Offset() int64 {
   278  	return c.BaseOffset
   279  }
   280  
   281  func (c *ControlBatch) Version() int {
   282  	return 2
   283  }
   284  
   285  // RecordBatch is an implementation of the RecordReader interface representing
   286  // regular record batches (v2).
   287  type RecordBatch struct {
   288  	Attributes           Attributes
   289  	PartitionLeaderEpoch int32
   290  	BaseOffset           int64
   291  	ProducerID           int64
   292  	ProducerEpoch        int16
   293  	BaseSequence         int32
   294  	Records              RecordReader
   295  }
   296  
   297  func (r *RecordBatch) ReadRecord() (*Record, error) {
   298  	return r.Records.ReadRecord()
   299  }
   300  
   301  func (r *RecordBatch) Offset() int64 {
   302  	return r.BaseOffset
   303  }
   304  
   305  func (r *RecordBatch) Version() int {
   306  	return 2
   307  }
   308  
   309  // MessageSet is an implementation of the RecordReader interface representing
   310  // regular message sets (v1).
   311  type MessageSet struct {
   312  	Attributes Attributes
   313  	BaseOffset int64
   314  	Records    RecordReader
   315  }
   316  
   317  func (m *MessageSet) ReadRecord() (*Record, error) {
   318  	return m.Records.ReadRecord()
   319  }
   320  
   321  func (m *MessageSet) Offset() int64 {
   322  	return m.BaseOffset
   323  }
   324  
   325  func (m *MessageSet) Version() int {
   326  	return 1
   327  }
   328  
   329  // RecordStream is an implementation of the RecordReader interface which
   330  // combines multiple underlying RecordReader and only expose records that
   331  // are not from control batches.
   332  type RecordStream struct {
   333  	Records []RecordReader
   334  	index   int
   335  }
   336  
   337  func (s *RecordStream) ReadRecord() (*Record, error) {
   338  	for {
   339  		if s.index < 0 || s.index >= len(s.Records) {
   340  			return nil, io.EOF
   341  		}
   342  
   343  		if _, isControl := s.Records[s.index].(*ControlBatch); isControl {
   344  			s.index++
   345  			continue
   346  		}
   347  
   348  		r, err := s.Records[s.index].ReadRecord()
   349  		if err != nil {
   350  			if errors.Is(err, io.EOF) {
   351  				s.index++
   352  				continue
   353  			}
   354  		}
   355  
   356  		return r, err
   357  	}
   358  }