github.com/streamdal/segmentio-kafka-go@v0.4.47-streamdal/protocol/protocol.go (about)

     1  package protocol
     2  
     3  import (
     4  	"errors"
     5  	"fmt"
     6  	"io"
     7  	"net"
     8  	"reflect"
     9  	"strconv"
    10  	"strings"
    11  )
    12  
    13  // Message is an interface implemented by all request and response types of the
    14  // kafka protocol.
    15  //
    16  // This interface is used mostly as a safe-guard to provide a compile-time check
    17  // for values passed to functions dealing kafka message types.
    18  type Message interface {
    19  	ApiKey() ApiKey
    20  }
    21  
    22  type ApiKey int16
    23  
    24  func (k ApiKey) String() string {
    25  	if i := int(k); i >= 0 && i < len(apiNames) {
    26  		return apiNames[i]
    27  	}
    28  	return strconv.Itoa(int(k))
    29  }
    30  
    31  func (k ApiKey) MinVersion() int16 { return k.apiType().minVersion() }
    32  
    33  func (k ApiKey) MaxVersion() int16 { return k.apiType().maxVersion() }
    34  
    35  func (k ApiKey) SelectVersion(minVersion, maxVersion int16) int16 {
    36  	min := k.MinVersion()
    37  	max := k.MaxVersion()
    38  	switch {
    39  	case min > maxVersion:
    40  		return min
    41  	case max < maxVersion:
    42  		return max
    43  	default:
    44  		return maxVersion
    45  	}
    46  }
    47  
    48  func (k ApiKey) apiType() apiType {
    49  	if i := int(k); i >= 0 && i < len(apiTypes) {
    50  		return apiTypes[i]
    51  	}
    52  	return apiType{}
    53  }
    54  
    55  const (
    56  	Produce                      ApiKey = 0
    57  	Fetch                        ApiKey = 1
    58  	ListOffsets                  ApiKey = 2
    59  	Metadata                     ApiKey = 3
    60  	LeaderAndIsr                 ApiKey = 4
    61  	StopReplica                  ApiKey = 5
    62  	UpdateMetadata               ApiKey = 6
    63  	ControlledShutdown           ApiKey = 7
    64  	OffsetCommit                 ApiKey = 8
    65  	OffsetFetch                  ApiKey = 9
    66  	FindCoordinator              ApiKey = 10
    67  	JoinGroup                    ApiKey = 11
    68  	Heartbeat                    ApiKey = 12
    69  	LeaveGroup                   ApiKey = 13
    70  	SyncGroup                    ApiKey = 14
    71  	DescribeGroups               ApiKey = 15
    72  	ListGroups                   ApiKey = 16
    73  	SaslHandshake                ApiKey = 17
    74  	ApiVersions                  ApiKey = 18
    75  	CreateTopics                 ApiKey = 19
    76  	DeleteTopics                 ApiKey = 20
    77  	DeleteRecords                ApiKey = 21
    78  	InitProducerId               ApiKey = 22
    79  	OffsetForLeaderEpoch         ApiKey = 23
    80  	AddPartitionsToTxn           ApiKey = 24
    81  	AddOffsetsToTxn              ApiKey = 25
    82  	EndTxn                       ApiKey = 26
    83  	WriteTxnMarkers              ApiKey = 27
    84  	TxnOffsetCommit              ApiKey = 28
    85  	DescribeAcls                 ApiKey = 29
    86  	CreateAcls                   ApiKey = 30
    87  	DeleteAcls                   ApiKey = 31
    88  	DescribeConfigs              ApiKey = 32
    89  	AlterConfigs                 ApiKey = 33
    90  	AlterReplicaLogDirs          ApiKey = 34
    91  	DescribeLogDirs              ApiKey = 35
    92  	SaslAuthenticate             ApiKey = 36
    93  	CreatePartitions             ApiKey = 37
    94  	CreateDelegationToken        ApiKey = 38
    95  	RenewDelegationToken         ApiKey = 39
    96  	ExpireDelegationToken        ApiKey = 40
    97  	DescribeDelegationToken      ApiKey = 41
    98  	DeleteGroups                 ApiKey = 42
    99  	ElectLeaders                 ApiKey = 43
   100  	IncrementalAlterConfigs      ApiKey = 44
   101  	AlterPartitionReassignments  ApiKey = 45
   102  	ListPartitionReassignments   ApiKey = 46
   103  	OffsetDelete                 ApiKey = 47
   104  	DescribeClientQuotas         ApiKey = 48
   105  	AlterClientQuotas            ApiKey = 49
   106  	DescribeUserScramCredentials ApiKey = 50
   107  	AlterUserScramCredentials    ApiKey = 51
   108  
   109  	numApis = 52
   110  )
   111  
   112  var apiNames = [numApis]string{
   113  	Produce:                      "Produce",
   114  	Fetch:                        "Fetch",
   115  	ListOffsets:                  "ListOffsets",
   116  	Metadata:                     "Metadata",
   117  	LeaderAndIsr:                 "LeaderAndIsr",
   118  	StopReplica:                  "StopReplica",
   119  	UpdateMetadata:               "UpdateMetadata",
   120  	ControlledShutdown:           "ControlledShutdown",
   121  	OffsetCommit:                 "OffsetCommit",
   122  	OffsetFetch:                  "OffsetFetch",
   123  	FindCoordinator:              "FindCoordinator",
   124  	JoinGroup:                    "JoinGroup",
   125  	Heartbeat:                    "Heartbeat",
   126  	LeaveGroup:                   "LeaveGroup",
   127  	SyncGroup:                    "SyncGroup",
   128  	DescribeGroups:               "DescribeGroups",
   129  	ListGroups:                   "ListGroups",
   130  	SaslHandshake:                "SaslHandshake",
   131  	ApiVersions:                  "ApiVersions",
   132  	CreateTopics:                 "CreateTopics",
   133  	DeleteTopics:                 "DeleteTopics",
   134  	DeleteRecords:                "DeleteRecords",
   135  	InitProducerId:               "InitProducerId",
   136  	OffsetForLeaderEpoch:         "OffsetForLeaderEpoch",
   137  	AddPartitionsToTxn:           "AddPartitionsToTxn",
   138  	AddOffsetsToTxn:              "AddOffsetsToTxn",
   139  	EndTxn:                       "EndTxn",
   140  	WriteTxnMarkers:              "WriteTxnMarkers",
   141  	TxnOffsetCommit:              "TxnOffsetCommit",
   142  	DescribeAcls:                 "DescribeAcls",
   143  	CreateAcls:                   "CreateAcls",
   144  	DeleteAcls:                   "DeleteAcls",
   145  	DescribeConfigs:              "DescribeConfigs",
   146  	AlterConfigs:                 "AlterConfigs",
   147  	AlterReplicaLogDirs:          "AlterReplicaLogDirs",
   148  	DescribeLogDirs:              "DescribeLogDirs",
   149  	SaslAuthenticate:             "SaslAuthenticate",
   150  	CreatePartitions:             "CreatePartitions",
   151  	CreateDelegationToken:        "CreateDelegationToken",
   152  	RenewDelegationToken:         "RenewDelegationToken",
   153  	ExpireDelegationToken:        "ExpireDelegationToken",
   154  	DescribeDelegationToken:      "DescribeDelegationToken",
   155  	DeleteGroups:                 "DeleteGroups",
   156  	ElectLeaders:                 "ElectLeaders",
   157  	IncrementalAlterConfigs:      "IncrementalAlterConfigs",
   158  	AlterPartitionReassignments:  "AlterPartitionReassignments",
   159  	ListPartitionReassignments:   "ListPartitionReassignments",
   160  	OffsetDelete:                 "OffsetDelete",
   161  	DescribeClientQuotas:         "DescribeClientQuotas",
   162  	AlterClientQuotas:            "AlterClientQuotas",
   163  	DescribeUserScramCredentials: "DescribeUserScramCredentials",
   164  	AlterUserScramCredentials:    "AlterUserScramCredentials",
   165  }
   166  
   167  type messageType struct {
   168  	version  int16
   169  	flexible bool
   170  	gotype   reflect.Type
   171  	decode   decodeFunc
   172  	encode   encodeFunc
   173  }
   174  
   175  func (t *messageType) new() Message {
   176  	return reflect.New(t.gotype).Interface().(Message)
   177  }
   178  
   179  type apiType struct {
   180  	requests  []messageType
   181  	responses []messageType
   182  }
   183  
   184  func (t apiType) minVersion() int16 {
   185  	if len(t.requests) == 0 {
   186  		return 0
   187  	}
   188  	return t.requests[0].version
   189  }
   190  
   191  func (t apiType) maxVersion() int16 {
   192  	if len(t.requests) == 0 {
   193  		return 0
   194  	}
   195  	return t.requests[len(t.requests)-1].version
   196  }
   197  
   198  var apiTypes [numApis]apiType
   199  
   200  // Register is automatically called by sub-packages are imported to install a
   201  // new pair of request/response message types.
   202  func Register(req, res Message) {
   203  	k1 := req.ApiKey()
   204  	k2 := res.ApiKey()
   205  
   206  	if k1 != k2 {
   207  		panic(fmt.Sprintf("[%T/%T]: request and response API keys mismatch: %d != %d", req, res, k1, k2))
   208  	}
   209  
   210  	apiTypes[k1] = apiType{
   211  		requests:  typesOf(req),
   212  		responses: typesOf(res),
   213  	}
   214  }
   215  
   216  // OverrideTypeMessage is an interface implemented by messages that want to override the standard
   217  // request/response types for a given API.
   218  type OverrideTypeMessage interface {
   219  	TypeKey() OverrideTypeKey
   220  }
   221  
   222  type OverrideTypeKey int16
   223  
   224  const (
   225  	RawProduceOverride OverrideTypeKey = 0
   226  )
   227  
   228  var overrideApiTypes [numApis]map[OverrideTypeKey]apiType
   229  
   230  func RegisterOverride(req, res Message, key OverrideTypeKey) {
   231  	k1 := req.ApiKey()
   232  	k2 := res.ApiKey()
   233  
   234  	if k1 != k2 {
   235  		panic(fmt.Sprintf("[%T/%T]: request and response API keys mismatch: %d != %d", req, res, k1, k2))
   236  	}
   237  
   238  	if overrideApiTypes[k1] == nil {
   239  		overrideApiTypes[k1] = make(map[OverrideTypeKey]apiType)
   240  	}
   241  	overrideApiTypes[k1][key] = apiType{
   242  		requests:  typesOf(req),
   243  		responses: typesOf(res),
   244  	}
   245  }
   246  
   247  func typesOf(v interface{}) []messageType {
   248  	return makeTypes(reflect.TypeOf(v).Elem())
   249  }
   250  
   251  func makeTypes(t reflect.Type) []messageType {
   252  	minVersion := int16(-1)
   253  	maxVersion := int16(-1)
   254  
   255  	// All future versions will be flexible (according to spec), so don't need to
   256  	// worry about maxes here.
   257  	minFlexibleVersion := int16(-1)
   258  
   259  	forEachStructField(t, func(_ reflect.Type, _ index, tag string) {
   260  		forEachStructTag(tag, func(tag structTag) bool {
   261  			if minVersion < 0 || tag.MinVersion < minVersion {
   262  				minVersion = tag.MinVersion
   263  			}
   264  			if maxVersion < 0 || tag.MaxVersion > maxVersion {
   265  				maxVersion = tag.MaxVersion
   266  			}
   267  			if tag.TagID > -2 && (minFlexibleVersion < 0 || tag.MinVersion < minFlexibleVersion) {
   268  				minFlexibleVersion = tag.MinVersion
   269  			}
   270  			return true
   271  		})
   272  	})
   273  
   274  	types := make([]messageType, 0, (maxVersion-minVersion)+1)
   275  
   276  	for v := minVersion; v <= maxVersion; v++ {
   277  		flexible := minFlexibleVersion >= 0 && v >= minFlexibleVersion
   278  
   279  		types = append(types, messageType{
   280  			version:  v,
   281  			gotype:   t,
   282  			flexible: flexible,
   283  			decode:   decodeFuncOf(t, v, flexible, structTag{}),
   284  			encode:   encodeFuncOf(t, v, flexible, structTag{}),
   285  		})
   286  	}
   287  
   288  	return types
   289  }
   290  
   291  type structTag struct {
   292  	MinVersion int16
   293  	MaxVersion int16
   294  	Compact    bool
   295  	Nullable   bool
   296  	TagID      int
   297  }
   298  
   299  func forEachStructTag(tag string, do func(structTag) bool) {
   300  	if tag == "-" {
   301  		return // special case to ignore the field
   302  	}
   303  
   304  	forEach(tag, '|', func(s string) bool {
   305  		tag := structTag{
   306  			MinVersion: -1,
   307  			MaxVersion: -1,
   308  
   309  			// Legitimate tag IDs can start at 0. We use -1 as a placeholder to indicate
   310  			// that the message type is flexible, so that leaves -2 as the default for
   311  			// indicating that there is no tag ID and the message is not flexible.
   312  			TagID: -2,
   313  		}
   314  
   315  		var err error
   316  		forEach(s, ',', func(s string) bool {
   317  			switch {
   318  			case strings.HasPrefix(s, "min="):
   319  				tag.MinVersion, err = parseVersion(s[4:])
   320  			case strings.HasPrefix(s, "max="):
   321  				tag.MaxVersion, err = parseVersion(s[4:])
   322  			case s == "tag":
   323  				tag.TagID = -1
   324  			case strings.HasPrefix(s, "tag="):
   325  				tag.TagID, err = strconv.Atoi(s[4:])
   326  			case s == "compact":
   327  				tag.Compact = true
   328  			case s == "nullable":
   329  				tag.Nullable = true
   330  			default:
   331  				err = fmt.Errorf("unrecognized option: %q", s)
   332  			}
   333  			return err == nil
   334  		})
   335  
   336  		if err != nil {
   337  			panic(fmt.Errorf("malformed struct tag: %w", err))
   338  		}
   339  
   340  		if tag.MinVersion < 0 && tag.MaxVersion >= 0 {
   341  			panic(fmt.Errorf("missing minimum version in struct tag: %q", s))
   342  		}
   343  
   344  		if tag.MaxVersion < 0 && tag.MinVersion >= 0 {
   345  			panic(fmt.Errorf("missing maximum version in struct tag: %q", s))
   346  		}
   347  
   348  		if tag.MinVersion > tag.MaxVersion {
   349  			panic(fmt.Errorf("invalid version range in struct tag: %q", s))
   350  		}
   351  
   352  		return do(tag)
   353  	})
   354  }
   355  
   356  func forEach(s string, sep byte, do func(string) bool) bool {
   357  	for len(s) != 0 {
   358  		p := ""
   359  		i := strings.IndexByte(s, sep)
   360  		if i < 0 {
   361  			p, s = s, ""
   362  		} else {
   363  			p, s = s[:i], s[i+1:]
   364  		}
   365  		if !do(p) {
   366  			return false
   367  		}
   368  	}
   369  	return true
   370  }
   371  
   372  func forEachStructField(t reflect.Type, do func(reflect.Type, index, string)) {
   373  	for i, n := 0, t.NumField(); i < n; i++ {
   374  		f := t.Field(i)
   375  
   376  		if f.PkgPath != "" && f.Name != "_" {
   377  			continue
   378  		}
   379  
   380  		kafkaTag, ok := f.Tag.Lookup("kafka")
   381  		if !ok {
   382  			kafkaTag = "|"
   383  		}
   384  
   385  		do(f.Type, indexOf(f), kafkaTag)
   386  	}
   387  }
   388  
   389  func parseVersion(s string) (int16, error) {
   390  	if !strings.HasPrefix(s, "v") {
   391  		return 0, fmt.Errorf("invalid version number: %q", s)
   392  	}
   393  	i, err := strconv.ParseInt(s[1:], 10, 16)
   394  	if err != nil {
   395  		return 0, fmt.Errorf("invalid version number: %q: %w", s, err)
   396  	}
   397  	if i < 0 {
   398  		return 0, fmt.Errorf("invalid negative version number: %q", s)
   399  	}
   400  	return int16(i), nil
   401  }
   402  
   403  func dontExpectEOF(err error) error {
   404  	if err != nil {
   405  		if errors.Is(err, io.EOF) {
   406  			return io.ErrUnexpectedEOF
   407  		}
   408  
   409  		return err
   410  	}
   411  
   412  	return nil
   413  }
   414  
   415  type Broker struct {
   416  	Rack string
   417  	Host string
   418  	Port int32
   419  	ID   int32
   420  }
   421  
   422  func (b Broker) String() string {
   423  	return net.JoinHostPort(b.Host, itoa(b.Port))
   424  }
   425  
   426  func (b Broker) Format(w fmt.State, v rune) {
   427  	switch v {
   428  	case 'd':
   429  		io.WriteString(w, itoa(b.ID))
   430  	case 's':
   431  		io.WriteString(w, b.String())
   432  	case 'v':
   433  		io.WriteString(w, itoa(b.ID))
   434  		io.WriteString(w, " ")
   435  		io.WriteString(w, b.String())
   436  		if b.Rack != "" {
   437  			io.WriteString(w, " ")
   438  			io.WriteString(w, b.Rack)
   439  		}
   440  	}
   441  }
   442  
   443  func itoa(i int32) string {
   444  	return strconv.Itoa(int(i))
   445  }
   446  
   447  type Topic struct {
   448  	Name       string
   449  	Error      int16
   450  	Partitions map[int32]Partition
   451  }
   452  
   453  type Partition struct {
   454  	ID       int32
   455  	Error    int16
   456  	Leader   int32
   457  	Replicas []int32
   458  	ISR      []int32
   459  	Offline  []int32
   460  }
   461  
   462  // RawExchanger is an extention to the Message interface to allow messages
   463  // to control the request response cycle for the message. This is currently
   464  // only used to facilitate v0 SASL Authenticate requests being written in
   465  // a non-standard fashion when the SASL Handshake was done at v0 but not
   466  // when done at v1.
   467  type RawExchanger interface {
   468  	// Required should return true when a RawExchange is needed.
   469  	// The passed in versions are the negotiated versions for the connection
   470  	// performing the request.
   471  	Required(versions map[ApiKey]int16) bool
   472  	// RawExchange is given the raw connection to the broker and the Message
   473  	// is responsible for writing itself to the connection as well as reading
   474  	// the response.
   475  	RawExchange(rw io.ReadWriter) (Message, error)
   476  }
   477  
   478  // BrokerMessage is an extension of the Message interface implemented by some
   479  // request types to customize the broker assignment logic.
   480  type BrokerMessage interface {
   481  	// Given a representation of the kafka cluster state as argument, returns
   482  	// the broker that the message should be routed to.
   483  	Broker(Cluster) (Broker, error)
   484  }
   485  
   486  // GroupMessage is an extension of the Message interface implemented by some
   487  // request types to inform the program that they should be routed to a group
   488  // coordinator.
   489  type GroupMessage interface {
   490  	// Returns the group configured on the message.
   491  	Group() string
   492  }
   493  
   494  // TransactionalMessage is an extension of the Message interface implemented by some
   495  // request types to inform the program that they should be routed to a transaction
   496  // coordinator.
   497  type TransactionalMessage interface {
   498  	// Returns the transactional id configured on the message.
   499  	Transaction() string
   500  }
   501  
   502  // PreparedMessage is an extension of the Message interface implemented by some
   503  // request types which may need to run some pre-processing on their state before
   504  // being sent.
   505  type PreparedMessage interface {
   506  	// Prepares the message before being sent to a kafka broker using the API
   507  	// version passed as argument.
   508  	Prepare(apiVersion int16)
   509  }
   510  
   511  // Splitter is an interface implemented by messages that can be split into
   512  // multiple requests and have their results merged back by a Merger.
   513  type Splitter interface {
   514  	// For a given cluster layout, returns the list of messages constructed
   515  	// from the receiver for each requests that should be sent to the cluster.
   516  	// The second return value is a Merger which can be used to merge back the
   517  	// results of each request into a single message (or an error).
   518  	Split(Cluster) ([]Message, Merger, error)
   519  }
   520  
   521  // Merger is an interface implemented by messages which can merge multiple
   522  // results into one response.
   523  type Merger interface {
   524  	// Given a list of message and associated results, merge them back into a
   525  	// response (or an error). The results must be either Message or error
   526  	// values, other types should trigger a panic.
   527  	Merge(messages []Message, results []interface{}) (Message, error)
   528  }
   529  
   530  // Result converts r to a Message or an error, or panics if r could not be
   531  // converted to these types.
   532  func Result(r interface{}) (Message, error) {
   533  	switch v := r.(type) {
   534  	case Message:
   535  		return v, nil
   536  	case error:
   537  		return nil, v
   538  	default:
   539  		panic(fmt.Errorf("BUG: result must be a message or an error but not %T", v))
   540  	}
   541  }