github.com/hoveychen/kafka-go@v0.4.42/protocol/protocol.go (about)

     1  package protocol
     2  
     3  import (
     4  	"errors"
     5  	"fmt"
     6  	"io"
     7  	"net"
     8  	"reflect"
     9  	"strconv"
    10  	"strings"
    11  )
    12  
    13  // Message is an interface implemented by all request and response types of the
    14  // kafka protocol.
    15  //
    16  // This interface is used mostly as a safe-guard to provide a compile-time check
    17  // for values passed to functions dealing kafka message types.
    18  type Message interface {
    19  	ApiKey() ApiKey
    20  }
    21  
    22  type ApiKey int16
    23  
    24  func (k ApiKey) String() string {
    25  	if i := int(k); i >= 0 && i < len(apiNames) {
    26  		return apiNames[i]
    27  	}
    28  	return strconv.Itoa(int(k))
    29  }
    30  
    31  func (k ApiKey) MinVersion() int16 { return k.apiType().minVersion() }
    32  
    33  func (k ApiKey) MaxVersion() int16 { return k.apiType().maxVersion() }
    34  
    35  func (k ApiKey) SelectVersion(minVersion, maxVersion int16) int16 {
    36  	min := k.MinVersion()
    37  	max := k.MaxVersion()
    38  	switch {
    39  	case min > maxVersion:
    40  		return min
    41  	case max < maxVersion:
    42  		return max
    43  	default:
    44  		return maxVersion
    45  	}
    46  }
    47  
    48  func (k ApiKey) apiType() apiType {
    49  	if i := int(k); i >= 0 && i < len(apiTypes) {
    50  		return apiTypes[i]
    51  	}
    52  	return apiType{}
    53  }
    54  
    55  const (
    56  	Produce                     ApiKey = 0
    57  	Fetch                       ApiKey = 1
    58  	ListOffsets                 ApiKey = 2
    59  	Metadata                    ApiKey = 3
    60  	LeaderAndIsr                ApiKey = 4
    61  	StopReplica                 ApiKey = 5
    62  	UpdateMetadata              ApiKey = 6
    63  	ControlledShutdown          ApiKey = 7
    64  	OffsetCommit                ApiKey = 8
    65  	OffsetFetch                 ApiKey = 9
    66  	FindCoordinator             ApiKey = 10
    67  	JoinGroup                   ApiKey = 11
    68  	Heartbeat                   ApiKey = 12
    69  	LeaveGroup                  ApiKey = 13
    70  	SyncGroup                   ApiKey = 14
    71  	DescribeGroups              ApiKey = 15
    72  	ListGroups                  ApiKey = 16
    73  	SaslHandshake               ApiKey = 17
    74  	ApiVersions                 ApiKey = 18
    75  	CreateTopics                ApiKey = 19
    76  	DeleteTopics                ApiKey = 20
    77  	DeleteRecords               ApiKey = 21
    78  	InitProducerId              ApiKey = 22
    79  	OffsetForLeaderEpoch        ApiKey = 23
    80  	AddPartitionsToTxn          ApiKey = 24
    81  	AddOffsetsToTxn             ApiKey = 25
    82  	EndTxn                      ApiKey = 26
    83  	WriteTxnMarkers             ApiKey = 27
    84  	TxnOffsetCommit             ApiKey = 28
    85  	DescribeAcls                ApiKey = 29
    86  	CreateAcls                  ApiKey = 30
    87  	DeleteAcls                  ApiKey = 31
    88  	DescribeConfigs             ApiKey = 32
    89  	AlterConfigs                ApiKey = 33
    90  	AlterReplicaLogDirs         ApiKey = 34
    91  	DescribeLogDirs             ApiKey = 35
    92  	SaslAuthenticate            ApiKey = 36
    93  	CreatePartitions            ApiKey = 37
    94  	CreateDelegationToken       ApiKey = 38
    95  	RenewDelegationToken        ApiKey = 39
    96  	ExpireDelegationToken       ApiKey = 40
    97  	DescribeDelegationToken     ApiKey = 41
    98  	DeleteGroups                ApiKey = 42
    99  	ElectLeaders                ApiKey = 43
   100  	IncrementalAlterConfigs     ApiKey = 44
   101  	AlterPartitionReassignments ApiKey = 45
   102  	ListPartitionReassignments  ApiKey = 46
   103  	OffsetDelete                ApiKey = 47
   104  	DescribeClientQuotas        ApiKey = 48
   105  	AlterClientQuotas           ApiKey = 49
   106  
   107  	numApis = 50
   108  )
   109  
   110  var apiNames = [numApis]string{
   111  	Produce:                     "Produce",
   112  	Fetch:                       "Fetch",
   113  	ListOffsets:                 "ListOffsets",
   114  	Metadata:                    "Metadata",
   115  	LeaderAndIsr:                "LeaderAndIsr",
   116  	StopReplica:                 "StopReplica",
   117  	UpdateMetadata:              "UpdateMetadata",
   118  	ControlledShutdown:          "ControlledShutdown",
   119  	OffsetCommit:                "OffsetCommit",
   120  	OffsetFetch:                 "OffsetFetch",
   121  	FindCoordinator:             "FindCoordinator",
   122  	JoinGroup:                   "JoinGroup",
   123  	Heartbeat:                   "Heartbeat",
   124  	LeaveGroup:                  "LeaveGroup",
   125  	SyncGroup:                   "SyncGroup",
   126  	DescribeGroups:              "DescribeGroups",
   127  	ListGroups:                  "ListGroups",
   128  	SaslHandshake:               "SaslHandshake",
   129  	ApiVersions:                 "ApiVersions",
   130  	CreateTopics:                "CreateTopics",
   131  	DeleteTopics:                "DeleteTopics",
   132  	DeleteRecords:               "DeleteRecords",
   133  	InitProducerId:              "InitProducerId",
   134  	OffsetForLeaderEpoch:        "OffsetForLeaderEpoch",
   135  	AddPartitionsToTxn:          "AddPartitionsToTxn",
   136  	AddOffsetsToTxn:             "AddOffsetsToTxn",
   137  	EndTxn:                      "EndTxn",
   138  	WriteTxnMarkers:             "WriteTxnMarkers",
   139  	TxnOffsetCommit:             "TxnOffsetCommit",
   140  	DescribeAcls:                "DescribeAcls",
   141  	CreateAcls:                  "CreateAcls",
   142  	DeleteAcls:                  "DeleteAcls",
   143  	DescribeConfigs:             "DescribeConfigs",
   144  	AlterConfigs:                "AlterConfigs",
   145  	AlterReplicaLogDirs:         "AlterReplicaLogDirs",
   146  	DescribeLogDirs:             "DescribeLogDirs",
   147  	SaslAuthenticate:            "SaslAuthenticate",
   148  	CreatePartitions:            "CreatePartitions",
   149  	CreateDelegationToken:       "CreateDelegationToken",
   150  	RenewDelegationToken:        "RenewDelegationToken",
   151  	ExpireDelegationToken:       "ExpireDelegationToken",
   152  	DescribeDelegationToken:     "DescribeDelegationToken",
   153  	DeleteGroups:                "DeleteGroups",
   154  	ElectLeaders:                "ElectLeaders",
   155  	IncrementalAlterConfigs:     "IncrementalAlterConfigs",
   156  	AlterPartitionReassignments: "AlterPartitionReassignments",
   157  	ListPartitionReassignments:  "ListPartitionReassignments",
   158  	OffsetDelete:                "OffsetDelete",
   159  	DescribeClientQuotas:        "DescribeClientQuotas",
   160  	AlterClientQuotas:           "AlterClientQuotas",
   161  }
   162  
   163  type messageType struct {
   164  	version  int16
   165  	flexible bool
   166  	gotype   reflect.Type
   167  	decode   decodeFunc
   168  	encode   encodeFunc
   169  }
   170  
   171  func (t *messageType) new() Message {
   172  	return reflect.New(t.gotype).Interface().(Message)
   173  }
   174  
   175  type apiType struct {
   176  	requests  []messageType
   177  	responses []messageType
   178  }
   179  
   180  func (t apiType) minVersion() int16 {
   181  	if len(t.requests) == 0 {
   182  		return 0
   183  	}
   184  	return t.requests[0].version
   185  }
   186  
   187  func (t apiType) maxVersion() int16 {
   188  	if len(t.requests) == 0 {
   189  		return 0
   190  	}
   191  	return t.requests[len(t.requests)-1].version
   192  }
   193  
   194  var apiTypes [numApis]apiType
   195  
   196  // Register is automatically called by sub-packages are imported to install a
   197  // new pair of request/response message types.
   198  func Register(req, res Message) {
   199  	k1 := req.ApiKey()
   200  	k2 := res.ApiKey()
   201  
   202  	if k1 != k2 {
   203  		panic(fmt.Sprintf("[%T/%T]: request and response API keys mismatch: %d != %d", req, res, k1, k2))
   204  	}
   205  
   206  	apiTypes[k1] = apiType{
   207  		requests:  typesOf(req),
   208  		responses: typesOf(res),
   209  	}
   210  }
   211  
   212  func typesOf(v interface{}) []messageType {
   213  	return makeTypes(reflect.TypeOf(v).Elem())
   214  }
   215  
   216  func makeTypes(t reflect.Type) []messageType {
   217  	minVersion := int16(-1)
   218  	maxVersion := int16(-1)
   219  
   220  	// All future versions will be flexible (according to spec), so don't need to
   221  	// worry about maxes here.
   222  	minFlexibleVersion := int16(-1)
   223  
   224  	forEachStructField(t, func(_ reflect.Type, _ index, tag string) {
   225  		forEachStructTag(tag, func(tag structTag) bool {
   226  			if minVersion < 0 || tag.MinVersion < minVersion {
   227  				minVersion = tag.MinVersion
   228  			}
   229  			if maxVersion < 0 || tag.MaxVersion > maxVersion {
   230  				maxVersion = tag.MaxVersion
   231  			}
   232  			if tag.TagID > -2 && (minFlexibleVersion < 0 || tag.MinVersion < minFlexibleVersion) {
   233  				minFlexibleVersion = tag.MinVersion
   234  			}
   235  			return true
   236  		})
   237  	})
   238  
   239  	types := make([]messageType, 0, (maxVersion-minVersion)+1)
   240  
   241  	for v := minVersion; v <= maxVersion; v++ {
   242  		flexible := minFlexibleVersion >= 0 && v >= minFlexibleVersion
   243  
   244  		types = append(types, messageType{
   245  			version:  v,
   246  			gotype:   t,
   247  			flexible: flexible,
   248  			decode:   decodeFuncOf(t, v, flexible, structTag{}),
   249  			encode:   encodeFuncOf(t, v, flexible, structTag{}),
   250  		})
   251  	}
   252  
   253  	return types
   254  }
   255  
   256  type structTag struct {
   257  	MinVersion int16
   258  	MaxVersion int16
   259  	Compact    bool
   260  	Nullable   bool
   261  	TagID      int
   262  }
   263  
   264  func forEachStructTag(tag string, do func(structTag) bool) {
   265  	if tag == "-" {
   266  		return // special case to ignore the field
   267  	}
   268  
   269  	forEach(tag, '|', func(s string) bool {
   270  		tag := structTag{
   271  			MinVersion: -1,
   272  			MaxVersion: -1,
   273  
   274  			// Legitimate tag IDs can start at 0. We use -1 as a placeholder to indicate
   275  			// that the message type is flexible, so that leaves -2 as the default for
   276  			// indicating that there is no tag ID and the message is not flexible.
   277  			TagID: -2,
   278  		}
   279  
   280  		var err error
   281  		forEach(s, ',', func(s string) bool {
   282  			switch {
   283  			case strings.HasPrefix(s, "min="):
   284  				tag.MinVersion, err = parseVersion(s[4:])
   285  			case strings.HasPrefix(s, "max="):
   286  				tag.MaxVersion, err = parseVersion(s[4:])
   287  			case s == "tag":
   288  				tag.TagID = -1
   289  			case strings.HasPrefix(s, "tag="):
   290  				tag.TagID, err = strconv.Atoi(s[4:])
   291  			case s == "compact":
   292  				tag.Compact = true
   293  			case s == "nullable":
   294  				tag.Nullable = true
   295  			default:
   296  				err = fmt.Errorf("unrecognized option: %q", s)
   297  			}
   298  			return err == nil
   299  		})
   300  
   301  		if err != nil {
   302  			panic(fmt.Errorf("malformed struct tag: %w", err))
   303  		}
   304  
   305  		if tag.MinVersion < 0 && tag.MaxVersion >= 0 {
   306  			panic(fmt.Errorf("missing minimum version in struct tag: %q", s))
   307  		}
   308  
   309  		if tag.MaxVersion < 0 && tag.MinVersion >= 0 {
   310  			panic(fmt.Errorf("missing maximum version in struct tag: %q", s))
   311  		}
   312  
   313  		if tag.MinVersion > tag.MaxVersion {
   314  			panic(fmt.Errorf("invalid version range in struct tag: %q", s))
   315  		}
   316  
   317  		return do(tag)
   318  	})
   319  }
   320  
   321  func forEach(s string, sep byte, do func(string) bool) bool {
   322  	for len(s) != 0 {
   323  		p := ""
   324  		i := strings.IndexByte(s, sep)
   325  		if i < 0 {
   326  			p, s = s, ""
   327  		} else {
   328  			p, s = s[:i], s[i+1:]
   329  		}
   330  		if !do(p) {
   331  			return false
   332  		}
   333  	}
   334  	return true
   335  }
   336  
   337  func forEachStructField(t reflect.Type, do func(reflect.Type, index, string)) {
   338  	for i, n := 0, t.NumField(); i < n; i++ {
   339  		f := t.Field(i)
   340  
   341  		if f.PkgPath != "" && f.Name != "_" {
   342  			continue
   343  		}
   344  
   345  		kafkaTag, ok := f.Tag.Lookup("kafka")
   346  		if !ok {
   347  			kafkaTag = "|"
   348  		}
   349  
   350  		do(f.Type, indexOf(f), kafkaTag)
   351  	}
   352  }
   353  
   354  func parseVersion(s string) (int16, error) {
   355  	if !strings.HasPrefix(s, "v") {
   356  		return 0, fmt.Errorf("invalid version number: %q", s)
   357  	}
   358  	i, err := strconv.ParseInt(s[1:], 10, 16)
   359  	if err != nil {
   360  		return 0, fmt.Errorf("invalid version number: %q: %w", s, err)
   361  	}
   362  	if i < 0 {
   363  		return 0, fmt.Errorf("invalid negative version number: %q", s)
   364  	}
   365  	return int16(i), nil
   366  }
   367  
   368  func dontExpectEOF(err error) error {
   369  	if err != nil {
   370  		if errors.Is(err, io.EOF) {
   371  			return io.ErrUnexpectedEOF
   372  		}
   373  
   374  		return err
   375  	}
   376  
   377  	return nil
   378  }
   379  
   380  type Broker struct {
   381  	Rack string
   382  	Host string
   383  	Port int32
   384  	ID   int32
   385  }
   386  
   387  func (b Broker) String() string {
   388  	return net.JoinHostPort(b.Host, itoa(b.Port))
   389  }
   390  
   391  func (b Broker) Format(w fmt.State, v rune) {
   392  	switch v {
   393  	case 'd':
   394  		io.WriteString(w, itoa(b.ID))
   395  	case 's':
   396  		io.WriteString(w, b.String())
   397  	case 'v':
   398  		io.WriteString(w, itoa(b.ID))
   399  		io.WriteString(w, " ")
   400  		io.WriteString(w, b.String())
   401  		if b.Rack != "" {
   402  			io.WriteString(w, " ")
   403  			io.WriteString(w, b.Rack)
   404  		}
   405  	}
   406  }
   407  
   408  func itoa(i int32) string {
   409  	return strconv.Itoa(int(i))
   410  }
   411  
   412  type Topic struct {
   413  	Name       string
   414  	Error      int16
   415  	Partitions map[int32]Partition
   416  }
   417  
   418  type Partition struct {
   419  	ID       int32
   420  	Error    int16
   421  	Leader   int32
   422  	Replicas []int32
   423  	ISR      []int32
   424  	Offline  []int32
   425  }
   426  
   427  // RawExchanger is an extention to the Message interface to allow messages
   428  // to control the request response cycle for the message. This is currently
   429  // only used to facilitate v0 SASL Authenticate requests being written in
   430  // a non-standard fashion when the SASL Handshake was done at v0 but not
   431  // when done at v1.
   432  type RawExchanger interface {
   433  	// Required should return true when a RawExchange is needed.
   434  	// The passed in versions are the negotiated versions for the connection
   435  	// performing the request.
   436  	Required(versions map[ApiKey]int16) bool
   437  	// RawExchange is given the raw connection to the broker and the Message
   438  	// is responsible for writing itself to the connection as well as reading
   439  	// the response.
   440  	RawExchange(rw io.ReadWriter) (Message, error)
   441  }
   442  
   443  // BrokerMessage is an extension of the Message interface implemented by some
   444  // request types to customize the broker assignment logic.
   445  type BrokerMessage interface {
   446  	// Given a representation of the kafka cluster state as argument, returns
   447  	// the broker that the message should be routed to.
   448  	Broker(Cluster) (Broker, error)
   449  }
   450  
   451  // GroupMessage is an extension of the Message interface implemented by some
   452  // request types to inform the program that they should be routed to a group
   453  // coordinator.
   454  type GroupMessage interface {
   455  	// Returns the group configured on the message.
   456  	Group() string
   457  }
   458  
   459  // TransactionalMessage is an extension of the Message interface implemented by some
   460  // request types to inform the program that they should be routed to a transaction
   461  // coordinator.
   462  type TransactionalMessage interface {
   463  	// Returns the transactional id configured on the message.
   464  	Transaction() string
   465  }
   466  
   467  // PreparedMessage is an extension of the Message interface implemented by some
   468  // request types which may need to run some pre-processing on their state before
   469  // being sent.
   470  type PreparedMessage interface {
   471  	// Prepares the message before being sent to a kafka broker using the API
   472  	// version passed as argument.
   473  	Prepare(apiVersion int16)
   474  }
   475  
   476  // Splitter is an interface implemented by messages that can be split into
   477  // multiple requests and have their results merged back by a Merger.
   478  type Splitter interface {
   479  	// For a given cluster layout, returns the list of messages constructed
   480  	// from the receiver for each requests that should be sent to the cluster.
   481  	// The second return value is a Merger which can be used to merge back the
   482  	// results of each request into a single message (or an error).
   483  	Split(Cluster) ([]Message, Merger, error)
   484  }
   485  
   486  // Merger is an interface implemented by messages which can merge multiple
   487  // results into one response.
   488  type Merger interface {
   489  	// Given a list of message and associated results, merge them back into a
   490  	// response (or an error). The results must be either Message or error
   491  	// values, other types should trigger a panic.
   492  	Merge(messages []Message, results []interface{}) (Message, error)
   493  }
   494  
   495  // Result converts r to a Message or an error, or panics if r could not be
   496  // converted to these types.
   497  func Result(r interface{}) (Message, error) {
   498  	switch v := r.(type) {
   499  	case Message:
   500  		return v, nil
   501  	case error:
   502  		return nil, v
   503  	default:
   504  		panic(fmt.Errorf("BUG: result must be a message or an error but not %T", v))
   505  	}
   506  }