github.com/QuangHoangHao/kafka-go@v0.4.36/syncgroup.go (about) 1 package kafka 2 3 import ( 4 "bufio" 5 "bytes" 6 "context" 7 "fmt" 8 "net" 9 "time" 10 11 "github.com/QuangHoangHao/kafka-go/protocol" 12 "github.com/QuangHoangHao/kafka-go/protocol/consumer" 13 "github.com/QuangHoangHao/kafka-go/protocol/syncgroup" 14 ) 15 16 // SyncGroupRequest is the request structure for the SyncGroup function. 17 type SyncGroupRequest struct { 18 // Address of the kafka broker to sent he request to. 19 Addr net.Addr 20 21 // GroupID of the group to sync. 22 GroupID string 23 24 // The generation of the group. 25 GenerationID int 26 27 // The member ID assigned by the group. 28 MemberID string 29 30 // The unique identifier for the consumer instance. 31 GroupInstanceID string 32 33 // The name for the class of protocols implemented by the group being joined. 34 ProtocolType string 35 36 // The group protocol name. 37 ProtocolName string 38 39 // The group member assignments. 40 Assignments []SyncGroupRequestAssignment 41 } 42 43 // SyncGroupRequestAssignment represents an assignement for a goroup memeber. 44 type SyncGroupRequestAssignment struct { 45 // The ID of the member to assign. 46 MemberID string 47 48 // The member assignment. 49 Assignment GroupProtocolAssignment 50 } 51 52 // SyncGroupResponse is the response structure for the SyncGroup function. 53 type SyncGroupResponse struct { 54 // An error that may have occurred when attempting to sync the group. 55 // 56 // The errors contain the kafka error code. Programs may use the standard 57 // errors.Is function to test the error against kafka error codes. 58 Error error 59 60 // The amount of time that the broker throttled the request. 61 Throttle time.Duration 62 63 // The group protocol type. 64 ProtocolType string 65 66 // The group protocol name. 67 ProtocolName string 68 69 // The member assignment. 70 Assignment GroupProtocolAssignment 71 } 72 73 // GroupProtocolAssignment represents an assignment of topics and partitions for a group memeber. 74 type GroupProtocolAssignment struct { 75 // The topics and partitions assigned to the group memeber. 76 AssignedPartitions map[string][]int 77 78 // UserData for the assignemnt. 79 UserData []byte 80 } 81 82 // SyncGroup sends a sync group request to the coordinator and returns the response. 83 func (c *Client) SyncGroup(ctx context.Context, req *SyncGroupRequest) (*SyncGroupResponse, error) { 84 syncGroup := syncgroup.Request{ 85 GroupID: req.GroupID, 86 GenerationID: int32(req.GenerationID), 87 MemberID: req.MemberID, 88 GroupInstanceID: req.GroupInstanceID, 89 ProtocolType: req.ProtocolType, 90 ProtocolName: req.ProtocolName, 91 Assignments: make([]syncgroup.RequestAssignment, 0, len(req.Assignments)), 92 } 93 94 for _, assignment := range req.Assignments { 95 assign := consumer.Assignment{ 96 Version: consumer.MaxVersionSupported, 97 AssignedPartitions: make([]consumer.TopicPartition, 0, len(assignment.Assignment.AssignedPartitions)), 98 UserData: assignment.Assignment.UserData, 99 } 100 101 for topic, partitions := range assignment.Assignment.AssignedPartitions { 102 tp := consumer.TopicPartition{ 103 Topic: topic, 104 Partitions: make([]int32, 0, len(partitions)), 105 } 106 for _, partition := range partitions { 107 tp.Partitions = append(tp.Partitions, int32(partition)) 108 } 109 assign.AssignedPartitions = append(assign.AssignedPartitions, tp) 110 } 111 112 assignBytes, err := protocol.Marshal(consumer.MaxVersionSupported, assign) 113 if err != nil { 114 return nil, fmt.Errorf("kafka.(*Client).SyncGroup: %w", err) 115 } 116 117 syncGroup.Assignments = append(syncGroup.Assignments, syncgroup.RequestAssignment{ 118 MemberID: assignment.MemberID, 119 Assignment: assignBytes, 120 }) 121 } 122 123 m, err := c.roundTrip(ctx, req.Addr, &syncGroup) 124 if err != nil { 125 return nil, fmt.Errorf("kafka.(*Client).SyncGroup: %w", err) 126 } 127 128 r := m.(*syncgroup.Response) 129 130 var assignment consumer.Assignment 131 err = protocol.Unmarshal(r.Assignments, consumer.MaxVersionSupported, &assignment) 132 if err != nil { 133 return nil, fmt.Errorf("kafka.(*Client).SyncGroup: %w", err) 134 } 135 136 res := &SyncGroupResponse{ 137 Throttle: makeDuration(r.ThrottleTimeMS), 138 Error: makeError(r.ErrorCode, ""), 139 ProtocolType: r.ProtocolType, 140 ProtocolName: r.ProtocolName, 141 Assignment: GroupProtocolAssignment{ 142 AssignedPartitions: make(map[string][]int, len(assignment.AssignedPartitions)), 143 UserData: assignment.UserData, 144 }, 145 } 146 partitions := map[string][]int{} 147 for _, topicPartition := range assignment.AssignedPartitions { 148 for _, partition := range topicPartition.Partitions { 149 partitions[topicPartition.Topic] = append(partitions[topicPartition.Topic], int(partition)) 150 } 151 } 152 res.Assignment.AssignedPartitions = partitions 153 154 return res, nil 155 } 156 157 type groupAssignment struct { 158 Version int16 159 Topics map[string][]int32 160 UserData []byte 161 } 162 163 func (t groupAssignment) size() int32 { 164 sz := sizeofInt16(t.Version) + sizeofInt16(int16(len(t.Topics))) 165 166 for topic, partitions := range t.Topics { 167 sz += sizeofString(topic) + sizeofInt32Array(partitions) 168 } 169 170 return sz + sizeofBytes(t.UserData) 171 } 172 173 func (t groupAssignment) writeTo(wb *writeBuffer) { 174 wb.writeInt16(t.Version) 175 wb.writeInt32(int32(len(t.Topics))) 176 177 for topic, partitions := range t.Topics { 178 wb.writeString(topic) 179 wb.writeInt32Array(partitions) 180 } 181 182 wb.writeBytes(t.UserData) 183 } 184 185 func (t *groupAssignment) readFrom(r *bufio.Reader, size int) (remain int, err error) { 186 // I came across this case when testing for compatibility with bsm/sarama-cluster. It 187 // appears in some cases, sarama-cluster can send a nil array entry. Admittedly, I 188 // didn't look too closely at it. 189 if size == 0 { 190 t.Topics = map[string][]int32{} 191 return 0, nil 192 } 193 194 if remain, err = readInt16(r, size, &t.Version); err != nil { 195 return 196 } 197 if remain, err = readMapStringInt32(r, remain, &t.Topics); err != nil { 198 return 199 } 200 if remain, err = readBytes(r, remain, &t.UserData); err != nil { 201 return 202 } 203 204 return 205 } 206 207 func (t groupAssignment) bytes() []byte { 208 buf := bytes.NewBuffer(nil) 209 t.writeTo(&writeBuffer{w: buf}) 210 return buf.Bytes() 211 } 212 213 type syncGroupRequestGroupAssignmentV0 struct { 214 // MemberID assigned by the group coordinator 215 MemberID string 216 217 // MemberAssignments holds client encoded assignments 218 // 219 // See consumer groups section of https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol 220 MemberAssignments []byte 221 } 222 223 func (t syncGroupRequestGroupAssignmentV0) size() int32 { 224 return sizeofString(t.MemberID) + 225 sizeofBytes(t.MemberAssignments) 226 } 227 228 func (t syncGroupRequestGroupAssignmentV0) writeTo(wb *writeBuffer) { 229 wb.writeString(t.MemberID) 230 wb.writeBytes(t.MemberAssignments) 231 } 232 233 type syncGroupRequestV0 struct { 234 // GroupID holds the unique group identifier 235 GroupID string 236 237 // GenerationID holds the generation of the group. 238 GenerationID int32 239 240 // MemberID assigned by the group coordinator 241 MemberID string 242 243 GroupAssignments []syncGroupRequestGroupAssignmentV0 244 } 245 246 func (t syncGroupRequestV0) size() int32 { 247 return sizeofString(t.GroupID) + 248 sizeofInt32(t.GenerationID) + 249 sizeofString(t.MemberID) + 250 sizeofArray(len(t.GroupAssignments), func(i int) int32 { return t.GroupAssignments[i].size() }) 251 } 252 253 func (t syncGroupRequestV0) writeTo(wb *writeBuffer) { 254 wb.writeString(t.GroupID) 255 wb.writeInt32(t.GenerationID) 256 wb.writeString(t.MemberID) 257 wb.writeArray(len(t.GroupAssignments), func(i int) { t.GroupAssignments[i].writeTo(wb) }) 258 } 259 260 type syncGroupResponseV0 struct { 261 // ErrorCode holds response error code 262 ErrorCode int16 263 264 // MemberAssignments holds client encoded assignments 265 // 266 // See consumer groups section of https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol 267 MemberAssignments []byte 268 } 269 270 func (t syncGroupResponseV0) size() int32 { 271 return sizeofInt16(t.ErrorCode) + 272 sizeofBytes(t.MemberAssignments) 273 } 274 275 func (t syncGroupResponseV0) writeTo(wb *writeBuffer) { 276 wb.writeInt16(t.ErrorCode) 277 wb.writeBytes(t.MemberAssignments) 278 } 279 280 func (t *syncGroupResponseV0) readFrom(r *bufio.Reader, sz int) (remain int, err error) { 281 if remain, err = readInt16(r, sz, &t.ErrorCode); err != nil { 282 return 283 } 284 if remain, err = readBytes(r, remain, &t.MemberAssignments); err != nil { 285 return 286 } 287 return 288 }