github.com/QuangHoangHao/kafka-go@v0.4.36/createtopics.go (about) 1 package kafka 2 3 import ( 4 "bufio" 5 "context" 6 "errors" 7 "fmt" 8 "net" 9 "time" 10 11 "github.com/QuangHoangHao/kafka-go/protocol/createtopics" 12 ) 13 14 // CreateTopicRequests represents a request sent to a kafka broker to create 15 // new topics. 16 type CreateTopicsRequest struct { 17 // Address of the kafka broker to send the request to. 18 Addr net.Addr 19 20 // List of topics to create and their configuration. 21 Topics []TopicConfig 22 23 // When set to true, topics are not created but the configuration is 24 // validated as if they were. 25 // 26 // This field will be ignored if the kafka broker did no support the 27 // CreateTopics API in version 1 or above. 28 ValidateOnly bool 29 } 30 31 // CreateTopicResponse represents a response from a kafka broker to a topic 32 // creation request. 33 type CreateTopicsResponse struct { 34 // The amount of time that the broker throttled the request. 35 // 36 // This field will be zero if the kafka broker did no support the 37 // CreateTopics API in version 2 or above. 38 Throttle time.Duration 39 40 // Mapping of topic names to errors that occurred while attempting to create 41 // the topics. 42 // 43 // The errors contain the kafka error code. Programs may use the standard 44 // errors.Is function to test the error against kafka error codes. 45 Errors map[string]error 46 } 47 48 // CreateTopics sends a topic creation request to a kafka broker and returns the 49 // response. 50 func (c *Client) CreateTopics(ctx context.Context, req *CreateTopicsRequest) (*CreateTopicsResponse, error) { 51 topics := make([]createtopics.RequestTopic, len(req.Topics)) 52 53 for i, t := range req.Topics { 54 topics[i] = createtopics.RequestTopic{ 55 Name: t.Topic, 56 NumPartitions: int32(t.NumPartitions), 57 ReplicationFactor: int16(t.ReplicationFactor), 58 Assignments: t.assignments(), 59 Configs: t.configs(), 60 } 61 } 62 63 m, err := c.roundTrip(ctx, req.Addr, &createtopics.Request{ 64 Topics: topics, 65 TimeoutMs: c.timeoutMs(ctx, defaultCreateTopicsTimeout), 66 ValidateOnly: req.ValidateOnly, 67 }) 68 69 if err != nil { 70 return nil, fmt.Errorf("kafka.(*Client).CreateTopics: %w", err) 71 } 72 73 res := m.(*createtopics.Response) 74 ret := &CreateTopicsResponse{ 75 Throttle: makeDuration(res.ThrottleTimeMs), 76 Errors: make(map[string]error, len(res.Topics)), 77 } 78 79 for _, t := range res.Topics { 80 ret.Errors[t.Name] = makeError(t.ErrorCode, t.ErrorMessage) 81 } 82 83 return ret, nil 84 } 85 86 type ConfigEntry struct { 87 ConfigName string 88 ConfigValue string 89 } 90 91 func (c ConfigEntry) toCreateTopicsRequestV0ConfigEntry() createTopicsRequestV0ConfigEntry { 92 return createTopicsRequestV0ConfigEntry(c) 93 } 94 95 type createTopicsRequestV0ConfigEntry struct { 96 ConfigName string 97 ConfigValue string 98 } 99 100 func (t createTopicsRequestV0ConfigEntry) size() int32 { 101 return sizeofString(t.ConfigName) + 102 sizeofString(t.ConfigValue) 103 } 104 105 func (t createTopicsRequestV0ConfigEntry) writeTo(wb *writeBuffer) { 106 wb.writeString(t.ConfigName) 107 wb.writeString(t.ConfigValue) 108 } 109 110 type ReplicaAssignment struct { 111 Partition int 112 // The list of brokers where the partition should be allocated. There must 113 // be as many entries in thie list as there are replicas of the partition. 114 // The first entry represents the broker that will be the preferred leader 115 // for the partition. 116 // 117 // This field changed in 0.4 from `int` to `[]int`. It was invalid to pass 118 // a single integer as this is supposed to be a list. While this introduces 119 // a breaking change, it probably never worked before. 120 Replicas []int 121 } 122 123 func (a *ReplicaAssignment) partitionIndex() int32 { 124 return int32(a.Partition) 125 } 126 127 func (a *ReplicaAssignment) brokerIDs() []int32 { 128 if len(a.Replicas) == 0 { 129 return nil 130 } 131 replicas := make([]int32, len(a.Replicas)) 132 for i, r := range a.Replicas { 133 replicas[i] = int32(r) 134 } 135 return replicas 136 } 137 138 func (a ReplicaAssignment) toCreateTopicsRequestV0ReplicaAssignment() createTopicsRequestV0ReplicaAssignment { 139 return createTopicsRequestV0ReplicaAssignment{ 140 Partition: int32(a.Partition), 141 Replicas: a.brokerIDs(), 142 } 143 } 144 145 type createTopicsRequestV0ReplicaAssignment struct { 146 Partition int32 147 Replicas []int32 148 } 149 150 func (t createTopicsRequestV0ReplicaAssignment) size() int32 { 151 return sizeofInt32(t.Partition) + 152 (int32(len(t.Replicas)+1) * sizeofInt32(0)) // N+1 because the array length is a int32 153 } 154 155 func (t createTopicsRequestV0ReplicaAssignment) writeTo(wb *writeBuffer) { 156 wb.writeInt32(t.Partition) 157 wb.writeInt32(int32(len(t.Replicas))) 158 for _, r := range t.Replicas { 159 wb.writeInt32(int32(r)) 160 } 161 } 162 163 type TopicConfig struct { 164 // Topic name 165 Topic string 166 167 // NumPartitions created. -1 indicates unset. 168 NumPartitions int 169 170 // ReplicationFactor for the topic. -1 indicates unset. 171 ReplicationFactor int 172 173 // ReplicaAssignments among kafka brokers for this topic partitions. If this 174 // is set num_partitions and replication_factor must be unset. 175 ReplicaAssignments []ReplicaAssignment 176 177 // ConfigEntries holds topic level configuration for topic to be set. 178 ConfigEntries []ConfigEntry 179 } 180 181 func (t *TopicConfig) assignments() []createtopics.RequestAssignment { 182 if len(t.ReplicaAssignments) == 0 { 183 return nil 184 } 185 assignments := make([]createtopics.RequestAssignment, len(t.ReplicaAssignments)) 186 for i, a := range t.ReplicaAssignments { 187 assignments[i] = createtopics.RequestAssignment{ 188 PartitionIndex: a.partitionIndex(), 189 BrokerIDs: a.brokerIDs(), 190 } 191 } 192 return assignments 193 } 194 195 func (t *TopicConfig) configs() []createtopics.RequestConfig { 196 if len(t.ConfigEntries) == 0 { 197 return nil 198 } 199 configs := make([]createtopics.RequestConfig, len(t.ConfigEntries)) 200 for i, c := range t.ConfigEntries { 201 configs[i] = createtopics.RequestConfig{ 202 Name: c.ConfigName, 203 Value: c.ConfigValue, 204 } 205 } 206 return configs 207 } 208 209 func (t TopicConfig) toCreateTopicsRequestV0Topic() createTopicsRequestV0Topic { 210 requestV0ReplicaAssignments := make([]createTopicsRequestV0ReplicaAssignment, 0, len(t.ReplicaAssignments)) 211 for _, a := range t.ReplicaAssignments { 212 requestV0ReplicaAssignments = append( 213 requestV0ReplicaAssignments, 214 a.toCreateTopicsRequestV0ReplicaAssignment()) 215 } 216 requestV0ConfigEntries := make([]createTopicsRequestV0ConfigEntry, 0, len(t.ConfigEntries)) 217 for _, c := range t.ConfigEntries { 218 requestV0ConfigEntries = append( 219 requestV0ConfigEntries, 220 c.toCreateTopicsRequestV0ConfigEntry()) 221 } 222 223 return createTopicsRequestV0Topic{ 224 Topic: t.Topic, 225 NumPartitions: int32(t.NumPartitions), 226 ReplicationFactor: int16(t.ReplicationFactor), 227 ReplicaAssignments: requestV0ReplicaAssignments, 228 ConfigEntries: requestV0ConfigEntries, 229 } 230 } 231 232 type createTopicsRequestV0Topic struct { 233 // Topic name 234 Topic string 235 236 // NumPartitions created. -1 indicates unset. 237 NumPartitions int32 238 239 // ReplicationFactor for the topic. -1 indicates unset. 240 ReplicationFactor int16 241 242 // ReplicaAssignments among kafka brokers for this topic partitions. If this 243 // is set num_partitions and replication_factor must be unset. 244 ReplicaAssignments []createTopicsRequestV0ReplicaAssignment 245 246 // ConfigEntries holds topic level configuration for topic to be set. 247 ConfigEntries []createTopicsRequestV0ConfigEntry 248 } 249 250 func (t createTopicsRequestV0Topic) size() int32 { 251 return sizeofString(t.Topic) + 252 sizeofInt32(t.NumPartitions) + 253 sizeofInt16(t.ReplicationFactor) + 254 sizeofArray(len(t.ReplicaAssignments), func(i int) int32 { return t.ReplicaAssignments[i].size() }) + 255 sizeofArray(len(t.ConfigEntries), func(i int) int32 { return t.ConfigEntries[i].size() }) 256 } 257 258 func (t createTopicsRequestV0Topic) writeTo(wb *writeBuffer) { 259 wb.writeString(t.Topic) 260 wb.writeInt32(t.NumPartitions) 261 wb.writeInt16(t.ReplicationFactor) 262 wb.writeArray(len(t.ReplicaAssignments), func(i int) { t.ReplicaAssignments[i].writeTo(wb) }) 263 wb.writeArray(len(t.ConfigEntries), func(i int) { t.ConfigEntries[i].writeTo(wb) }) 264 } 265 266 // See http://kafka.apache.org/protocol.html#The_Messages_CreateTopics 267 type createTopicsRequestV0 struct { 268 // Topics contains n array of single topic creation requests. Can not 269 // have multiple entries for the same topic. 270 Topics []createTopicsRequestV0Topic 271 272 // Timeout ms to wait for a topic to be completely created on the 273 // controller node. Values <= 0 will trigger topic creation and return immediately 274 Timeout int32 275 } 276 277 func (t createTopicsRequestV0) size() int32 { 278 return sizeofArray(len(t.Topics), func(i int) int32 { return t.Topics[i].size() }) + 279 sizeofInt32(t.Timeout) 280 } 281 282 func (t createTopicsRequestV0) writeTo(wb *writeBuffer) { 283 wb.writeArray(len(t.Topics), func(i int) { t.Topics[i].writeTo(wb) }) 284 wb.writeInt32(t.Timeout) 285 } 286 287 type createTopicsResponseV0TopicError struct { 288 // Topic name 289 Topic string 290 291 // ErrorCode holds response error code 292 ErrorCode int16 293 } 294 295 func (t createTopicsResponseV0TopicError) size() int32 { 296 return sizeofString(t.Topic) + 297 sizeofInt16(t.ErrorCode) 298 } 299 300 func (t createTopicsResponseV0TopicError) writeTo(wb *writeBuffer) { 301 wb.writeString(t.Topic) 302 wb.writeInt16(t.ErrorCode) 303 } 304 305 func (t *createTopicsResponseV0TopicError) readFrom(r *bufio.Reader, size int) (remain int, err error) { 306 if remain, err = readString(r, size, &t.Topic); err != nil { 307 return 308 } 309 if remain, err = readInt16(r, remain, &t.ErrorCode); err != nil { 310 return 311 } 312 return 313 } 314 315 // See http://kafka.apache.org/protocol.html#The_Messages_CreateTopics 316 type createTopicsResponseV0 struct { 317 TopicErrors []createTopicsResponseV0TopicError 318 } 319 320 func (t createTopicsResponseV0) size() int32 { 321 return sizeofArray(len(t.TopicErrors), func(i int) int32 { return t.TopicErrors[i].size() }) 322 } 323 324 func (t createTopicsResponseV0) writeTo(wb *writeBuffer) { 325 wb.writeArray(len(t.TopicErrors), func(i int) { t.TopicErrors[i].writeTo(wb) }) 326 } 327 328 func (t *createTopicsResponseV0) readFrom(r *bufio.Reader, size int) (remain int, err error) { 329 fn := func(r *bufio.Reader, size int) (fnRemain int, fnErr error) { 330 var topic createTopicsResponseV0TopicError 331 if fnRemain, fnErr = (&topic).readFrom(r, size); err != nil { 332 return 333 } 334 t.TopicErrors = append(t.TopicErrors, topic) 335 return 336 } 337 if remain, err = readArrayWith(r, size, fn); err != nil { 338 return 339 } 340 341 return 342 } 343 344 func (c *Conn) createTopics(request createTopicsRequestV0) (createTopicsResponseV0, error) { 345 var response createTopicsResponseV0 346 347 err := c.writeOperation( 348 func(deadline time.Time, id int32) error { 349 if request.Timeout == 0 { 350 now := time.Now() 351 deadline = adjustDeadlineForRTT(deadline, now, defaultRTT) 352 request.Timeout = milliseconds(deadlineToTimeout(deadline, now)) 353 } 354 return c.writeRequest(createTopics, v0, id, request) 355 }, 356 func(deadline time.Time, size int) error { 357 return expectZeroSize(func() (remain int, err error) { 358 return (&response).readFrom(&c.rbuf, size) 359 }()) 360 }, 361 ) 362 if err != nil { 363 return response, err 364 } 365 for _, tr := range response.TopicErrors { 366 if tr.ErrorCode != 0 { 367 return response, Error(tr.ErrorCode) 368 } 369 } 370 371 return response, nil 372 } 373 374 // CreateTopics creates one topic per provided configuration with idempotent 375 // operational semantics. In other words, if CreateTopics is invoked with a 376 // configuration for an existing topic, it will have no effect. 377 func (c *Conn) CreateTopics(topics ...TopicConfig) error { 378 requestV0Topics := make([]createTopicsRequestV0Topic, 0, len(topics)) 379 for _, t := range topics { 380 requestV0Topics = append( 381 requestV0Topics, 382 t.toCreateTopicsRequestV0Topic()) 383 } 384 385 _, err := c.createTopics(createTopicsRequestV0{ 386 Topics: requestV0Topics, 387 }) 388 if err != nil { 389 if errors.Is(err, TopicAlreadyExists) { 390 // ok 391 return nil 392 } 393 394 return err 395 } 396 397 return nil 398 }