github.com/segmentio/kafka-go@v0.4.48-0.20240318174348-3f6244eb34fd/read.go (about) 1 package kafka 2 3 import ( 4 "bufio" 5 "errors" 6 "fmt" 7 "io" 8 "reflect" 9 ) 10 11 var errShortRead = errors.New("not enough bytes available to load the response") 12 13 func peekRead(r *bufio.Reader, sz int, n int, f func([]byte)) (int, error) { 14 if n > sz { 15 return sz, errShortRead 16 } 17 b, err := r.Peek(n) 18 if err != nil { 19 return sz, err 20 } 21 f(b) 22 return discardN(r, sz, n) 23 } 24 25 func readInt8(r *bufio.Reader, sz int, v *int8) (int, error) { 26 return peekRead(r, sz, 1, func(b []byte) { *v = makeInt8(b) }) 27 } 28 29 func readInt16(r *bufio.Reader, sz int, v *int16) (int, error) { 30 return peekRead(r, sz, 2, func(b []byte) { *v = makeInt16(b) }) 31 } 32 33 func readInt32(r *bufio.Reader, sz int, v *int32) (int, error) { 34 return peekRead(r, sz, 4, func(b []byte) { *v = makeInt32(b) }) 35 } 36 37 func readInt64(r *bufio.Reader, sz int, v *int64) (int, error) { 38 return peekRead(r, sz, 8, func(b []byte) { *v = makeInt64(b) }) 39 } 40 41 func readVarInt(r *bufio.Reader, sz int, v *int64) (remain int, err error) { 42 // Optimistically assume that most of the time, there will be data buffered 43 // in the reader. If this is not the case, the buffer will be refilled after 44 // consuming zero bytes from the input. 45 input, _ := r.Peek(r.Buffered()) 46 x := uint64(0) 47 s := uint(0) 48 49 for { 50 if len(input) > sz { 51 input = input[:sz] 52 } 53 54 for i, b := range input { 55 if b < 0x80 { 56 x |= uint64(b) << s 57 *v = int64(x>>1) ^ -(int64(x) & 1) 58 n, err := r.Discard(i + 1) 59 return sz - n, err 60 } 61 62 x |= uint64(b&0x7f) << s 63 s += 7 64 } 65 66 // Make room in the input buffer to load more data from the underlying 67 // stream. The x and s variables are left untouched, ensuring that the 68 // varint decoding can continue on the next loop iteration. 69 n, _ := r.Discard(len(input)) 70 sz -= n 71 if sz == 0 { 72 return 0, errShortRead 73 } 74 75 // Fill the buffer: ask for one more byte, but in practice the reader 76 // will load way more from the underlying stream. 77 if _, err := r.Peek(1); err != nil { 78 if errors.Is(err, io.EOF) { 79 err = errShortRead 80 } 81 return sz, err 82 } 83 84 // Grab as many bytes as possible from the buffer, then go on to the 85 // next loop iteration which is going to consume it. 86 input, _ = r.Peek(r.Buffered()) 87 } 88 } 89 90 func readBool(r *bufio.Reader, sz int, v *bool) (int, error) { 91 return peekRead(r, sz, 1, func(b []byte) { *v = b[0] != 0 }) 92 } 93 94 func readString(r *bufio.Reader, sz int, v *string) (int, error) { 95 return readStringWith(r, sz, func(r *bufio.Reader, sz int, n int) (remain int, err error) { 96 *v, remain, err = readNewString(r, sz, n) 97 return 98 }) 99 } 100 101 func readStringWith(r *bufio.Reader, sz int, cb func(*bufio.Reader, int, int) (int, error)) (int, error) { 102 var err error 103 var len int16 104 105 if sz, err = readInt16(r, sz, &len); err != nil { 106 return sz, err 107 } 108 109 n := int(len) 110 if n > sz { 111 return sz, errShortRead 112 } 113 114 return cb(r, sz, n) 115 } 116 117 func readNewString(r *bufio.Reader, sz int, n int) (string, int, error) { 118 b, sz, err := readNewBytes(r, sz, n) 119 return string(b), sz, err 120 } 121 122 func readBytes(r *bufio.Reader, sz int, v *[]byte) (int, error) { 123 return readBytesWith(r, sz, func(r *bufio.Reader, sz int, n int) (remain int, err error) { 124 *v, remain, err = readNewBytes(r, sz, n) 125 return 126 }) 127 } 128 129 func readBytesWith(r *bufio.Reader, sz int, cb func(*bufio.Reader, int, int) (int, error)) (int, error) { 130 var err error 131 var n int 132 133 if sz, err = readArrayLen(r, sz, &n); err != nil { 134 return sz, err 135 } 136 137 if n > sz { 138 return sz, errShortRead 139 } 140 141 return cb(r, sz, n) 142 } 143 144 func readNewBytes(r *bufio.Reader, sz int, n int) ([]byte, int, error) { 145 var err error 146 var b []byte 147 var shortRead bool 148 149 if n > 0 { 150 if sz < n { 151 n = sz 152 shortRead = true 153 } 154 155 b = make([]byte, n) 156 n, err = io.ReadFull(r, b) 157 b = b[:n] 158 sz -= n 159 160 if err == nil && shortRead { 161 err = errShortRead 162 } 163 } 164 165 return b, sz, err 166 } 167 168 func readArrayLen(r *bufio.Reader, sz int, n *int) (int, error) { 169 var err error 170 var len int32 171 if sz, err = readInt32(r, sz, &len); err != nil { 172 return sz, err 173 } 174 *n = int(len) 175 return sz, nil 176 } 177 178 func readArrayWith(r *bufio.Reader, sz int, cb func(*bufio.Reader, int) (int, error)) (int, error) { 179 var err error 180 var len int32 181 182 if sz, err = readInt32(r, sz, &len); err != nil { 183 return sz, err 184 } 185 186 for n := int(len); n > 0; n-- { 187 if sz, err = cb(r, sz); err != nil { 188 break 189 } 190 } 191 192 return sz, err 193 } 194 195 func readStringArray(r *bufio.Reader, sz int, v *[]string) (remain int, err error) { 196 var content []string 197 fn := func(r *bufio.Reader, size int) (fnRemain int, fnErr error) { 198 var value string 199 if fnRemain, fnErr = readString(r, size, &value); fnErr != nil { 200 return 201 } 202 content = append(content, value) 203 return 204 } 205 if remain, err = readArrayWith(r, sz, fn); err != nil { 206 return 207 } 208 209 *v = content 210 return 211 } 212 213 func readMapStringInt32(r *bufio.Reader, sz int, v *map[string][]int32) (remain int, err error) { 214 var len int32 215 if remain, err = readInt32(r, sz, &len); err != nil { 216 return 217 } 218 219 content := make(map[string][]int32, len) 220 for i := 0; i < int(len); i++ { 221 var key string 222 var values []int32 223 224 if remain, err = readString(r, remain, &key); err != nil { 225 return 226 } 227 228 fn := func(r *bufio.Reader, size int) (fnRemain int, fnErr error) { 229 var value int32 230 if fnRemain, fnErr = readInt32(r, size, &value); fnErr != nil { 231 return 232 } 233 values = append(values, value) 234 return 235 } 236 if remain, err = readArrayWith(r, remain, fn); err != nil { 237 return 238 } 239 240 content[key] = values 241 } 242 *v = content 243 244 return 245 } 246 247 func read(r *bufio.Reader, sz int, a interface{}) (int, error) { 248 switch v := a.(type) { 249 case *int8: 250 return readInt8(r, sz, v) 251 case *int16: 252 return readInt16(r, sz, v) 253 case *int32: 254 return readInt32(r, sz, v) 255 case *int64: 256 return readInt64(r, sz, v) 257 case *bool: 258 return readBool(r, sz, v) 259 case *string: 260 return readString(r, sz, v) 261 case *[]byte: 262 return readBytes(r, sz, v) 263 } 264 switch v := reflect.ValueOf(a).Elem(); v.Kind() { 265 case reflect.Struct: 266 return readStruct(r, sz, v) 267 case reflect.Slice: 268 return readSlice(r, sz, v) 269 default: 270 panic(fmt.Sprintf("unsupported type: %T", a)) 271 } 272 } 273 274 func readStruct(r *bufio.Reader, sz int, v reflect.Value) (int, error) { 275 var err error 276 for i, n := 0, v.NumField(); i != n; i++ { 277 if sz, err = read(r, sz, v.Field(i).Addr().Interface()); err != nil { 278 return sz, err 279 } 280 } 281 return sz, nil 282 } 283 284 func readSlice(r *bufio.Reader, sz int, v reflect.Value) (int, error) { 285 var err error 286 var len int32 287 288 if sz, err = readInt32(r, sz, &len); err != nil { 289 return sz, err 290 } 291 292 if n := int(len); n < 0 { 293 v.Set(reflect.Zero(v.Type())) 294 } else { 295 v.Set(reflect.MakeSlice(v.Type(), n, n)) 296 297 for i := 0; i != n; i++ { 298 if sz, err = read(r, sz, v.Index(i).Addr().Interface()); err != nil { 299 return sz, err 300 } 301 } 302 } 303 304 return sz, nil 305 } 306 307 func readFetchResponseHeaderV2(r *bufio.Reader, size int) (throttle int32, watermark int64, remain int, err error) { 308 var n int32 309 var p struct { 310 Partition int32 311 ErrorCode int16 312 HighwaterMarkOffset int64 313 MessageSetSize int32 314 } 315 316 if remain, err = readInt32(r, size, &throttle); err != nil { 317 return 318 } 319 320 if remain, err = readInt32(r, remain, &n); err != nil { 321 return 322 } 323 324 // This error should never trigger, unless there's a bug in the kafka client 325 // or server. 326 if n != 1 { 327 err = fmt.Errorf("1 kafka topic was expected in the fetch response but the client received %d", n) 328 return 329 } 330 331 // We ignore the topic name because we've requests messages for a single 332 // topic, unless there's a bug in the kafka server we will have received 333 // the name of the topic that we requested. 334 if remain, err = discardString(r, remain); err != nil { 335 return 336 } 337 338 if remain, err = readInt32(r, remain, &n); err != nil { 339 return 340 } 341 342 // This error should never trigger, unless there's a bug in the kafka client 343 // or server. 344 if n != 1 { 345 err = fmt.Errorf("1 kafka partition was expected in the fetch response but the client received %d", n) 346 return 347 } 348 349 if remain, err = read(r, remain, &p); err != nil { 350 return 351 } 352 353 if p.ErrorCode != 0 { 354 err = Error(p.ErrorCode) 355 return 356 } 357 358 // This error should never trigger, unless there's a bug in the kafka client 359 // or server. 360 if remain != int(p.MessageSetSize) { 361 err = fmt.Errorf("the size of the message set in a fetch response doesn't match the number of remaining bytes (message set size = %d, remaining bytes = %d)", p.MessageSetSize, remain) 362 return 363 } 364 365 watermark = p.HighwaterMarkOffset 366 return 367 } 368 369 func readFetchResponseHeaderV5(r *bufio.Reader, size int) (throttle int32, watermark int64, remain int, err error) { 370 var n int32 371 type AbortedTransaction struct { 372 ProducerId int64 373 FirstOffset int64 374 } 375 var p struct { 376 Partition int32 377 ErrorCode int16 378 HighwaterMarkOffset int64 379 LastStableOffset int64 380 LogStartOffset int64 381 } 382 var messageSetSize int32 383 var abortedTransactions []AbortedTransaction 384 385 if remain, err = readInt32(r, size, &throttle); err != nil { 386 return 387 } 388 389 if remain, err = readInt32(r, remain, &n); err != nil { 390 return 391 } 392 393 // This error should never trigger, unless there's a bug in the kafka client 394 // or server. 395 if n != 1 { 396 err = fmt.Errorf("1 kafka topic was expected in the fetch response but the client received %d", n) 397 return 398 } 399 400 // We ignore the topic name because we've requests messages for a single 401 // topic, unless there's a bug in the kafka server we will have received 402 // the name of the topic that we requested. 403 if remain, err = discardString(r, remain); err != nil { 404 return 405 } 406 407 if remain, err = readInt32(r, remain, &n); err != nil { 408 return 409 } 410 411 // This error should never trigger, unless there's a bug in the kafka client 412 // or server. 413 if n != 1 { 414 err = fmt.Errorf("1 kafka partition was expected in the fetch response but the client received %d", n) 415 return 416 } 417 418 if remain, err = read(r, remain, &p); err != nil { 419 return 420 } 421 422 var abortedTransactionLen int 423 if remain, err = readArrayLen(r, remain, &abortedTransactionLen); err != nil { 424 return 425 } 426 427 if abortedTransactionLen == -1 { 428 abortedTransactions = nil 429 } else { 430 abortedTransactions = make([]AbortedTransaction, abortedTransactionLen) 431 for i := 0; i < abortedTransactionLen; i++ { 432 if remain, err = read(r, remain, &abortedTransactions[i]); err != nil { 433 return 434 } 435 } 436 } 437 438 if p.ErrorCode != 0 { 439 err = Error(p.ErrorCode) 440 return 441 } 442 443 remain, err = readInt32(r, remain, &messageSetSize) 444 if err != nil { 445 return 446 } 447 448 // This error should never trigger, unless there's a bug in the kafka client 449 // or server. 450 if remain != int(messageSetSize) { 451 err = fmt.Errorf("the size of the message set in a fetch response doesn't match the number of remaining bytes (message set size = %d, remaining bytes = %d)", messageSetSize, remain) 452 return 453 } 454 455 watermark = p.HighwaterMarkOffset 456 return 457 458 } 459 460 func readFetchResponseHeaderV10(r *bufio.Reader, size int) (throttle int32, watermark int64, remain int, err error) { 461 var n int32 462 var errorCode int16 463 type AbortedTransaction struct { 464 ProducerId int64 465 FirstOffset int64 466 } 467 var p struct { 468 Partition int32 469 ErrorCode int16 470 HighwaterMarkOffset int64 471 LastStableOffset int64 472 LogStartOffset int64 473 } 474 var messageSetSize int32 475 var abortedTransactions []AbortedTransaction 476 477 if remain, err = readInt32(r, size, &throttle); err != nil { 478 return 479 } 480 481 if remain, err = readInt16(r, remain, &errorCode); err != nil { 482 return 483 } 484 if errorCode != 0 { 485 err = Error(errorCode) 486 return 487 } 488 489 if remain, err = discardInt32(r, remain); err != nil { 490 return 491 } 492 493 if remain, err = readInt32(r, remain, &n); err != nil { 494 return 495 } 496 497 // This error should never trigger, unless there's a bug in the kafka client 498 // or server. 499 if n != 1 { 500 err = fmt.Errorf("1 kafka topic was expected in the fetch response but the client received %d", n) 501 return 502 } 503 504 // We ignore the topic name because we've requests messages for a single 505 // topic, unless there's a bug in the kafka server we will have received 506 // the name of the topic that we requested. 507 if remain, err = discardString(r, remain); err != nil { 508 return 509 } 510 511 if remain, err = readInt32(r, remain, &n); err != nil { 512 return 513 } 514 515 // This error should never trigger, unless there's a bug in the kafka client 516 // or server. 517 if n != 1 { 518 err = fmt.Errorf("1 kafka partition was expected in the fetch response but the client received %d", n) 519 return 520 } 521 522 if remain, err = read(r, remain, &p); err != nil { 523 return 524 } 525 526 var abortedTransactionLen int 527 if remain, err = readArrayLen(r, remain, &abortedTransactionLen); err != nil { 528 return 529 } 530 531 if abortedTransactionLen == -1 { 532 abortedTransactions = nil 533 } else { 534 abortedTransactions = make([]AbortedTransaction, abortedTransactionLen) 535 for i := 0; i < abortedTransactionLen; i++ { 536 if remain, err = read(r, remain, &abortedTransactions[i]); err != nil { 537 return 538 } 539 } 540 } 541 542 if p.ErrorCode != 0 { 543 err = Error(p.ErrorCode) 544 return 545 } 546 547 remain, err = readInt32(r, remain, &messageSetSize) 548 if err != nil { 549 return 550 } 551 552 // This error should never trigger, unless there's a bug in the kafka client 553 // or server. 554 if remain != int(messageSetSize) { 555 err = fmt.Errorf("the size of the message set in a fetch response doesn't match the number of remaining bytes (message set size = %d, remaining bytes = %d)", messageSetSize, remain) 556 return 557 } 558 559 watermark = p.HighwaterMarkOffset 560 return 561 562 }