github.com/rbisecke/kafka-go@v0.4.27/read.go (about) 1 package kafka 2 3 import ( 4 "bufio" 5 "errors" 6 "fmt" 7 "io" 8 "reflect" 9 ) 10 11 type readable interface { 12 readFrom(*bufio.Reader, int) (int, error) 13 } 14 15 var errShortRead = errors.New("not enough bytes available to load the response") 16 17 func peekRead(r *bufio.Reader, sz int, n int, f func([]byte)) (int, error) { 18 if n > sz { 19 return sz, errShortRead 20 } 21 b, err := r.Peek(n) 22 if err != nil { 23 return sz, err 24 } 25 f(b) 26 return discardN(r, sz, n) 27 } 28 29 func readInt8(r *bufio.Reader, sz int, v *int8) (int, error) { 30 return peekRead(r, sz, 1, func(b []byte) { *v = makeInt8(b) }) 31 } 32 33 func readInt16(r *bufio.Reader, sz int, v *int16) (int, error) { 34 return peekRead(r, sz, 2, func(b []byte) { *v = makeInt16(b) }) 35 } 36 37 func readInt32(r *bufio.Reader, sz int, v *int32) (int, error) { 38 return peekRead(r, sz, 4, func(b []byte) { *v = makeInt32(b) }) 39 } 40 41 func readInt64(r *bufio.Reader, sz int, v *int64) (int, error) { 42 return peekRead(r, sz, 8, func(b []byte) { *v = makeInt64(b) }) 43 } 44 45 func readVarInt(r *bufio.Reader, sz int, v *int64) (remain int, err error) { 46 // Optimistically assume that most of the time, there will be data buffered 47 // in the reader. If this is not the case, the buffer will be refilled after 48 // consuming zero bytes from the input. 49 input, _ := r.Peek(r.Buffered()) 50 x := uint64(0) 51 s := uint(0) 52 53 for { 54 if len(input) > sz { 55 input = input[:sz] 56 } 57 58 for i, b := range input { 59 if b < 0x80 { 60 x |= uint64(b) << s 61 *v = int64(x>>1) ^ -(int64(x) & 1) 62 n, err := r.Discard(i + 1) 63 return sz - n, err 64 } 65 66 x |= uint64(b&0x7f) << s 67 s += 7 68 } 69 70 // Make room in the input buffer to load more data from the underlying 71 // stream. The x and s variables are left untouched, ensuring that the 72 // varint decoding can continue on the next loop iteration. 73 n, _ := r.Discard(len(input)) 74 sz -= n 75 if sz == 0 { 76 return 0, errShortRead 77 } 78 79 // Fill the buffer: ask for one more byte, but in practice the reader 80 // will load way more from the underlying stream. 81 if _, err := r.Peek(1); err != nil { 82 if err == io.EOF { 83 err = errShortRead 84 } 85 return sz, err 86 } 87 88 // Grab as many bytes as possible from the buffer, then go on to the 89 // next loop iteration which is going to consume it. 90 input, _ = r.Peek(r.Buffered()) 91 } 92 } 93 94 func readBool(r *bufio.Reader, sz int, v *bool) (int, error) { 95 return peekRead(r, sz, 1, func(b []byte) { *v = b[0] != 0 }) 96 } 97 98 func readString(r *bufio.Reader, sz int, v *string) (int, error) { 99 return readStringWith(r, sz, func(r *bufio.Reader, sz int, n int) (remain int, err error) { 100 *v, remain, err = readNewString(r, sz, n) 101 return 102 }) 103 } 104 105 func readStringWith(r *bufio.Reader, sz int, cb func(*bufio.Reader, int, int) (int, error)) (int, error) { 106 var err error 107 var len int16 108 109 if sz, err = readInt16(r, sz, &len); err != nil { 110 return sz, err 111 } 112 113 n := int(len) 114 if n > sz { 115 return sz, errShortRead 116 } 117 118 return cb(r, sz, n) 119 } 120 121 func readNewString(r *bufio.Reader, sz int, n int) (string, int, error) { 122 b, sz, err := readNewBytes(r, sz, n) 123 return string(b), sz, err 124 } 125 126 func readBytes(r *bufio.Reader, sz int, v *[]byte) (int, error) { 127 return readBytesWith(r, sz, func(r *bufio.Reader, sz int, n int) (remain int, err error) { 128 *v, remain, err = readNewBytes(r, sz, n) 129 return 130 }) 131 } 132 133 func readBytesWith(r *bufio.Reader, sz int, cb func(*bufio.Reader, int, int) (int, error)) (int, error) { 134 var err error 135 var n int 136 137 if sz, err = readArrayLen(r, sz, &n); err != nil { 138 return sz, err 139 } 140 141 if n > sz { 142 return sz, errShortRead 143 } 144 145 return cb(r, sz, n) 146 } 147 148 func readNewBytes(r *bufio.Reader, sz int, n int) ([]byte, int, error) { 149 var err error 150 var b []byte 151 var shortRead bool 152 153 if n > 0 { 154 if sz < n { 155 n = sz 156 shortRead = true 157 } 158 159 b = make([]byte, n) 160 n, err = io.ReadFull(r, b) 161 b = b[:n] 162 sz -= n 163 164 if err == nil && shortRead { 165 err = errShortRead 166 } 167 } 168 169 return b, sz, err 170 } 171 172 func readArrayLen(r *bufio.Reader, sz int, n *int) (int, error) { 173 var err error 174 var len int32 175 if sz, err = readInt32(r, sz, &len); err != nil { 176 return sz, err 177 } 178 *n = int(len) 179 return sz, nil 180 } 181 182 func readArrayWith(r *bufio.Reader, sz int, cb func(*bufio.Reader, int) (int, error)) (int, error) { 183 var err error 184 var len int32 185 186 if sz, err = readInt32(r, sz, &len); err != nil { 187 return sz, err 188 } 189 190 for n := int(len); n > 0; n-- { 191 if sz, err = cb(r, sz); err != nil { 192 break 193 } 194 } 195 196 return sz, err 197 } 198 199 func readStringArray(r *bufio.Reader, sz int, v *[]string) (remain int, err error) { 200 var content []string 201 fn := func(r *bufio.Reader, size int) (fnRemain int, fnErr error) { 202 var value string 203 if fnRemain, fnErr = readString(r, size, &value); fnErr != nil { 204 return 205 } 206 content = append(content, value) 207 return 208 } 209 if remain, err = readArrayWith(r, sz, fn); err != nil { 210 return 211 } 212 213 *v = content 214 return 215 } 216 217 func readMapStringInt32(r *bufio.Reader, sz int, v *map[string][]int32) (remain int, err error) { 218 var len int32 219 if remain, err = readInt32(r, sz, &len); err != nil { 220 return 221 } 222 223 content := make(map[string][]int32, len) 224 for i := 0; i < int(len); i++ { 225 var key string 226 var values []int32 227 228 if remain, err = readString(r, remain, &key); err != nil { 229 return 230 } 231 232 fn := func(r *bufio.Reader, size int) (fnRemain int, fnErr error) { 233 var value int32 234 if fnRemain, fnErr = readInt32(r, size, &value); fnErr != nil { 235 return 236 } 237 values = append(values, value) 238 return 239 } 240 if remain, err = readArrayWith(r, remain, fn); err != nil { 241 return 242 } 243 244 content[key] = values 245 } 246 *v = content 247 248 return 249 } 250 251 func read(r *bufio.Reader, sz int, a interface{}) (int, error) { 252 switch v := a.(type) { 253 case *int8: 254 return readInt8(r, sz, v) 255 case *int16: 256 return readInt16(r, sz, v) 257 case *int32: 258 return readInt32(r, sz, v) 259 case *int64: 260 return readInt64(r, sz, v) 261 case *bool: 262 return readBool(r, sz, v) 263 case *string: 264 return readString(r, sz, v) 265 case *[]byte: 266 return readBytes(r, sz, v) 267 } 268 switch v := reflect.ValueOf(a).Elem(); v.Kind() { 269 case reflect.Struct: 270 return readStruct(r, sz, v) 271 case reflect.Slice: 272 return readSlice(r, sz, v) 273 default: 274 panic(fmt.Sprintf("unsupported type: %T", a)) 275 } 276 } 277 278 func readAll(r *bufio.Reader, sz int, ptrs ...interface{}) (int, error) { 279 var err error 280 281 for _, ptr := range ptrs { 282 if sz, err = readPtr(r, sz, ptr); err != nil { 283 break 284 } 285 } 286 287 return sz, err 288 } 289 290 func readPtr(r *bufio.Reader, sz int, ptr interface{}) (int, error) { 291 switch v := ptr.(type) { 292 case *int8: 293 return readInt8(r, sz, v) 294 case *int16: 295 return readInt16(r, sz, v) 296 case *int32: 297 return readInt32(r, sz, v) 298 case *int64: 299 return readInt64(r, sz, v) 300 case *string: 301 return readString(r, sz, v) 302 case *[]byte: 303 return readBytes(r, sz, v) 304 case readable: 305 return v.readFrom(r, sz) 306 default: 307 panic(fmt.Sprintf("unsupported type: %T", v)) 308 } 309 } 310 311 func readStruct(r *bufio.Reader, sz int, v reflect.Value) (int, error) { 312 var err error 313 for i, n := 0, v.NumField(); i != n; i++ { 314 if sz, err = read(r, sz, v.Field(i).Addr().Interface()); err != nil { 315 return sz, err 316 } 317 } 318 return sz, nil 319 } 320 321 func readSlice(r *bufio.Reader, sz int, v reflect.Value) (int, error) { 322 var err error 323 var len int32 324 325 if sz, err = readInt32(r, sz, &len); err != nil { 326 return sz, err 327 } 328 329 if n := int(len); n < 0 { 330 v.Set(reflect.Zero(v.Type())) 331 } else { 332 v.Set(reflect.MakeSlice(v.Type(), n, n)) 333 334 for i := 0; i != n; i++ { 335 if sz, err = read(r, sz, v.Index(i).Addr().Interface()); err != nil { 336 return sz, err 337 } 338 } 339 } 340 341 return sz, nil 342 } 343 344 func readFetchResponseHeaderV2(r *bufio.Reader, size int) (throttle int32, watermark int64, remain int, err error) { 345 var n int32 346 var p struct { 347 Partition int32 348 ErrorCode int16 349 HighwaterMarkOffset int64 350 MessageSetSize int32 351 } 352 353 if remain, err = readInt32(r, size, &throttle); err != nil { 354 return 355 } 356 357 if remain, err = readInt32(r, remain, &n); err != nil { 358 return 359 } 360 361 // This error should never trigger, unless there's a bug in the kafka client 362 // or server. 363 if n != 1 { 364 err = fmt.Errorf("1 kafka topic was expected in the fetch response but the client received %d", n) 365 return 366 } 367 368 // We ignore the topic name because we've requests messages for a single 369 // topic, unless there's a bug in the kafka server we will have received 370 // the name of the topic that we requested. 371 if remain, err = discardString(r, remain); err != nil { 372 return 373 } 374 375 if remain, err = readInt32(r, remain, &n); err != nil { 376 return 377 } 378 379 // This error should never trigger, unless there's a bug in the kafka client 380 // or server. 381 if n != 1 { 382 err = fmt.Errorf("1 kafka partition was expected in the fetch response but the client received %d", n) 383 return 384 } 385 386 if remain, err = read(r, remain, &p); err != nil { 387 return 388 } 389 390 if p.ErrorCode != 0 { 391 err = Error(p.ErrorCode) 392 return 393 } 394 395 // This error should never trigger, unless there's a bug in the kafka client 396 // or server. 397 if remain != int(p.MessageSetSize) { 398 err = fmt.Errorf("the size of the message set in a fetch response doesn't match the number of remaining bytes (message set size = %d, remaining bytes = %d)", p.MessageSetSize, remain) 399 return 400 } 401 402 watermark = p.HighwaterMarkOffset 403 return 404 } 405 406 func readFetchResponseHeaderV5(r *bufio.Reader, size int) (throttle int32, watermark int64, remain int, err error) { 407 var n int32 408 type AbortedTransaction struct { 409 ProducerId int64 410 FirstOffset int64 411 } 412 var p struct { 413 Partition int32 414 ErrorCode int16 415 HighwaterMarkOffset int64 416 LastStableOffset int64 417 LogStartOffset int64 418 } 419 var messageSetSize int32 420 var abortedTransactions []AbortedTransaction 421 422 if remain, err = readInt32(r, size, &throttle); err != nil { 423 return 424 } 425 426 if remain, err = readInt32(r, remain, &n); err != nil { 427 return 428 } 429 430 // This error should never trigger, unless there's a bug in the kafka client 431 // or server. 432 if n != 1 { 433 err = fmt.Errorf("1 kafka topic was expected in the fetch response but the client received %d", n) 434 return 435 } 436 437 // We ignore the topic name because we've requests messages for a single 438 // topic, unless there's a bug in the kafka server we will have received 439 // the name of the topic that we requested. 440 if remain, err = discardString(r, remain); err != nil { 441 return 442 } 443 444 if remain, err = readInt32(r, remain, &n); err != nil { 445 return 446 } 447 448 // This error should never trigger, unless there's a bug in the kafka client 449 // or server. 450 if n != 1 { 451 err = fmt.Errorf("1 kafka partition was expected in the fetch response but the client received %d", n) 452 return 453 } 454 455 if remain, err = read(r, remain, &p); err != nil { 456 return 457 } 458 459 var abortedTransactionLen int 460 if remain, err = readArrayLen(r, remain, &abortedTransactionLen); err != nil { 461 return 462 } 463 464 if abortedTransactionLen == -1 { 465 abortedTransactions = nil 466 } else { 467 abortedTransactions = make([]AbortedTransaction, abortedTransactionLen) 468 for i := 0; i < abortedTransactionLen; i++ { 469 if remain, err = read(r, remain, &abortedTransactions[i]); err != nil { 470 return 471 } 472 } 473 } 474 475 if p.ErrorCode != 0 { 476 err = Error(p.ErrorCode) 477 return 478 } 479 480 remain, err = readInt32(r, remain, &messageSetSize) 481 if err != nil { 482 return 483 } 484 485 // This error should never trigger, unless there's a bug in the kafka client 486 // or server. 487 if remain != int(messageSetSize) { 488 err = fmt.Errorf("the size of the message set in a fetch response doesn't match the number of remaining bytes (message set size = %d, remaining bytes = %d)", messageSetSize, remain) 489 return 490 } 491 492 watermark = p.HighwaterMarkOffset 493 return 494 495 } 496 497 func readFetchResponseHeaderV10(r *bufio.Reader, size int) (throttle int32, watermark int64, remain int, err error) { 498 var n int32 499 var errorCode int16 500 type AbortedTransaction struct { 501 ProducerId int64 502 FirstOffset int64 503 } 504 var p struct { 505 Partition int32 506 ErrorCode int16 507 HighwaterMarkOffset int64 508 LastStableOffset int64 509 LogStartOffset int64 510 } 511 var messageSetSize int32 512 var abortedTransactions []AbortedTransaction 513 514 if remain, err = readInt32(r, size, &throttle); err != nil { 515 return 516 } 517 518 if remain, err = readInt16(r, remain, &errorCode); err != nil { 519 return 520 } 521 if errorCode != 0 { 522 err = Error(errorCode) 523 return 524 } 525 526 if remain, err = discardInt32(r, remain); err != nil { 527 return 528 } 529 530 if remain, err = readInt32(r, remain, &n); err != nil { 531 return 532 } 533 534 // This error should never trigger, unless there's a bug in the kafka client 535 // or server. 536 if n != 1 { 537 err = fmt.Errorf("1 kafka topic was expected in the fetch response but the client received %d", n) 538 return 539 } 540 541 // We ignore the topic name because we've requests messages for a single 542 // topic, unless there's a bug in the kafka server we will have received 543 // the name of the topic that we requested. 544 if remain, err = discardString(r, remain); err != nil { 545 return 546 } 547 548 if remain, err = readInt32(r, remain, &n); err != nil { 549 return 550 } 551 552 // This error should never trigger, unless there's a bug in the kafka client 553 // or server. 554 if n != 1 { 555 err = fmt.Errorf("1 kafka partition was expected in the fetch response but the client received %d", n) 556 return 557 } 558 559 if remain, err = read(r, remain, &p); err != nil { 560 return 561 } 562 563 var abortedTransactionLen int 564 if remain, err = readArrayLen(r, remain, &abortedTransactionLen); err != nil { 565 return 566 } 567 568 if abortedTransactionLen == -1 { 569 abortedTransactions = nil 570 } else { 571 abortedTransactions = make([]AbortedTransaction, abortedTransactionLen) 572 for i := 0; i < abortedTransactionLen; i++ { 573 if remain, err = read(r, remain, &abortedTransactions[i]); err != nil { 574 return 575 } 576 } 577 } 578 579 if p.ErrorCode != 0 { 580 err = Error(p.ErrorCode) 581 return 582 } 583 584 remain, err = readInt32(r, remain, &messageSetSize) 585 if err != nil { 586 return 587 } 588 589 // This error should never trigger, unless there's a bug in the kafka client 590 // or server. 591 if remain != int(messageSetSize) { 592 err = fmt.Errorf("the size of the message set in a fetch response doesn't match the number of remaining bytes (message set size = %d, remaining bytes = %d)", messageSetSize, remain) 593 return 594 } 595 596 watermark = p.HighwaterMarkOffset 597 return 598 599 } 600 601 func readMessageHeader(r *bufio.Reader, sz int) (offset int64, attributes int8, timestamp int64, remain int, err error) { 602 var version int8 603 604 if remain, err = readInt64(r, sz, &offset); err != nil { 605 return 606 } 607 608 // On discarding the message size and CRC: 609 // --------------------------------------- 610 // 611 // - Not sure why kafka gives the message size here, we already have the 612 // number of remaining bytes in the response and kafka should only truncate 613 // the trailing message. 614 // 615 // - TCP is already taking care of ensuring data integrity, no need to 616 // waste resources doing it a second time so we just skip the message CRC. 617 // 618 if remain, err = discardN(r, remain, 8); err != nil { 619 return 620 } 621 622 if remain, err = readInt8(r, remain, &version); err != nil { 623 return 624 } 625 626 if remain, err = readInt8(r, remain, &attributes); err != nil { 627 return 628 } 629 630 switch version { 631 case 0: 632 case 1: 633 remain, err = readInt64(r, remain, ×tamp) 634 default: 635 err = fmt.Errorf("unsupported message version %d found in fetch response", version) 636 } 637 638 return 639 }