github.com/Kolosok86/http@v0.1.2/transfer.go (about) 1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package http 6 7 import ( 8 "bufio" 9 "bytes" 10 "errors" 11 "fmt" 12 "io" 13 "reflect" 14 "sort" 15 "strconv" 16 "strings" 17 "sync" 18 "time" 19 20 "github.com/Kolosok86/http/httptrace" 21 "github.com/Kolosok86/http/internal" 22 "github.com/Kolosok86/http/internal/ascii" 23 "github.com/Kolosok86/http/textproto" 24 "golang.org/x/net/http/httpguts" 25 ) 26 27 // ErrLineTooLong is returned when reading request or response bodies 28 // with malformed chunked encoding. 29 var ErrLineTooLong = internal.ErrLineTooLong 30 31 type errorReader struct { 32 err error 33 } 34 35 func (r errorReader) Read(p []byte) (n int, err error) { 36 return 0, r.err 37 } 38 39 type byteReader struct { 40 b byte 41 done bool 42 } 43 44 func (br *byteReader) Read(p []byte) (n int, err error) { 45 if br.done { 46 return 0, io.EOF 47 } 48 if len(p) == 0 { 49 return 0, nil 50 } 51 br.done = true 52 p[0] = br.b 53 return 1, io.EOF 54 } 55 56 // transferWriter inspects the fields of a user-supplied Request or Response, 57 // sanitizes them without changing the user object and provides methods for 58 // writing the respective header, body and trailer in wire format. 59 type transferWriter struct { 60 Method string 61 Body io.Reader 62 BodyCloser io.Closer 63 ResponseToHEAD bool 64 ContentLength int64 // -1 means unknown, 0 means exactly none 65 Close bool 66 TransferEncoding []string 67 Header Header 68 Trailer Header 69 IsResponse bool 70 bodyReadError error // any non-EOF error from reading Body 71 72 FlushHeaders bool // flush headers to network before body 73 ByteReadCh chan readResult // non-nil if probeRequestBody called 74 } 75 76 func newTransferWriter(r any) (t *transferWriter, err error) { 77 t = &transferWriter{} 78 79 // Extract relevant fields 80 atLeastHTTP11 := false 81 switch rr := r.(type) { 82 case *Request: 83 if rr.ContentLength != 0 && rr.Body == nil { 84 return nil, fmt.Errorf("http: Request.ContentLength=%d with nil Body", rr.ContentLength) 85 } 86 t.Method = valueOrDefault(rr.Method, "GET") 87 t.Close = rr.Close 88 t.TransferEncoding = rr.TransferEncoding 89 t.Header = rr.Header 90 t.Trailer = rr.Trailer 91 t.Body = rr.Body 92 t.BodyCloser = rr.Body 93 t.ContentLength = rr.outgoingLength() 94 if t.ContentLength < 0 && len(t.TransferEncoding) == 0 && t.shouldSendChunkedRequestBody() { 95 t.TransferEncoding = []string{"chunked"} 96 } 97 // If there's a body, conservatively flush the headers 98 // to any bufio.Writer we're writing to, just in case 99 // the server needs the headers early, before we copy 100 // the body and possibly block. We make an exception 101 // for the common standard library in-memory types, 102 // though, to avoid unnecessary TCP packets on the 103 // wire. (Issue 22088.) 104 if t.ContentLength != 0 && !isKnownInMemoryReader(t.Body) { 105 t.FlushHeaders = true 106 } 107 108 atLeastHTTP11 = true // Transport requests are always 1.1 or 2.0 109 case *Response: 110 t.IsResponse = true 111 if rr.Request != nil { 112 t.Method = rr.Request.Method 113 } 114 t.Body = rr.Body 115 t.BodyCloser = rr.Body 116 t.ContentLength = rr.ContentLength 117 t.Close = rr.Close 118 t.TransferEncoding = rr.TransferEncoding 119 t.Header = rr.Header 120 t.Trailer = rr.Trailer 121 atLeastHTTP11 = rr.ProtoAtLeast(1, 1) 122 t.ResponseToHEAD = noResponseBodyExpected(t.Method) 123 } 124 125 // Sanitize Body,ContentLength,TransferEncoding 126 if t.ResponseToHEAD { 127 t.Body = nil 128 if chunked(t.TransferEncoding) { 129 t.ContentLength = -1 130 } 131 } else { 132 if !atLeastHTTP11 || t.Body == nil { 133 t.TransferEncoding = nil 134 } 135 if chunked(t.TransferEncoding) { 136 t.ContentLength = -1 137 } else if t.Body == nil { // no chunking, no body 138 t.ContentLength = 0 139 } 140 } 141 142 // Sanitize Trailer 143 if !chunked(t.TransferEncoding) { 144 t.Trailer = nil 145 } 146 147 return t, nil 148 } 149 150 // shouldSendChunkedRequestBody reports whether we should try to send a 151 // chunked request body to the server. In particular, the case we really 152 // want to prevent is sending a GET or other typically-bodyless request to a 153 // server with a chunked body when the body has zero bytes, since GETs with 154 // bodies (while acceptable according to specs), even zero-byte chunked 155 // bodies, are approximately never seen in the wild and confuse most 156 // servers. See Issue 18257, as one example. 157 // 158 // The only reason we'd send such a request is if the user set the Body to a 159 // non-nil value (say, io.NopCloser(bytes.NewReader(nil))) and didn't 160 // set ContentLength, or NewRequest set it to -1 (unknown), so then we assume 161 // there's bytes to send. 162 // 163 // This code tries to read a byte from the Request.Body in such cases to see 164 // whether the body actually has content (super rare) or is actually just 165 // a non-nil content-less ReadCloser (the more common case). In that more 166 // common case, we act as if their Body were nil instead, and don't send 167 // a body. 168 func (t *transferWriter) shouldSendChunkedRequestBody() bool { 169 // Note that t.ContentLength is the corrected content length 170 // from rr.outgoingLength, so 0 actually means zero, not unknown. 171 if t.ContentLength >= 0 || t.Body == nil { // redundant checks; caller did them 172 return false 173 } 174 if t.Method == "CONNECT" { 175 return false 176 } 177 if requestMethodUsuallyLacksBody(t.Method) { 178 // Only probe the Request.Body for GET/HEAD/DELETE/etc 179 // requests, because it's only those types of requests 180 // that confuse servers. 181 t.probeRequestBody() // adjusts t.Body, t.ContentLength 182 return t.Body != nil 183 } 184 // For all other request types (PUT, POST, PATCH, or anything 185 // made-up we've never heard of), assume it's normal and the server 186 // can deal with a chunked request body. Maybe we'll adjust this 187 // later. 188 return true 189 } 190 191 // probeRequestBody reads a byte from t.Body to see whether it's empty 192 // (returns io.EOF right away). 193 // 194 // But because we've had problems with this blocking users in the past 195 // (issue 17480) when the body is a pipe (perhaps waiting on the response 196 // headers before the pipe is fed data), we need to be careful and bound how 197 // long we wait for it. This delay will only affect users if all the following 198 // are true: 199 // - the request body blocks 200 // - the content length is not set (or set to -1) 201 // - the method doesn't usually have a body (GET, HEAD, DELETE, ...) 202 // - there is no transfer-encoding=chunked already set. 203 // 204 // In other words, this delay will not normally affect anybody, and there 205 // are workarounds if it does. 206 func (t *transferWriter) probeRequestBody() { 207 t.ByteReadCh = make(chan readResult, 1) 208 go func(body io.Reader) { 209 var buf [1]byte 210 var rres readResult 211 rres.n, rres.err = body.Read(buf[:]) 212 if rres.n == 1 { 213 rres.b = buf[0] 214 } 215 t.ByteReadCh <- rres 216 close(t.ByteReadCh) 217 }(t.Body) 218 timer := time.NewTimer(200 * time.Millisecond) 219 select { 220 case rres := <-t.ByteReadCh: 221 timer.Stop() 222 if rres.n == 0 && rres.err == io.EOF { 223 // It was empty. 224 t.Body = nil 225 t.ContentLength = 0 226 } else if rres.n == 1 { 227 if rres.err != nil { 228 t.Body = io.MultiReader(&byteReader{b: rres.b}, errorReader{rres.err}) 229 } else { 230 t.Body = io.MultiReader(&byteReader{b: rres.b}, t.Body) 231 } 232 } else if rres.err != nil { 233 t.Body = errorReader{rres.err} 234 } 235 case <-timer.C: 236 // Too slow. Don't wait. Read it later, and keep 237 // assuming that this is ContentLength == -1 238 // (unknown), which means we'll send a 239 // "Transfer-Encoding: chunked" header. 240 t.Body = io.MultiReader(finishAsyncByteRead{t}, t.Body) 241 // Request that Request.Write flush the headers to the 242 // network before writing the body, since our body may not 243 // become readable until it's seen the response headers. 244 t.FlushHeaders = true 245 } 246 } 247 248 func noResponseBodyExpected(requestMethod string) bool { 249 return requestMethod == "HEAD" 250 } 251 252 func (t *transferWriter) shouldSendContentLength() bool { 253 if chunked(t.TransferEncoding) { 254 return false 255 } 256 if t.ContentLength > 0 { 257 return true 258 } 259 if t.ContentLength < 0 { 260 return false 261 } 262 // Many servers expect a Content-Length for these methods 263 if t.Method == "POST" || t.Method == "PUT" || t.Method == "PATCH" { 264 return true 265 } 266 if t.ContentLength == 0 && isIdentity(t.TransferEncoding) { 267 if t.Method == "GET" || t.Method == "HEAD" { 268 return false 269 } 270 return true 271 } 272 273 return false 274 } 275 276 func (t *transferWriter) writeHeader(w io.Writer, trace *httptrace.ClientTrace) error { 277 if t.Close && !hasToken(t.Header.get("Connection"), "close") { 278 if trace != nil && trace.WroteHeaderField != nil { 279 trace.WroteHeaderField("Connection", []string{"close"}) 280 } 281 } 282 283 // Write Content-Length and/or Transfer-Encoding whose Values are a 284 // function of the sanitized field triple (Body, ContentLength, 285 // TransferEncoding) 286 if t.shouldSendContentLength() { 287 if _, err := io.WriteString(w, "Content-Length: "); err != nil { 288 return err 289 } 290 if _, err := io.WriteString(w, strconv.FormatInt(t.ContentLength, 10)+"\r\n"); err != nil { 291 return err 292 } 293 if trace != nil && trace.WroteHeaderField != nil { 294 trace.WroteHeaderField("Content-Length", []string{strconv.FormatInt(t.ContentLength, 10)}) 295 } 296 } else if chunked(t.TransferEncoding) { 297 if _, err := io.WriteString(w, "Transfer-Encoding: chunked\r\n"); err != nil { 298 return err 299 } 300 if trace != nil && trace.WroteHeaderField != nil { 301 trace.WroteHeaderField("Transfer-Encoding", []string{"chunked"}) 302 } 303 } 304 305 // Write Trailer header 306 if t.Trailer != nil { 307 keys := make([]string, 0, len(t.Trailer)) 308 for k := range t.Trailer { 309 k = CanonicalHeaderKey(k) 310 switch k { 311 case "Transfer-Encoding", "Trailer", "Content-Length": 312 return badStringError("invalid Trailer Key", k) 313 } 314 keys = append(keys, k) 315 } 316 if len(keys) > 0 { 317 sort.Strings(keys) 318 // TODO: could do better allocation-wise here, but trailers are rare, 319 // so being lazy for now. 320 if _, err := io.WriteString(w, "Trailer: "+strings.Join(keys, ",")+"\r\n"); err != nil { 321 return err 322 } 323 if trace != nil && trace.WroteHeaderField != nil { 324 trace.WroteHeaderField("Trailer", keys) 325 } 326 } 327 } 328 329 return nil 330 } 331 332 // always closes t.BodyCloser 333 func (t *transferWriter) writeBody(w io.Writer) (err error) { 334 var ncopy int64 335 closed := false 336 defer func() { 337 if closed || t.BodyCloser == nil { 338 return 339 } 340 if closeErr := t.BodyCloser.Close(); closeErr != nil && err == nil { 341 err = closeErr 342 } 343 }() 344 345 // Write body. We "unwrap" the body first if it was wrapped in a 346 // nopCloser or readTrackingBody. This is to ensure that we can take advantage of 347 // OS-level optimizations in the event that the body is an 348 // *os.File. 349 if t.Body != nil { 350 var body = t.unwrapBody() 351 if chunked(t.TransferEncoding) { 352 if bw, ok := w.(*bufio.Writer); ok && !t.IsResponse { 353 w = &internal.FlushAfterChunkWriter{Writer: bw} 354 } 355 cw := internal.NewChunkedWriter(w) 356 _, err = t.doBodyCopy(cw, body) 357 if err == nil { 358 err = cw.Close() 359 } 360 } else if t.ContentLength == -1 { 361 dst := w 362 if t.Method == "CONNECT" { 363 dst = bufioFlushWriter{dst} 364 } 365 ncopy, err = t.doBodyCopy(dst, body) 366 } else { 367 ncopy, err = t.doBodyCopy(w, io.LimitReader(body, t.ContentLength)) 368 if err != nil { 369 return err 370 } 371 var nextra int64 372 nextra, err = t.doBodyCopy(io.Discard, body) 373 ncopy += nextra 374 } 375 if err != nil { 376 return err 377 } 378 } 379 if t.BodyCloser != nil { 380 closed = true 381 if err := t.BodyCloser.Close(); err != nil { 382 return err 383 } 384 } 385 386 if !t.ResponseToHEAD && t.ContentLength != -1 && t.ContentLength != ncopy { 387 return fmt.Errorf("http: ContentLength=%d with Body length %d", 388 t.ContentLength, ncopy) 389 } 390 391 if chunked(t.TransferEncoding) { 392 // Write Trailer header 393 if t.Trailer != nil { 394 if err := t.Trailer.Write(w); err != nil { 395 return err 396 } 397 } 398 // Last chunk, empty trailer 399 _, err = io.WriteString(w, "\r\n") 400 } 401 return err 402 } 403 404 // doBodyCopy wraps a copy operation, with any resulting error also 405 // being saved in bodyReadError. 406 // 407 // This function is only intended for use in writeBody. 408 func (t *transferWriter) doBodyCopy(dst io.Writer, src io.Reader) (n int64, err error) { 409 n, err = io.Copy(dst, src) 410 if err != nil && err != io.EOF { 411 t.bodyReadError = err 412 } 413 return 414 } 415 416 // unwrapBody unwraps the body's inner reader if it's a 417 // nopCloser. This is to ensure that body writes sourced from local 418 // files (*os.File types) are properly optimized. 419 // 420 // This function is only intended for use in writeBody. 421 func (t *transferWriter) unwrapBody() io.Reader { 422 if r, ok := unwrapNopCloser(t.Body); ok { 423 return r 424 } 425 if r, ok := t.Body.(*readTrackingBody); ok { 426 r.didRead = true 427 return r.ReadCloser 428 } 429 return t.Body 430 } 431 432 type transferReader struct { 433 // Input 434 Header Header 435 StatusCode int 436 RequestMethod string 437 ProtoMajor int 438 ProtoMinor int 439 // Output 440 Body io.ReadCloser 441 ContentLength int64 442 Chunked bool 443 Close bool 444 Trailer Header 445 } 446 447 func (t *transferReader) protoAtLeast(m, n int) bool { 448 return t.ProtoMajor > m || (t.ProtoMajor == m && t.ProtoMinor >= n) 449 } 450 451 // bodyAllowedForStatus reports whether a given response status code 452 // permits a body. See RFC 7230, section 3.3. 453 func bodyAllowedForStatus(status int) bool { 454 switch { 455 case status >= 100 && status <= 199: 456 return false 457 case status == 204: 458 return false 459 case status == 304: 460 return false 461 } 462 return true 463 } 464 465 var ( 466 suppressedHeaders304 = []string{"Content-Type", "Content-Length", "Transfer-Encoding"} 467 suppressedHeadersNoBody = []string{"Content-Length", "Transfer-Encoding"} 468 excludedHeadersNoBody = map[string]bool{"Content-Length": true, "Transfer-Encoding": true} 469 ) 470 471 func suppressedHeaders(status int) []string { 472 switch { 473 case status == 304: 474 // RFC 7232 section 4.1 475 return suppressedHeaders304 476 case !bodyAllowedForStatus(status): 477 return suppressedHeadersNoBody 478 } 479 return nil 480 } 481 482 // msg is *Request or *Response. 483 func readTransfer(msg any, r *bufio.Reader) (err error) { 484 t := &transferReader{RequestMethod: "GET"} 485 486 // Unify input 487 isResponse := false 488 switch rr := msg.(type) { 489 case *Response: 490 t.Header = rr.Header 491 t.StatusCode = rr.StatusCode 492 t.ProtoMajor = rr.ProtoMajor 493 t.ProtoMinor = rr.ProtoMinor 494 t.Close = shouldClose(t.ProtoMajor, t.ProtoMinor, t.Header, true) 495 isResponse = true 496 if rr.Request != nil { 497 t.RequestMethod = rr.Request.Method 498 } 499 case *Request: 500 t.Header = rr.Header 501 t.RequestMethod = rr.Method 502 t.ProtoMajor = rr.ProtoMajor 503 t.ProtoMinor = rr.ProtoMinor 504 // Transfer semantics for Requests are exactly like those for 505 // Responses with status code 200, responding to a GET method 506 t.StatusCode = 200 507 t.Close = rr.Close 508 default: 509 panic("unexpected type") 510 } 511 512 // Default to HTTP/1.1 513 if t.ProtoMajor == 0 && t.ProtoMinor == 0 { 514 t.ProtoMajor, t.ProtoMinor = 1, 1 515 } 516 517 // Transfer-Encoding: chunked, and overriding Content-Length. 518 if err := t.parseTransferEncoding(); err != nil { 519 return err 520 } 521 522 realLength, err := fixLength(isResponse, t.StatusCode, t.RequestMethod, t.Header, t.Chunked) 523 if err != nil { 524 return err 525 } 526 if isResponse && t.RequestMethod == "HEAD" { 527 if n, err := parseContentLength(t.Header.get("Content-Length")); err != nil { 528 return err 529 } else { 530 t.ContentLength = n 531 } 532 } else { 533 t.ContentLength = realLength 534 } 535 536 // Trailer 537 t.Trailer, err = fixTrailer(t.Header, t.Chunked) 538 if err != nil { 539 return err 540 } 541 542 // If there is no Content-Length or chunked Transfer-Encoding on a *Response 543 // and the status is not 1xx, 204 or 304, then the body is unbounded. 544 // See RFC 7230, section 3.3. 545 switch msg.(type) { 546 case *Response: 547 if realLength == -1 && !t.Chunked && bodyAllowedForStatus(t.StatusCode) { 548 // Unbounded body. 549 t.Close = true 550 } 551 } 552 553 // Prepare body reader. ContentLength < 0 means chunked encoding 554 // or close connection when finished, since multipart is not supported yet 555 switch { 556 case t.Chunked: 557 if isResponse && (noResponseBodyExpected(t.RequestMethod) || !bodyAllowedForStatus(t.StatusCode)) { 558 t.Body = NoBody 559 } else { 560 t.Body = &body{src: internal.NewChunkedReader(r), hdr: msg, r: r, closing: t.Close} 561 } 562 case realLength == 0: 563 t.Body = NoBody 564 case realLength > 0: 565 t.Body = &body{src: io.LimitReader(r, realLength), closing: t.Close} 566 default: 567 // realLength < 0, i.e. "Content-Length" not mentioned in header 568 if t.Close { 569 // Close semantics (i.e. HTTP/1.0) 570 t.Body = &body{src: r, closing: t.Close} 571 } else { 572 // Persistent connection (i.e. HTTP/1.1) 573 t.Body = NoBody 574 } 575 } 576 577 // Unify output 578 switch rr := msg.(type) { 579 case *Request: 580 rr.Body = t.Body 581 rr.ContentLength = t.ContentLength 582 if t.Chunked { 583 rr.TransferEncoding = []string{"chunked"} 584 } 585 rr.Close = t.Close 586 rr.Trailer = t.Trailer 587 case *Response: 588 rr.Body = t.Body 589 rr.ContentLength = t.ContentLength 590 if t.Chunked { 591 rr.TransferEncoding = []string{"chunked"} 592 } 593 rr.Close = t.Close 594 rr.Trailer = t.Trailer 595 } 596 597 return nil 598 } 599 600 // Checks whether chunked is part of the encodings stack. 601 func chunked(te []string) bool { return len(te) > 0 && te[0] == "chunked" } 602 603 // Checks whether the encoding is explicitly "identity". 604 func isIdentity(te []string) bool { return len(te) == 1 && te[0] == "identity" } 605 606 // unsupportedTEError reports unsupported transfer-encodings. 607 type unsupportedTEError struct { 608 err string 609 } 610 611 func (uste *unsupportedTEError) Error() string { 612 return uste.err 613 } 614 615 // isUnsupportedTEError checks if the error is of type 616 // unsupportedTEError. It is usually invoked with a non-nil err. 617 func isUnsupportedTEError(err error) bool { 618 _, ok := err.(*unsupportedTEError) 619 return ok 620 } 621 622 // parseTransferEncoding sets t.Chunked based on the Transfer-Encoding header. 623 func (t *transferReader) parseTransferEncoding() error { 624 raw, present := t.Header["Transfer-Encoding"] 625 if !present { 626 return nil 627 } 628 delete(t.Header, "Transfer-Encoding") 629 630 // Issue 12785; ignore Transfer-Encoding on HTTP/1.0 requests. 631 if !t.protoAtLeast(1, 1) { 632 return nil 633 } 634 635 // Like nginx, we only support a single Transfer-Encoding header field, and 636 // only if set to "chunked". This is one of the most security sensitive 637 // surfaces in HTTP/1.1 due to the risk of request smuggling, so we keep it 638 // strict and simple. 639 if len(raw) != 1 { 640 return &unsupportedTEError{fmt.Sprintf("too many transfer encodings: %q", raw)} 641 } 642 if !ascii.EqualFold(raw[0], "chunked") { 643 return &unsupportedTEError{fmt.Sprintf("unsupported transfer encoding: %q", raw[0])} 644 } 645 646 // RFC 7230 3.3.2 says "A sender MUST NOT send a Content-Length header field 647 // in any message that contains a Transfer-Encoding header field." 648 // 649 // but also: "If a message is received with both a Transfer-Encoding and a 650 // Content-Length header field, the Transfer-Encoding overrides the 651 // Content-Length. Such a message might indicate an attempt to perform 652 // request smuggling (Section 9.5) or response splitting (Section 9.4) and 653 // ought to be handled as an error. A sender MUST remove the received 654 // Content-Length field prior to forwarding such a message downstream." 655 // 656 // Reportedly, these appear in the wild. 657 delete(t.Header, "Content-Length") 658 659 t.Chunked = true 660 return nil 661 } 662 663 // Determine the expected body length, using RFC 7230 Section 3.3. This 664 // function is not a method, because ultimately it should be shared by 665 // ReadResponse and ReadRequest. 666 func fixLength(isResponse bool, status int, requestMethod string, header Header, chunked bool) (int64, error) { 667 isRequest := !isResponse 668 contentLens := header["Content-Length"] 669 670 // Hardening against HTTP request smuggling 671 if len(contentLens) > 1 { 672 // Per RFC 7230 Section 3.3.2, prevent multiple 673 // Content-Length headers if they differ in value. 674 // If there are dups of the value, remove the dups. 675 // See Issue 16490. 676 first := textproto.TrimString(contentLens[0]) 677 for _, ct := range contentLens[1:] { 678 if first != textproto.TrimString(ct) { 679 return 0, fmt.Errorf("http: message cannot contain multiple Content-Length headers; got %q", contentLens) 680 } 681 } 682 683 // deduplicate Content-Length 684 header.Del("Content-Length") 685 header.Add("Content-Length", first) 686 687 contentLens = header["Content-Length"] 688 } 689 690 // Logic based on response type or status 691 if isResponse && noResponseBodyExpected(requestMethod) { 692 return 0, nil 693 } 694 if status/100 == 1 { 695 return 0, nil 696 } 697 switch status { 698 case 204, 304: 699 return 0, nil 700 } 701 702 // Logic based on Transfer-Encoding 703 if chunked { 704 return -1, nil 705 } 706 707 // Logic based on Content-Length 708 var cl string 709 if len(contentLens) == 1 { 710 cl = textproto.TrimString(contentLens[0]) 711 } 712 if cl != "" { 713 n, err := parseContentLength(cl) 714 if err != nil { 715 return -1, err 716 } 717 return n, nil 718 } 719 header.Del("Content-Length") 720 721 if isRequest { 722 // RFC 7230 neither explicitly permits nor forbids an 723 // entity-body on a GET request so we permit one if 724 // declared, but we default to 0 here (not -1 below) 725 // if there's no mention of a body. 726 // Likewise, all other request methods are assumed to have 727 // no body if neither Transfer-Encoding chunked nor a 728 // Content-Length are set. 729 return 0, nil 730 } 731 732 // Body-EOF logic based on other methods (like closing, or chunked coding) 733 return -1, nil 734 } 735 736 // Determine whether to hang up after sending a request and body, or 737 // receiving a response and body 738 // 'header' is the request headers. 739 func shouldClose(major, minor int, header Header, removeCloseHeader bool) bool { 740 if major < 1 { 741 return true 742 } 743 744 conv := header["Connection"] 745 hasClose := httpguts.HeaderValuesContainsToken(conv, "close") 746 if major == 1 && minor == 0 { 747 return hasClose || !httpguts.HeaderValuesContainsToken(conv, "keep-alive") 748 } 749 750 if hasClose && removeCloseHeader { 751 header.Del("Connection") 752 } 753 754 return hasClose 755 } 756 757 // Parse the trailer header. 758 func fixTrailer(header Header, chunked bool) (Header, error) { 759 vv, ok := header["Trailer"] 760 if !ok { 761 return nil, nil 762 } 763 if !chunked { 764 // Trailer and no chunking: 765 // this is an invalid use case for trailer header. 766 // Nevertheless, no error will be returned and we 767 // let users decide if this is a valid HTTP message. 768 // The Trailer header will be kept in Response.Header 769 // but not populate Response.Trailer. 770 // See issue #27197. 771 return nil, nil 772 } 773 header.Del("Trailer") 774 775 trailer := make(Header) 776 var err error 777 for _, v := range vv { 778 foreachHeaderElement(v, func(key string) { 779 key = CanonicalHeaderKey(key) 780 switch key { 781 case "Transfer-Encoding", "Trailer", "Content-Length": 782 if err == nil { 783 err = badStringError("bad trailer Key", key) 784 return 785 } 786 } 787 trailer[key] = nil 788 }) 789 } 790 if err != nil { 791 return nil, err 792 } 793 if len(trailer) == 0 { 794 return nil, nil 795 } 796 return trailer, nil 797 } 798 799 // body turns a Reader into a ReadCloser. 800 // Close ensures that the body has been fully read 801 // and then reads the trailer if necessary. 802 type body struct { 803 src io.Reader 804 hdr any // non-nil (Response or Request) value means read trailer 805 r *bufio.Reader // underlying wire-format reader for the trailer 806 closing bool // is the connection to be closed after reading body? 807 doEarlyClose bool // whether Close should stop early 808 809 mu sync.Mutex // guards following, and calls to Read and Close 810 sawEOF bool 811 closed bool 812 earlyClose bool // Close called and we didn't read to the end of src 813 onHitEOF func() // if non-nil, func to call when EOF is Read 814 } 815 816 // ErrBodyReadAfterClose is returned when reading a Request or Response 817 // Body after the body has been closed. This typically happens when the body is 818 // read after an HTTP Handler calls WriteHeader or Write on its 819 // ResponseWriter. 820 var ErrBodyReadAfterClose = errors.New("http: invalid Read on closed Body") 821 822 func (b *body) Read(p []byte) (n int, err error) { 823 b.mu.Lock() 824 defer b.mu.Unlock() 825 if b.closed { 826 return 0, ErrBodyReadAfterClose 827 } 828 return b.readLocked(p) 829 } 830 831 // Must hold b.mu. 832 func (b *body) readLocked(p []byte) (n int, err error) { 833 if b.sawEOF { 834 return 0, io.EOF 835 } 836 n, err = b.src.Read(p) 837 838 if err == io.EOF { 839 b.sawEOF = true 840 // Chunked case. Read the trailer. 841 if b.hdr != nil { 842 if e := b.readTrailer(); e != nil { 843 err = e 844 // Something went wrong in the trailer, we must not allow any 845 // further reads of any kind to succeed from body, nor any 846 // subsequent requests on the server connection. See 847 // golang.org/issue/12027 848 b.sawEOF = false 849 b.closed = true 850 } 851 b.hdr = nil 852 } else { 853 // If the server declared the Content-Length, our body is a LimitedReader 854 // and we need to check whether this EOF arrived early. 855 if lr, ok := b.src.(*io.LimitedReader); ok && lr.N > 0 { 856 err = io.ErrUnexpectedEOF 857 } 858 } 859 } 860 861 // If we can return an EOF here along with the read data, do 862 // so. This is optional per the io.Reader contract, but doing 863 // so helps the HTTP transport code recycle its connection 864 // earlier (since it will see this EOF itself), even if the 865 // client doesn't do future reads or Close. 866 if err == nil && n > 0 { 867 if lr, ok := b.src.(*io.LimitedReader); ok && lr.N == 0 { 868 err = io.EOF 869 b.sawEOF = true 870 } 871 } 872 873 if b.sawEOF && b.onHitEOF != nil { 874 b.onHitEOF() 875 } 876 877 return n, err 878 } 879 880 var ( 881 singleCRLF = []byte("\r\n") 882 doubleCRLF = []byte("\r\n\r\n") 883 ) 884 885 func seeUpcomingDoubleCRLF(r *bufio.Reader) bool { 886 for peekSize := 4; ; peekSize++ { 887 // This loop stops when Peek returns an error, 888 // which it does when r's buffer has been filled. 889 buf, err := r.Peek(peekSize) 890 if bytes.HasSuffix(buf, doubleCRLF) { 891 return true 892 } 893 if err != nil { 894 break 895 } 896 } 897 return false 898 } 899 900 var errTrailerEOF = errors.New("http: unexpected EOF reading trailer") 901 902 func (b *body) readTrailer() error { 903 // The common case, since nobody uses trailers. 904 buf, err := b.r.Peek(2) 905 if bytes.Equal(buf, singleCRLF) { 906 b.r.Discard(2) 907 return nil 908 } 909 if len(buf) < 2 { 910 return errTrailerEOF 911 } 912 if err != nil { 913 return err 914 } 915 916 // Make sure there's a header terminator coming up, to prevent 917 // a DoS with an unbounded size Trailer. It's not easy to 918 // slip in a LimitReader here, as textproto.NewReader requires 919 // a concrete *bufio.Reader. Also, we can't get all the way 920 // back up to our conn's LimitedReader that *might* be backing 921 // this bufio.Reader. Instead, a hack: we iteratively Peek up 922 // to the bufio.Reader's max size, looking for a double CRLF. 923 // This limits the trailer to the underlying buffer size, typically 4kB. 924 if !seeUpcomingDoubleCRLF(b.r) { 925 return errors.New("http: suspiciously long trailer after chunked body") 926 } 927 928 hdr, _, err := textproto.NewReader(b.r).ReadMIMEHeader() 929 if err != nil { 930 if err == io.EOF { 931 return errTrailerEOF 932 } 933 return err 934 } 935 switch rr := b.hdr.(type) { 936 case *Request: 937 mergeSetHeader(&rr.Trailer, Header(hdr)) 938 case *Response: 939 mergeSetHeader(&rr.Trailer, Header(hdr)) 940 } 941 return nil 942 } 943 944 func mergeSetHeader(dst *Header, src Header) { 945 if *dst == nil { 946 *dst = src 947 return 948 } 949 for k, vv := range src { 950 (*dst)[k] = vv 951 } 952 } 953 954 // unreadDataSizeLocked returns the number of bytes of unread input. 955 // It returns -1 if unknown. 956 // b.mu must be held. 957 func (b *body) unreadDataSizeLocked() int64 { 958 if lr, ok := b.src.(*io.LimitedReader); ok { 959 return lr.N 960 } 961 return -1 962 } 963 964 func (b *body) Close() error { 965 b.mu.Lock() 966 defer b.mu.Unlock() 967 if b.closed { 968 return nil 969 } 970 var err error 971 switch { 972 case b.sawEOF: 973 // Already saw EOF, so no need going to look for it. 974 case b.hdr == nil && b.closing: 975 // no trailer and closing the connection next. 976 // no point in reading to EOF. 977 case b.doEarlyClose: 978 // Read up to maxPostHandlerReadBytes bytes of the body, looking 979 // for EOF (and trailers), so we can re-use this connection. 980 if lr, ok := b.src.(*io.LimitedReader); ok && lr.N > maxPostHandlerReadBytes { 981 // There was a declared Content-Length, and we have more bytes remaining 982 // than our maxPostHandlerReadBytes tolerance. So, give up. 983 b.earlyClose = true 984 } else { 985 var n int64 986 // Consume the body, or, which will also lead to us reading 987 // the trailer headers after the body, if present. 988 n, err = io.CopyN(io.Discard, bodyLocked{b}, maxPostHandlerReadBytes) 989 if err == io.EOF { 990 err = nil 991 } 992 if n == maxPostHandlerReadBytes { 993 b.earlyClose = true 994 } 995 } 996 default: 997 // Fully consume the body, which will also lead to us reading 998 // the trailer headers after the body, if present. 999 _, err = io.Copy(io.Discard, bodyLocked{b}) 1000 } 1001 b.closed = true 1002 return err 1003 } 1004 1005 func (b *body) didEarlyClose() bool { 1006 b.mu.Lock() 1007 defer b.mu.Unlock() 1008 return b.earlyClose 1009 } 1010 1011 // bodyRemains reports whether future Read calls might 1012 // yield data. 1013 func (b *body) bodyRemains() bool { 1014 b.mu.Lock() 1015 defer b.mu.Unlock() 1016 return !b.sawEOF 1017 } 1018 1019 func (b *body) registerOnHitEOF(fn func()) { 1020 b.mu.Lock() 1021 defer b.mu.Unlock() 1022 b.onHitEOF = fn 1023 } 1024 1025 // bodyLocked is an io.Reader reading from a *body when its mutex is 1026 // already held. 1027 type bodyLocked struct { 1028 b *body 1029 } 1030 1031 func (bl bodyLocked) Read(p []byte) (n int, err error) { 1032 if bl.b.closed { 1033 return 0, ErrBodyReadAfterClose 1034 } 1035 return bl.b.readLocked(p) 1036 } 1037 1038 // parseContentLength trims whitespace from s and returns -1 if no value 1039 // is set, or the value if it's >= 0. 1040 func parseContentLength(cl string) (int64, error) { 1041 cl = textproto.TrimString(cl) 1042 if cl == "" { 1043 return -1, nil 1044 } 1045 n, err := strconv.ParseUint(cl, 10, 63) 1046 if err != nil { 1047 return 0, badStringError("bad Content-Length", cl) 1048 } 1049 return int64(n), nil 1050 1051 } 1052 1053 // finishAsyncByteRead finishes reading the 1-byte sniff 1054 // from the ContentLength==0, Body!=nil case. 1055 type finishAsyncByteRead struct { 1056 tw *transferWriter 1057 } 1058 1059 func (fr finishAsyncByteRead) Read(p []byte) (n int, err error) { 1060 if len(p) == 0 { 1061 return 1062 } 1063 rres := <-fr.tw.ByteReadCh 1064 n, err = rres.n, rres.err 1065 if n == 1 { 1066 p[0] = rres.b 1067 } 1068 if err == nil { 1069 err = io.EOF 1070 } 1071 return 1072 } 1073 1074 var nopCloserType = reflect.TypeOf(io.NopCloser(nil)) 1075 var nopCloserWriterToType = reflect.TypeOf(io.NopCloser(struct { 1076 io.Reader 1077 io.WriterTo 1078 }{})) 1079 1080 // unwrapNopCloser return the underlying reader and true if r is a NopCloser 1081 // else it return false. 1082 func unwrapNopCloser(r io.Reader) (underlyingReader io.Reader, isNopCloser bool) { 1083 switch reflect.TypeOf(r) { 1084 case nopCloserType, nopCloserWriterToType: 1085 return reflect.ValueOf(r).Field(0).Interface().(io.Reader), true 1086 default: 1087 return nil, false 1088 } 1089 } 1090 1091 // isKnownInMemoryReader reports whether r is a type known to not 1092 // block on Read. Its caller uses this as an optional optimization to 1093 // send fewer TCP packets. 1094 func isKnownInMemoryReader(r io.Reader) bool { 1095 switch r.(type) { 1096 case *bytes.Reader, *bytes.Buffer, *strings.Reader: 1097 return true 1098 } 1099 if r, ok := unwrapNopCloser(r); ok { 1100 return isKnownInMemoryReader(r) 1101 } 1102 if r, ok := r.(*readTrackingBody); ok { 1103 return isKnownInMemoryReader(r.ReadCloser) 1104 } 1105 return false 1106 } 1107 1108 // bufioFlushWriter is an io.Writer wrapper that flushes all writes 1109 // on its wrapped writer if it's a *bufio.Writer. 1110 type bufioFlushWriter struct{ w io.Writer } 1111 1112 func (fw bufioFlushWriter) Write(p []byte) (n int, err error) { 1113 n, err = fw.w.Write(p) 1114 if bw, ok := fw.w.(*bufio.Writer); n > 0 && ok { 1115 ferr := bw.Flush() 1116 if ferr != nil && err == nil { 1117 err = ferr 1118 } 1119 } 1120 return 1121 }