github.com/ck00004/CobaltStrikeParser-Go@v1.0.14/lib/http/transfer.go (about) 1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package http 6 7 import ( 8 "bufio" 9 "bytes" 10 "errors" 11 "fmt" 12 "io" 13 "net/textproto" 14 "reflect" 15 "sort" 16 "strconv" 17 "strings" 18 "sync" 19 "time" 20 21 "github.com/ck00004/CobaltStrikeParser-Go/lib/http/httptrace" 22 23 "github.com/ck00004/CobaltStrikeParser-Go/lib/http/internal" 24 25 "github.com/ck00004/CobaltStrikeParser-Go/lib/http/internal/ascii" 26 27 "golang.org/x/net/http/httpguts" 28 ) 29 30 // ErrLineTooLong is returned when reading request or response bodies 31 // with malformed chunked encoding. 32 var ErrLineTooLong = internal.ErrLineTooLong 33 34 type errorReader struct { 35 err error 36 } 37 38 func (r errorReader) Read(p []byte) (n int, err error) { 39 return 0, r.err 40 } 41 42 type byteReader struct { 43 b byte 44 done bool 45 } 46 47 func (br *byteReader) Read(p []byte) (n int, err error) { 48 if br.done { 49 return 0, io.EOF 50 } 51 if len(p) == 0 { 52 return 0, nil 53 } 54 br.done = true 55 p[0] = br.b 56 return 1, io.EOF 57 } 58 59 // transferWriter inspects the fields of a user-supplied Request or Response, 60 // sanitizes them without changing the user object and provides methods for 61 // writing the respective header, body and trailer in wire format. 62 type transferWriter struct { 63 Method string 64 Body io.Reader 65 BodyCloser io.Closer 66 ResponseToHEAD bool 67 ContentLength int64 // -1 means unknown, 0 means exactly none 68 Close bool 69 TransferEncoding []string 70 Header Header 71 Trailer Header 72 IsResponse bool 73 bodyReadError error // any non-EOF error from reading Body 74 75 FlushHeaders bool // flush headers to network before body 76 ByteReadCh chan readResult // non-nil if probeRequestBody called 77 } 78 79 func newTransferWriter(r interface{}) (t *transferWriter, err error) { 80 t = &transferWriter{} 81 82 // Extract relevant fields 83 atLeastHTTP11 := false 84 switch rr := r.(type) { 85 case *Request: 86 if rr.ContentLength != 0 && rr.Body == nil { 87 return nil, fmt.Errorf("http: Request.ContentLength=%d with nil Body", rr.ContentLength) 88 } 89 t.Method = valueOrDefault(rr.Method, "GET") 90 t.Close = rr.Close 91 t.TransferEncoding = rr.TransferEncoding 92 t.Header = rr.Header 93 t.Trailer = rr.Trailer 94 t.Body = rr.Body 95 t.BodyCloser = rr.Body 96 t.ContentLength = rr.outgoingLength() 97 if t.ContentLength < 0 && len(t.TransferEncoding) == 0 && t.shouldSendChunkedRequestBody() { 98 t.TransferEncoding = []string{"chunked"} 99 } 100 // If there's a body, conservatively flush the headers 101 // to any bufio.Writer we're writing to, just in case 102 // the server needs the headers early, before we copy 103 // the body and possibly block. We make an exception 104 // for the common standard library in-memory types, 105 // though, to avoid unnecessary TCP packets on the 106 // wire. (Issue 22088.) 107 if t.ContentLength != 0 && !isKnownInMemoryReader(t.Body) { 108 t.FlushHeaders = true 109 } 110 111 atLeastHTTP11 = true // Transport requests are always 1.1 or 2.0 112 case *Response: 113 t.IsResponse = true 114 if rr.Request != nil { 115 t.Method = rr.Request.Method 116 } 117 t.Body = rr.Body 118 t.BodyCloser = rr.Body 119 t.ContentLength = rr.ContentLength 120 t.Close = rr.Close 121 t.TransferEncoding = rr.TransferEncoding 122 t.Header = rr.Header 123 t.Trailer = rr.Trailer 124 atLeastHTTP11 = rr.ProtoAtLeast(1, 1) 125 t.ResponseToHEAD = noResponseBodyExpected(t.Method) 126 } 127 128 // Sanitize Body,ContentLength,TransferEncoding 129 if t.ResponseToHEAD { 130 t.Body = nil 131 if chunked(t.TransferEncoding) { 132 t.ContentLength = -1 133 } 134 } else { 135 if !atLeastHTTP11 || t.Body == nil { 136 t.TransferEncoding = nil 137 } 138 if chunked(t.TransferEncoding) { 139 t.ContentLength = -1 140 } else if t.Body == nil { // no chunking, no body 141 t.ContentLength = 0 142 } 143 } 144 145 // Sanitize Trailer 146 if !chunked(t.TransferEncoding) { 147 t.Trailer = nil 148 } 149 150 return t, nil 151 } 152 153 // shouldSendChunkedRequestBody reports whether we should try to send a 154 // chunked request body to the server. In particular, the case we really 155 // want to prevent is sending a GET or other typically-bodyless request to a 156 // server with a chunked body when the body has zero bytes, since GETs with 157 // bodies (while acceptable according to specs), even zero-byte chunked 158 // bodies, are approximately never seen in the wild and confuse most 159 // servers. See Issue 18257, as one example. 160 // 161 // The only reason we'd send such a request is if the user set the Body to a 162 // non-nil value (say, io.NopCloser(bytes.NewReader(nil))) and didn't 163 // set ContentLength, or NewRequest set it to -1 (unknown), so then we assume 164 // there's bytes to send. 165 // 166 // This code tries to read a byte from the Request.Body in such cases to see 167 // whether the body actually has content (super rare) or is actually just 168 // a non-nil content-less ReadCloser (the more common case). In that more 169 // common case, we act as if their Body were nil instead, and don't send 170 // a body. 171 func (t *transferWriter) shouldSendChunkedRequestBody() bool { 172 // Note that t.ContentLength is the corrected content length 173 // from rr.outgoingLength, so 0 actually means zero, not unknown. 174 if t.ContentLength >= 0 || t.Body == nil { // redundant checks; caller did them 175 return false 176 } 177 if t.Method == "CONNECT" { 178 return false 179 } 180 if requestMethodUsuallyLacksBody(t.Method) { 181 // Only probe the Request.Body for GET/HEAD/DELETE/etc 182 // requests, because it's only those types of requests 183 // that confuse servers. 184 t.probeRequestBody() // adjusts t.Body, t.ContentLength 185 return t.Body != nil 186 } 187 // For all other request types (PUT, POST, PATCH, or anything 188 // made-up we've never heard of), assume it's normal and the server 189 // can deal with a chunked request body. Maybe we'll adjust this 190 // later. 191 return true 192 } 193 194 // probeRequestBody reads a byte from t.Body to see whether it's empty 195 // (returns io.EOF right away). 196 // 197 // But because we've had problems with this blocking users in the past 198 // (issue 17480) when the body is a pipe (perhaps waiting on the response 199 // headers before the pipe is fed data), we need to be careful and bound how 200 // long we wait for it. This delay will only affect users if all the following 201 // are true: 202 // * the request body blocks 203 // * the content length is not set (or set to -1) 204 // * the method doesn't usually have a body (GET, HEAD, DELETE, ...) 205 // * there is no transfer-encoding=chunked already set. 206 // In other words, this delay will not normally affect anybody, and there 207 // are workarounds if it does. 208 func (t *transferWriter) probeRequestBody() { 209 t.ByteReadCh = make(chan readResult, 1) 210 go func(body io.Reader) { 211 var buf [1]byte 212 var rres readResult 213 rres.n, rres.err = body.Read(buf[:]) 214 if rres.n == 1 { 215 rres.b = buf[0] 216 } 217 t.ByteReadCh <- rres 218 }(t.Body) 219 timer := time.NewTimer(200 * time.Millisecond) 220 select { 221 case rres := <-t.ByteReadCh: 222 timer.Stop() 223 if rres.n == 0 && rres.err == io.EOF { 224 // It was empty. 225 t.Body = nil 226 t.ContentLength = 0 227 } else if rres.n == 1 { 228 if rres.err != nil { 229 t.Body = io.MultiReader(&byteReader{b: rres.b}, errorReader{rres.err}) 230 } else { 231 t.Body = io.MultiReader(&byteReader{b: rres.b}, t.Body) 232 } 233 } else if rres.err != nil { 234 t.Body = errorReader{rres.err} 235 } 236 case <-timer.C: 237 // Too slow. Don't wait. Read it later, and keep 238 // assuming that this is ContentLength == -1 239 // (unknown), which means we'll send a 240 // "Transfer-Encoding: chunked" header. 241 t.Body = io.MultiReader(finishAsyncByteRead{t}, t.Body) 242 // Request that Request.Write flush the headers to the 243 // network before writing the body, since our body may not 244 // become readable until it's seen the response headers. 245 t.FlushHeaders = true 246 } 247 } 248 249 func noResponseBodyExpected(requestMethod string) bool { 250 return requestMethod == "HEAD" 251 } 252 253 func (t *transferWriter) shouldSendContentLength() bool { 254 if chunked(t.TransferEncoding) { 255 return false 256 } 257 if t.ContentLength > 0 { 258 return true 259 } 260 if t.ContentLength < 0 { 261 return false 262 } 263 // Many servers expect a Content-Length for these methods 264 if t.Method == "POST" || t.Method == "PUT" || t.Method == "PATCH" { 265 return true 266 } 267 if t.ContentLength == 0 && isIdentity(t.TransferEncoding) { 268 if t.Method == "GET" || t.Method == "HEAD" { 269 return false 270 } 271 return true 272 } 273 274 return false 275 } 276 277 func (t *transferWriter) writeHeader(w io.Writer, trace *httptrace.ClientTrace) error { 278 if t.Close && !hasToken(t.Header.get("Connection"), "close") { 279 if _, err := io.WriteString(w, "Connection: close\r\n"); err != nil { 280 return err 281 } 282 if trace != nil && trace.WroteHeaderField != nil { 283 trace.WroteHeaderField("Connection", []string{"close"}) 284 } 285 } 286 287 // Write Content-Length and/or Transfer-Encoding whose values are a 288 // function of the sanitized field triple (Body, ContentLength, 289 // TransferEncoding) 290 if t.shouldSendContentLength() { 291 if _, err := io.WriteString(w, "Content-Length: "); err != nil { 292 return err 293 } 294 if _, err := io.WriteString(w, strconv.FormatInt(t.ContentLength, 10)+"\r\n"); err != nil { 295 return err 296 } 297 if trace != nil && trace.WroteHeaderField != nil { 298 trace.WroteHeaderField("Content-Length", []string{strconv.FormatInt(t.ContentLength, 10)}) 299 } 300 } else if chunked(t.TransferEncoding) { 301 if _, err := io.WriteString(w, "Transfer-Encoding: chunked\r\n"); err != nil { 302 return err 303 } 304 if trace != nil && trace.WroteHeaderField != nil { 305 trace.WroteHeaderField("Transfer-Encoding", []string{"chunked"}) 306 } 307 } 308 309 // Write Trailer header 310 if t.Trailer != nil { 311 keys := make([]string, 0, len(t.Trailer)) 312 for k := range t.Trailer { 313 k = CanonicalHeaderKey(k) 314 switch k { 315 case "Transfer-Encoding", "Trailer", "Content-Length": 316 return badStringError("invalid Trailer key", k) 317 } 318 keys = append(keys, k) 319 } 320 if len(keys) > 0 { 321 sort.Strings(keys) 322 // TODO: could do better allocation-wise here, but trailers are rare, 323 // so being lazy for now. 324 if _, err := io.WriteString(w, "Trailer: "+strings.Join(keys, ",")+"\r\n"); err != nil { 325 return err 326 } 327 if trace != nil && trace.WroteHeaderField != nil { 328 trace.WroteHeaderField("Trailer", keys) 329 } 330 } 331 } 332 333 return nil 334 } 335 336 // always closes t.BodyCloser 337 func (t *transferWriter) writeBody(w io.Writer) (err error) { 338 var ncopy int64 339 closed := false 340 defer func() { 341 if closed || t.BodyCloser == nil { 342 return 343 } 344 if closeErr := t.BodyCloser.Close(); closeErr != nil && err == nil { 345 err = closeErr 346 } 347 }() 348 349 // Write body. We "unwrap" the body first if it was wrapped in a 350 // nopCloser or readTrackingBody. This is to ensure that we can take advantage of 351 // OS-level optimizations in the event that the body is an 352 // *os.File. 353 if t.Body != nil { 354 var body = t.unwrapBody() 355 if chunked(t.TransferEncoding) { 356 if bw, ok := w.(*bufio.Writer); ok && !t.IsResponse { 357 w = &internal.FlushAfterChunkWriter{Writer: bw} 358 } 359 cw := internal.NewChunkedWriter(w) 360 _, err = t.doBodyCopy(cw, body) 361 if err == nil { 362 err = cw.Close() 363 } 364 } else if t.ContentLength == -1 { 365 dst := w 366 if t.Method == "CONNECT" { 367 dst = bufioFlushWriter{dst} 368 } 369 ncopy, err = t.doBodyCopy(dst, body) 370 } else { 371 ncopy, err = t.doBodyCopy(w, io.LimitReader(body, t.ContentLength)) 372 if err != nil { 373 return err 374 } 375 var nextra int64 376 nextra, err = t.doBodyCopy(io.Discard, body) 377 ncopy += nextra 378 } 379 if err != nil { 380 return err 381 } 382 } 383 if t.BodyCloser != nil { 384 closed = true 385 if err := t.BodyCloser.Close(); err != nil { 386 return err 387 } 388 } 389 390 if !t.ResponseToHEAD && t.ContentLength != -1 && t.ContentLength != ncopy { 391 return fmt.Errorf("http: ContentLength=%d with Body length %d", 392 t.ContentLength, ncopy) 393 } 394 395 if chunked(t.TransferEncoding) { 396 // Write Trailer header 397 if t.Trailer != nil { 398 if err := t.Trailer.Write(w); err != nil { 399 return err 400 } 401 } 402 // Last chunk, empty trailer 403 _, err = io.WriteString(w, "\r\n") 404 } 405 return err 406 } 407 408 // doBodyCopy wraps a copy operation, with any resulting error also 409 // being saved in bodyReadError. 410 // 411 // This function is only intended for use in writeBody. 412 func (t *transferWriter) doBodyCopy(dst io.Writer, src io.Reader) (n int64, err error) { 413 n, err = io.Copy(dst, src) 414 if err != nil && err != io.EOF { 415 t.bodyReadError = err 416 } 417 return 418 } 419 420 // unwrapBodyReader unwraps the body's inner reader if it's a 421 // nopCloser. This is to ensure that body writes sourced from local 422 // files (*os.File types) are properly optimized. 423 // 424 // This function is only intended for use in writeBody. 425 func (t *transferWriter) unwrapBody() io.Reader { 426 if reflect.TypeOf(t.Body) == nopCloserType { 427 return reflect.ValueOf(t.Body).Field(0).Interface().(io.Reader) 428 } 429 if r, ok := t.Body.(*readTrackingBody); ok { 430 r.didRead = true 431 return r.ReadCloser 432 } 433 return t.Body 434 } 435 436 type transferReader struct { 437 // Input 438 Header Header 439 StatusCode int 440 RequestMethod string 441 ProtoMajor int 442 ProtoMinor int 443 // Output 444 Body io.ReadCloser 445 ContentLength int64 446 Chunked bool 447 Close bool 448 Trailer Header 449 } 450 451 func (t *transferReader) protoAtLeast(m, n int) bool { 452 return t.ProtoMajor > m || (t.ProtoMajor == m && t.ProtoMinor >= n) 453 } 454 455 // bodyAllowedForStatus reports whether a given response status code 456 // permits a body. See RFC 7230, section 3.3. 457 func bodyAllowedForStatus(status int) bool { 458 switch { 459 case status >= 100 && status <= 199: 460 return false 461 case status == 204: 462 return false 463 case status == 304: 464 return false 465 } 466 return true 467 } 468 469 var ( 470 suppressedHeaders304 = []string{"Content-Type", "Content-Length", "Transfer-Encoding"} 471 suppressedHeadersNoBody = []string{"Content-Length", "Transfer-Encoding"} 472 ) 473 474 func suppressedHeaders(status int) []string { 475 switch { 476 case status == 304: 477 // RFC 7232 section 4.1 478 return suppressedHeaders304 479 case !bodyAllowedForStatus(status): 480 return suppressedHeadersNoBody 481 } 482 return nil 483 } 484 485 // msg is *Request or *Response. 486 func readTransfer(msg interface{}, r *bufio.Reader) (err error) { 487 t := &transferReader{RequestMethod: "GET"} 488 489 // Unify input 490 isResponse := false 491 switch rr := msg.(type) { 492 case *Response: 493 t.Header = rr.Header 494 t.StatusCode = rr.StatusCode 495 t.ProtoMajor = rr.ProtoMajor 496 t.ProtoMinor = rr.ProtoMinor 497 t.Close = shouldClose(t.ProtoMajor, t.ProtoMinor, t.Header, true) 498 isResponse = true 499 if rr.Request != nil { 500 t.RequestMethod = rr.Request.Method 501 } 502 case *Request: 503 t.Header = rr.Header 504 t.RequestMethod = rr.Method 505 t.ProtoMajor = rr.ProtoMajor 506 t.ProtoMinor = rr.ProtoMinor 507 // Transfer semantics for Requests are exactly like those for 508 // Responses with status code 200, responding to a GET method 509 t.StatusCode = 200 510 t.Close = rr.Close 511 default: 512 panic("unexpected type") 513 } 514 515 // Default to HTTP/1.1 516 if t.ProtoMajor == 0 && t.ProtoMinor == 0 { 517 t.ProtoMajor, t.ProtoMinor = 1, 1 518 } 519 520 // Transfer-Encoding: chunked, and overriding Content-Length. 521 if err := t.parseTransferEncoding(); err != nil { 522 return err 523 } 524 525 realLength, err := fixLength(isResponse, t.StatusCode, t.RequestMethod, t.Header, t.Chunked) 526 if err != nil { 527 return err 528 } 529 if isResponse && t.RequestMethod == "HEAD" { 530 if n, err := parseContentLength(t.Header.get("Content-Length")); err != nil { 531 return err 532 } else { 533 t.ContentLength = n 534 } 535 } else { 536 t.ContentLength = realLength 537 } 538 539 // Trailer 540 t.Trailer, err = fixTrailer(t.Header, t.Chunked) 541 if err != nil { 542 return err 543 } 544 545 // If there is no Content-Length or chunked Transfer-Encoding on a *Response 546 // and the status is not 1xx, 204 or 304, then the body is unbounded. 547 // See RFC 7230, section 3.3. 548 switch msg.(type) { 549 case *Response: 550 if realLength == -1 && !t.Chunked && bodyAllowedForStatus(t.StatusCode) { 551 // Unbounded body. 552 t.Close = true 553 } 554 } 555 556 // Prepare body reader. ContentLength < 0 means chunked encoding 557 // or close connection when finished, since multipart is not supported yet 558 switch { 559 case t.Chunked: 560 if noResponseBodyExpected(t.RequestMethod) || !bodyAllowedForStatus(t.StatusCode) { 561 t.Body = NoBody 562 } else { 563 t.Body = &body{src: internal.NewChunkedReader(r), hdr: msg, r: r, closing: t.Close} 564 } 565 case realLength == 0: 566 t.Body = NoBody 567 case realLength > 0: 568 t.Body = &body{src: io.LimitReader(r, realLength), closing: t.Close} 569 default: 570 // realLength < 0, i.e. "Content-Length" not mentioned in header 571 if t.Close { 572 // Close semantics (i.e. HTTP/1.0) 573 t.Body = &body{src: r, closing: t.Close} 574 } else { 575 // Persistent connection (i.e. HTTP/1.1) 576 t.Body = NoBody 577 } 578 } 579 580 // Unify output 581 switch rr := msg.(type) { 582 case *Request: 583 rr.Body = t.Body 584 rr.ContentLength = t.ContentLength 585 if t.Chunked { 586 rr.TransferEncoding = []string{"chunked"} 587 } 588 rr.Close = t.Close 589 rr.Trailer = t.Trailer 590 case *Response: 591 rr.Body = t.Body 592 rr.ContentLength = t.ContentLength 593 if t.Chunked { 594 rr.TransferEncoding = []string{"chunked"} 595 } 596 rr.Close = t.Close 597 rr.Trailer = t.Trailer 598 } 599 600 return nil 601 } 602 603 // Checks whether chunked is part of the encodings stack 604 func chunked(te []string) bool { return len(te) > 0 && te[0] == "chunked" } 605 606 // Checks whether the encoding is explicitly "identity". 607 func isIdentity(te []string) bool { return len(te) == 1 && te[0] == "identity" } 608 609 // unsupportedTEError reports unsupported transfer-encodings. 610 type unsupportedTEError struct { 611 err string 612 } 613 614 func (uste *unsupportedTEError) Error() string { 615 return uste.err 616 } 617 618 // isUnsupportedTEError checks if the error is of type 619 // unsupportedTEError. It is usually invoked with a non-nil err. 620 func isUnsupportedTEError(err error) bool { 621 _, ok := err.(*unsupportedTEError) 622 return ok 623 } 624 625 // parseTransferEncoding sets t.Chunked based on the Transfer-Encoding header. 626 func (t *transferReader) parseTransferEncoding() error { 627 raw, present := t.Header["Transfer-Encoding"] 628 if !present { 629 return nil 630 } 631 delete(t.Header, "Transfer-Encoding") 632 633 // Issue 12785; ignore Transfer-Encoding on HTTP/1.0 requests. 634 if !t.protoAtLeast(1, 1) { 635 return nil 636 } 637 638 // Like nginx, we only support a single Transfer-Encoding header field, and 639 // only if set to "chunked". This is one of the most security sensitive 640 // surfaces in HTTP/1.1 due to the risk of request smuggling, so we keep it 641 // strict and simple. 642 if len(raw) != 1 { 643 return &unsupportedTEError{fmt.Sprintf("too many transfer encodings: %q", raw)} 644 } 645 if !ascii.EqualFold(textproto.TrimString(raw[0]), "chunked") { 646 return &unsupportedTEError{fmt.Sprintf("unsupported transfer encoding: %q", raw[0])} 647 } 648 649 // RFC 7230 3.3.2 says "A sender MUST NOT send a Content-Length header field 650 // in any message that contains a Transfer-Encoding header field." 651 // 652 // but also: "If a message is received with both a Transfer-Encoding and a 653 // Content-Length header field, the Transfer-Encoding overrides the 654 // Content-Length. Such a message might indicate an attempt to perform 655 // request smuggling (Section 9.5) or response splitting (Section 9.4) and 656 // ought to be handled as an error. A sender MUST remove the received 657 // Content-Length field prior to forwarding such a message downstream." 658 // 659 // Reportedly, these appear in the wild. 660 delete(t.Header, "Content-Length") 661 662 t.Chunked = true 663 return nil 664 } 665 666 // Determine the expected body length, using RFC 7230 Section 3.3. This 667 // function is not a method, because ultimately it should be shared by 668 // ReadResponse and ReadRequest. 669 func fixLength(isResponse bool, status int, requestMethod string, header Header, chunked bool) (int64, error) { 670 isRequest := !isResponse 671 contentLens := header["Content-Length"] 672 673 // Hardening against HTTP request smuggling 674 if len(contentLens) > 1 { 675 // Per RFC 7230 Section 3.3.2, prevent multiple 676 // Content-Length headers if they differ in value. 677 // If there are dups of the value, remove the dups. 678 // See Issue 16490. 679 first := textproto.TrimString(contentLens[0]) 680 for _, ct := range contentLens[1:] { 681 if first != textproto.TrimString(ct) { 682 return 0, fmt.Errorf("http: message cannot contain multiple Content-Length headers; got %q", contentLens) 683 } 684 } 685 686 // deduplicate Content-Length 687 header.Del("Content-Length") 688 header.Add("Content-Length", first) 689 690 contentLens = header["Content-Length"] 691 } 692 693 // Logic based on response type or status 694 if noResponseBodyExpected(requestMethod) { 695 // For HTTP requests, as part of hardening against request 696 // smuggling (RFC 7230), don't allow a Content-Length header for 697 // methods which don't permit bodies. As an exception, allow 698 // exactly one Content-Length header if its value is "0". 699 if isRequest && len(contentLens) > 0 && !(len(contentLens) == 1 && contentLens[0] == "0") { 700 return 0, fmt.Errorf("http: method cannot contain a Content-Length; got %q", contentLens) 701 } 702 return 0, nil 703 } 704 if status/100 == 1 { 705 return 0, nil 706 } 707 switch status { 708 case 204, 304: 709 return 0, nil 710 } 711 712 // Logic based on Transfer-Encoding 713 if chunked { 714 return -1, nil 715 } 716 717 // Logic based on Content-Length 718 var cl string 719 if len(contentLens) == 1 { 720 cl = textproto.TrimString(contentLens[0]) 721 } 722 if cl != "" { 723 n, err := parseContentLength(cl) 724 if err != nil { 725 return -1, err 726 } 727 return n, nil 728 } 729 header.Del("Content-Length") 730 731 if isRequest { 732 // RFC 7230 neither explicitly permits nor forbids an 733 // entity-body on a GET request so we permit one if 734 // declared, but we default to 0 here (not -1 below) 735 // if there's no mention of a body. 736 // Likewise, all other request methods are assumed to have 737 // no body if neither Transfer-Encoding chunked nor a 738 // Content-Length are set. 739 return 0, nil 740 } 741 742 // Body-EOF logic based on other methods (like closing, or chunked coding) 743 return -1, nil 744 } 745 746 // Determine whether to hang up after sending a request and body, or 747 // receiving a response and body 748 // 'header' is the request headers 749 func shouldClose(major, minor int, header Header, removeCloseHeader bool) bool { 750 if major < 1 { 751 return true 752 } 753 754 conv := header["Connection"] 755 hasClose := httpguts.HeaderValuesContainsToken(conv, "close") 756 if major == 1 && minor == 0 { 757 return hasClose || !httpguts.HeaderValuesContainsToken(conv, "keep-alive") 758 } 759 760 if hasClose && removeCloseHeader { 761 header.Del("Connection") 762 } 763 764 return hasClose 765 } 766 767 // Parse the trailer header 768 func fixTrailer(header Header, chunked bool) (Header, error) { 769 vv, ok := header["Trailer"] 770 if !ok { 771 return nil, nil 772 } 773 if !chunked { 774 // Trailer and no chunking: 775 // this is an invalid use case for trailer header. 776 // Nevertheless, no error will be returned and we 777 // let users decide if this is a valid HTTP message. 778 // The Trailer header will be kept in Response.Header 779 // but not populate Response.Trailer. 780 // See issue #27197. 781 return nil, nil 782 } 783 header.Del("Trailer") 784 785 trailer := make(Header) 786 var err error 787 for _, v := range vv { 788 foreachHeaderElement(v, func(key string) { 789 key = CanonicalHeaderKey(key) 790 switch key { 791 case "Transfer-Encoding", "Trailer", "Content-Length": 792 if err == nil { 793 err = badStringError("bad trailer key", key) 794 return 795 } 796 } 797 trailer[key] = nil 798 }) 799 } 800 if err != nil { 801 return nil, err 802 } 803 if len(trailer) == 0 { 804 return nil, nil 805 } 806 return trailer, nil 807 } 808 809 // body turns a Reader into a ReadCloser. 810 // Close ensures that the body has been fully read 811 // and then reads the trailer if necessary. 812 type body struct { 813 src io.Reader 814 hdr interface{} // non-nil (Response or Request) value means read trailer 815 r *bufio.Reader // underlying wire-format reader for the trailer 816 closing bool // is the connection to be closed after reading body? 817 doEarlyClose bool // whether Close should stop early 818 819 mu sync.Mutex // guards following, and calls to Read and Close 820 sawEOF bool 821 closed bool 822 earlyClose bool // Close called and we didn't read to the end of src 823 onHitEOF func() // if non-nil, func to call when EOF is Read 824 } 825 826 // ErrBodyReadAfterClose is returned when reading a Request or Response 827 // Body after the body has been closed. This typically happens when the body is 828 // read after an HTTP Handler calls WriteHeader or Write on its 829 // ResponseWriter. 830 var ErrBodyReadAfterClose = errors.New("http: invalid Read on closed Body") 831 832 func (b *body) Read(p []byte) (n int, err error) { 833 b.mu.Lock() 834 defer b.mu.Unlock() 835 if b.closed { 836 return 0, ErrBodyReadAfterClose 837 } 838 return b.readLocked(p) 839 } 840 841 // Must hold b.mu. 842 func (b *body) readLocked(p []byte) (n int, err error) { 843 if b.sawEOF { 844 return 0, io.EOF 845 } 846 n, err = b.src.Read(p) 847 848 if err == io.EOF { 849 b.sawEOF = true 850 // Chunked case. Read the trailer. 851 if b.hdr != nil { 852 if e := b.readTrailer(); e != nil { 853 err = e 854 // Something went wrong in the trailer, we must not allow any 855 // further reads of any kind to succeed from body, nor any 856 // subsequent requests on the server connection. See 857 // golang.org/issue/12027 858 b.sawEOF = false 859 b.closed = true 860 } 861 b.hdr = nil 862 } else { 863 // If the server declared the Content-Length, our body is a LimitedReader 864 // and we need to check whether this EOF arrived early. 865 if lr, ok := b.src.(*io.LimitedReader); ok && lr.N > 0 { 866 err = io.ErrUnexpectedEOF 867 } 868 } 869 } 870 871 // If we can return an EOF here along with the read data, do 872 // so. This is optional per the io.Reader contract, but doing 873 // so helps the HTTP transport code recycle its connection 874 // earlier (since it will see this EOF itself), even if the 875 // client doesn't do future reads or Close. 876 if err == nil && n > 0 { 877 if lr, ok := b.src.(*io.LimitedReader); ok && lr.N == 0 { 878 err = io.EOF 879 b.sawEOF = true 880 } 881 } 882 883 if b.sawEOF && b.onHitEOF != nil { 884 b.onHitEOF() 885 } 886 887 return n, err 888 } 889 890 var ( 891 singleCRLF = []byte("\r\n") 892 doubleCRLF = []byte("\r\n\r\n") 893 ) 894 895 func seeUpcomingDoubleCRLF(r *bufio.Reader) bool { 896 for peekSize := 4; ; peekSize++ { 897 // This loop stops when Peek returns an error, 898 // which it does when r's buffer has been filled. 899 buf, err := r.Peek(peekSize) 900 if bytes.HasSuffix(buf, doubleCRLF) { 901 return true 902 } 903 if err != nil { 904 break 905 } 906 } 907 return false 908 } 909 910 var errTrailerEOF = errors.New("http: unexpected EOF reading trailer") 911 912 func (b *body) readTrailer() error { 913 // The common case, since nobody uses trailers. 914 buf, err := b.r.Peek(2) 915 if bytes.Equal(buf, singleCRLF) { 916 b.r.Discard(2) 917 return nil 918 } 919 if len(buf) < 2 { 920 return errTrailerEOF 921 } 922 if err != nil { 923 return err 924 } 925 926 // Make sure there's a header terminator coming up, to prevent 927 // a DoS with an unbounded size Trailer. It's not easy to 928 // slip in a LimitReader here, as textproto.NewReader requires 929 // a concrete *bufio.Reader. Also, we can't get all the way 930 // back up to our conn's LimitedReader that *might* be backing 931 // this bufio.Reader. Instead, a hack: we iteratively Peek up 932 // to the bufio.Reader's max size, looking for a double CRLF. 933 // This limits the trailer to the underlying buffer size, typically 4kB. 934 if !seeUpcomingDoubleCRLF(b.r) { 935 return errors.New("http: suspiciously long trailer after chunked body") 936 } 937 938 hdr, err := textproto.NewReader(b.r).ReadMIMEHeader() 939 if err != nil { 940 if err == io.EOF { 941 return errTrailerEOF 942 } 943 return err 944 } 945 switch rr := b.hdr.(type) { 946 case *Request: 947 mergeSetHeader(&rr.Trailer, Header(hdr)) 948 case *Response: 949 mergeSetHeader(&rr.Trailer, Header(hdr)) 950 } 951 return nil 952 } 953 954 func mergeSetHeader(dst *Header, src Header) { 955 if *dst == nil { 956 *dst = src 957 return 958 } 959 for k, vv := range src { 960 (*dst)[k] = vv 961 } 962 } 963 964 // unreadDataSizeLocked returns the number of bytes of unread input. 965 // It returns -1 if unknown. 966 // b.mu must be held. 967 func (b *body) unreadDataSizeLocked() int64 { 968 if lr, ok := b.src.(*io.LimitedReader); ok { 969 return lr.N 970 } 971 return -1 972 } 973 974 func (b *body) Close() error { 975 b.mu.Lock() 976 defer b.mu.Unlock() 977 if b.closed { 978 return nil 979 } 980 var err error 981 switch { 982 case b.sawEOF: 983 // Already saw EOF, so no need going to look for it. 984 case b.hdr == nil && b.closing: 985 // no trailer and closing the connection next. 986 // no point in reading to EOF. 987 case b.doEarlyClose: 988 // Read up to maxPostHandlerReadBytes bytes of the body, looking 989 // for EOF (and trailers), so we can re-use this connection. 990 if lr, ok := b.src.(*io.LimitedReader); ok && lr.N > maxPostHandlerReadBytes { 991 // There was a declared Content-Length, and we have more bytes remaining 992 // than our maxPostHandlerReadBytes tolerance. So, give up. 993 b.earlyClose = true 994 } else { 995 var n int64 996 // Consume the body, or, which will also lead to us reading 997 // the trailer headers after the body, if present. 998 n, err = io.CopyN(io.Discard, bodyLocked{b}, maxPostHandlerReadBytes) 999 if err == io.EOF { 1000 err = nil 1001 } 1002 if n == maxPostHandlerReadBytes { 1003 b.earlyClose = true 1004 } 1005 } 1006 default: 1007 // Fully consume the body, which will also lead to us reading 1008 // the trailer headers after the body, if present. 1009 _, err = io.Copy(io.Discard, bodyLocked{b}) 1010 } 1011 b.closed = true 1012 return err 1013 } 1014 1015 func (b *body) didEarlyClose() bool { 1016 b.mu.Lock() 1017 defer b.mu.Unlock() 1018 return b.earlyClose 1019 } 1020 1021 // bodyRemains reports whether future Read calls might 1022 // yield data. 1023 func (b *body) bodyRemains() bool { 1024 b.mu.Lock() 1025 defer b.mu.Unlock() 1026 return !b.sawEOF 1027 } 1028 1029 func (b *body) registerOnHitEOF(fn func()) { 1030 b.mu.Lock() 1031 defer b.mu.Unlock() 1032 b.onHitEOF = fn 1033 } 1034 1035 // bodyLocked is a io.Reader reading from a *body when its mutex is 1036 // already held. 1037 type bodyLocked struct { 1038 b *body 1039 } 1040 1041 func (bl bodyLocked) Read(p []byte) (n int, err error) { 1042 if bl.b.closed { 1043 return 0, ErrBodyReadAfterClose 1044 } 1045 return bl.b.readLocked(p) 1046 } 1047 1048 // parseContentLength trims whitespace from s and returns -1 if no value 1049 // is set, or the value if it's >= 0. 1050 func parseContentLength(cl string) (int64, error) { 1051 cl = textproto.TrimString(cl) 1052 if cl == "" { 1053 return -1, nil 1054 } 1055 n, err := strconv.ParseUint(cl, 10, 63) 1056 if err != nil { 1057 return 0, badStringError("bad Content-Length", cl) 1058 } 1059 return int64(n), nil 1060 1061 } 1062 1063 // finishAsyncByteRead finishes reading the 1-byte sniff 1064 // from the ContentLength==0, Body!=nil case. 1065 type finishAsyncByteRead struct { 1066 tw *transferWriter 1067 } 1068 1069 func (fr finishAsyncByteRead) Read(p []byte) (n int, err error) { 1070 if len(p) == 0 { 1071 return 1072 } 1073 rres := <-fr.tw.ByteReadCh 1074 n, err = rres.n, rres.err 1075 if n == 1 { 1076 p[0] = rres.b 1077 } 1078 return 1079 } 1080 1081 var nopCloserType = reflect.TypeOf(io.NopCloser(nil)) 1082 1083 // isKnownInMemoryReader reports whether r is a type known to not 1084 // block on Read. Its caller uses this as an optional optimization to 1085 // send fewer TCP packets. 1086 func isKnownInMemoryReader(r io.Reader) bool { 1087 switch r.(type) { 1088 case *bytes.Reader, *bytes.Buffer, *strings.Reader: 1089 return true 1090 } 1091 if reflect.TypeOf(r) == nopCloserType { 1092 return isKnownInMemoryReader(reflect.ValueOf(r).Field(0).Interface().(io.Reader)) 1093 } 1094 if r, ok := r.(*readTrackingBody); ok { 1095 return isKnownInMemoryReader(r.ReadCloser) 1096 } 1097 return false 1098 } 1099 1100 // bufioFlushWriter is an io.Writer wrapper that flushes all writes 1101 // on its wrapped writer if it's a *bufio.Writer. 1102 type bufioFlushWriter struct{ w io.Writer } 1103 1104 func (fw bufioFlushWriter) Write(p []byte) (n int, err error) { 1105 n, err = fw.w.Write(p) 1106 if bw, ok := fw.w.(*bufio.Writer); n > 0 && ok { 1107 ferr := bw.Flush() 1108 if ferr != nil && err == nil { 1109 err = ferr 1110 } 1111 } 1112 return 1113 }