github.com/AESNooper/go/src@v0.0.0-20220218095104-b56a4ab1bbbb/net/http/internal/chunked.go (about) 1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // The wire protocol for HTTP's "chunked" Transfer-Encoding. 6 7 // Package internal contains HTTP internals shared by net/http and 8 // net/http/httputil. 9 package internal 10 11 import ( 12 "bufio" 13 "bytes" 14 "errors" 15 "fmt" 16 "io" 17 ) 18 19 const maxLineLength = 4096 // assumed <= bufio.defaultBufSize 20 21 var ErrLineTooLong = errors.New("header line too long") 22 23 // NewChunkedReader returns a new chunkedReader that translates the data read from r 24 // out of HTTP "chunked" format before returning it. 25 // The chunkedReader returns io.EOF when the final 0-length chunk is read. 26 // 27 // NewChunkedReader is not needed by normal applications. The http package 28 // automatically decodes chunking when reading response bodies. 29 func NewChunkedReader(r io.Reader) io.Reader { 30 br, ok := r.(*bufio.Reader) 31 if !ok { 32 br = bufio.NewReader(r) 33 } 34 return &chunkedReader{r: br} 35 } 36 37 type chunkedReader struct { 38 r *bufio.Reader 39 n uint64 // unread bytes in chunk 40 err error 41 buf [2]byte 42 checkEnd bool // whether need to check for \r\n chunk footer 43 } 44 45 func (cr *chunkedReader) beginChunk() { 46 // chunk-size CRLF 47 var line []byte 48 line, cr.err = readChunkLine(cr.r) 49 if cr.err != nil { 50 return 51 } 52 cr.n, cr.err = parseHexUint(line) 53 if cr.err != nil { 54 return 55 } 56 if cr.n == 0 { 57 cr.err = io.EOF 58 } 59 } 60 61 func (cr *chunkedReader) chunkHeaderAvailable() bool { 62 n := cr.r.Buffered() 63 if n > 0 { 64 peek, _ := cr.r.Peek(n) 65 return bytes.IndexByte(peek, '\n') >= 0 66 } 67 return false 68 } 69 70 func (cr *chunkedReader) Read(b []uint8) (n int, err error) { 71 for cr.err == nil { 72 if cr.checkEnd { 73 if n > 0 && cr.r.Buffered() < 2 { 74 // We have some data. Return early (per the io.Reader 75 // contract) instead of potentially blocking while 76 // reading more. 77 break 78 } 79 if _, cr.err = io.ReadFull(cr.r, cr.buf[:2]); cr.err == nil { 80 if string(cr.buf[:]) != "\r\n" { 81 cr.err = errors.New("malformed chunked encoding") 82 break 83 } 84 } else { 85 if cr.err == io.EOF { 86 cr.err = io.ErrUnexpectedEOF 87 } 88 break 89 } 90 cr.checkEnd = false 91 } 92 if cr.n == 0 { 93 if n > 0 && !cr.chunkHeaderAvailable() { 94 // We've read enough. Don't potentially block 95 // reading a new chunk header. 96 break 97 } 98 cr.beginChunk() 99 continue 100 } 101 if len(b) == 0 { 102 break 103 } 104 rbuf := b 105 if uint64(len(rbuf)) > cr.n { 106 rbuf = rbuf[:cr.n] 107 } 108 var n0 int 109 n0, cr.err = cr.r.Read(rbuf) 110 n += n0 111 b = b[n0:] 112 cr.n -= uint64(n0) 113 // If we're at the end of a chunk, read the next two 114 // bytes to verify they are "\r\n". 115 if cr.n == 0 && cr.err == nil { 116 cr.checkEnd = true 117 } else if cr.err == io.EOF { 118 cr.err = io.ErrUnexpectedEOF 119 } 120 } 121 return n, cr.err 122 } 123 124 // Read a line of bytes (up to \n) from b. 125 // Give up if the line exceeds maxLineLength. 126 // The returned bytes are owned by the bufio.Reader 127 // so they are only valid until the next bufio read. 128 func readChunkLine(b *bufio.Reader) ([]byte, error) { 129 p, err := b.ReadSlice('\n') 130 if err != nil { 131 // We always know when EOF is coming. 132 // If the caller asked for a line, there should be a line. 133 if err == io.EOF { 134 err = io.ErrUnexpectedEOF 135 } else if err == bufio.ErrBufferFull { 136 err = ErrLineTooLong 137 } 138 return nil, err 139 } 140 if len(p) >= maxLineLength { 141 return nil, ErrLineTooLong 142 } 143 p = trimTrailingWhitespace(p) 144 p, err = removeChunkExtension(p) 145 if err != nil { 146 return nil, err 147 } 148 return p, nil 149 } 150 151 func trimTrailingWhitespace(b []byte) []byte { 152 for len(b) > 0 && isASCIISpace(b[len(b)-1]) { 153 b = b[:len(b)-1] 154 } 155 return b 156 } 157 158 func isASCIISpace(b byte) bool { 159 return b == ' ' || b == '\t' || b == '\n' || b == '\r' 160 } 161 162 var semi = []byte(";") 163 164 // removeChunkExtension removes any chunk-extension from p. 165 // For example, 166 // "0" => "0" 167 // "0;token" => "0" 168 // "0;token=val" => "0" 169 // `0;token="quoted string"` => "0" 170 func removeChunkExtension(p []byte) ([]byte, error) { 171 p, _, _ = bytes.Cut(p, semi) 172 // TODO: care about exact syntax of chunk extensions? We're 173 // ignoring and stripping them anyway. For now just never 174 // return an error. 175 return p, nil 176 } 177 178 // NewChunkedWriter returns a new chunkedWriter that translates writes into HTTP 179 // "chunked" format before writing them to w. Closing the returned chunkedWriter 180 // sends the final 0-length chunk that marks the end of the stream but does 181 // not send the final CRLF that appears after trailers; trailers and the last 182 // CRLF must be written separately. 183 // 184 // NewChunkedWriter is not needed by normal applications. The http 185 // package adds chunking automatically if handlers don't set a 186 // Content-Length header. Using newChunkedWriter inside a handler 187 // would result in double chunking or chunking with a Content-Length 188 // length, both of which are wrong. 189 func NewChunkedWriter(w io.Writer) io.WriteCloser { 190 return &chunkedWriter{w} 191 } 192 193 // Writing to chunkedWriter translates to writing in HTTP chunked Transfer 194 // Encoding wire format to the underlying Wire chunkedWriter. 195 type chunkedWriter struct { 196 Wire io.Writer 197 } 198 199 // Write the contents of data as one chunk to Wire. 200 // NOTE: Note that the corresponding chunk-writing procedure in Conn.Write has 201 // a bug since it does not check for success of io.WriteString 202 func (cw *chunkedWriter) Write(data []byte) (n int, err error) { 203 204 // Don't send 0-length data. It looks like EOF for chunked encoding. 205 if len(data) == 0 { 206 return 0, nil 207 } 208 209 if _, err = fmt.Fprintf(cw.Wire, "%x\r\n", len(data)); err != nil { 210 return 0, err 211 } 212 if n, err = cw.Wire.Write(data); err != nil { 213 return 214 } 215 if n != len(data) { 216 err = io.ErrShortWrite 217 return 218 } 219 if _, err = io.WriteString(cw.Wire, "\r\n"); err != nil { 220 return 221 } 222 if bw, ok := cw.Wire.(*FlushAfterChunkWriter); ok { 223 err = bw.Flush() 224 } 225 return 226 } 227 228 func (cw *chunkedWriter) Close() error { 229 _, err := io.WriteString(cw.Wire, "0\r\n") 230 return err 231 } 232 233 // FlushAfterChunkWriter signals from the caller of NewChunkedWriter 234 // that each chunk should be followed by a flush. It is used by the 235 // http.Transport code to keep the buffering behavior for headers and 236 // trailers, but flush out chunks aggressively in the middle for 237 // request bodies which may be generated slowly. See Issue 6574. 238 type FlushAfterChunkWriter struct { 239 *bufio.Writer 240 } 241 242 func parseHexUint(v []byte) (n uint64, err error) { 243 for i, b := range v { 244 switch { 245 case '0' <= b && b <= '9': 246 b = b - '0' 247 case 'a' <= b && b <= 'f': 248 b = b - 'a' + 10 249 case 'A' <= b && b <= 'F': 250 b = b - 'A' + 10 251 default: 252 return 0, errors.New("invalid byte in chunk length") 253 } 254 if i == 16 { 255 return 0, errors.New("http chunk length too large") 256 } 257 n <<= 4 258 n |= uint64(b) 259 } 260 return 261 }