github.com/google/martian/v3@v3.3.3/h2/relay.go (about) 1 // Copyright 2021 Google Inc. All rights reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package h2 16 17 import ( 18 "bytes" 19 "container/list" 20 "errors" 21 "fmt" 22 "io" 23 "math" 24 "sync" 25 "sync/atomic" 26 27 "github.com/google/martian/v3/log" 28 "golang.org/x/net/http2" 29 "golang.org/x/net/http2/hpack" 30 ) 31 32 const ( 33 // See: https://httpwg.org/specs/rfc7540.html#SettingValues 34 initialMaxFrameSize = 16384 35 initialMaxHeaderTableSize = 4096 36 37 // See: https://tools.ietf.org/html/rfc7540#section-6.9.2 38 defaultInitialWindowSize = 65535 39 40 // headersPriorityMetadataLength is the length of the priority metadata that optionally occurs at 41 // the beginning of the payload of the header frame. 42 // 43 // See: https://tools.ietf.org/html/rfc7540#section-6.2 44 headersPriorityMetadataLength = 5 45 46 // pushPromiseMetadataLength is the length of the metadata that is part of the payload of the 47 // pushPromise frame. This does not include the padding length octet, which isn't needed due to 48 // the relaxed security constraints of a development proxy. 49 // 50 // See: https://tools.ietf.org/html/rfc7540#section-6.6 51 pushPromiseMetadataLength = 4 52 53 // outputChannelSize is the size of the output channel. Roughly, it should be large enough to 54 // allow a window's worth of frames to minimize synchronization overhead. 55 outputChannelSize = 15 56 ) 57 58 // relay encapsulates a flow of h2 traffic in one direction. 59 type relay struct { 60 dir Direction 61 62 // srcLabel and destLabel are used only to create debugging messages. 63 srcLabel, destLabel string 64 65 src *http2.Framer 66 67 // destMu guards writes to dest, which may occur on from either the `relayFrames` thread of 68 // this relay or `peer`. `peer` writes WINDOW_UPDATE frames to this relay when it receives 69 // DATA frames. 70 destMu sync.Mutex 71 dest *http2.Framer 72 73 // maxFrameSize is set by the peer relay and is accessed atomically. 74 maxFrameSize uint32 75 76 // The decoder and encoder settings can be adjusted by the peer connection so access to these 77 // fields must be guarded. 78 decoderMu sync.Mutex 79 decoder *hpack.Decoder 80 81 encoderMu sync.Mutex 82 encoder *hpack.Encoder 83 reencoded bytes.Buffer // handle to the output buffer of `encoder` 84 85 // headerBuffer collects header fragments that are received across multiple frames, i.e., 86 // when there are continuation frames. 87 headerBuffer bytes.Buffer 88 continuationState continuationState 89 90 // flowMu guards access to flow-control related fields. 91 flowMu sync.Mutex 92 initialWindowSize uint32 93 connectionWindowSize int // "global" connection-level window size 94 // outputBuffers is output pending available window size per-stream 95 outputBuffers map[uint32]*outputBuffer 96 // output stores stream output that is ready to be sent over HTTP/2. It provides a way to 97 // guarantee frame order without blocking on each frame being sent. 98 output chan queuedFrame 99 100 enableDebugLogs *bool 101 102 // The following fields depend on a circular dependency between the relays in opposite directions 103 // so must be set explicitly after initialization. 104 105 // processors stores per HTTP/2 stream processors. 106 processors *streamProcessors 107 108 peer *relay // relay for traffic from the peer 109 } 110 111 // newRelay initializes a relay for the given direction. This performs only partial initialization 112 // due to circular dependency. 113 func newRelay( 114 dir Direction, 115 srcLabel, destLabel string, 116 src, dest *http2.Framer, 117 enableDebugLogs *bool, 118 ) *relay { 119 ret := &relay{ 120 dir: dir, 121 srcLabel: srcLabel, 122 destLabel: destLabel, 123 src: src, 124 dest: dest, 125 maxFrameSize: initialMaxFrameSize, 126 decoder: hpack.NewDecoder(initialMaxHeaderTableSize, nil), 127 initialWindowSize: defaultInitialWindowSize, 128 connectionWindowSize: defaultInitialWindowSize, 129 outputBuffers: make(map[uint32]*outputBuffer), 130 output: make(chan queuedFrame, outputChannelSize), 131 enableDebugLogs: enableDebugLogs, 132 } 133 ret.encoder = hpack.NewEncoder(&ret.reencoded) 134 135 // These limits seem to be part of the Go implementation of hpack. They exist because in a 136 // production system, there must be limits on the resources requested by clients. However, this 137 // is irrevelevant in a development proxy context. 138 ret.decoder.SetAllowedMaxDynamicTableSize(math.MaxUint32) 139 ret.encoder.SetMaxDynamicTableSizeLimit(math.MaxUint32) 140 return ret 141 } 142 143 // relayFrames reads frames from `f.src` to `f.dest` until an error occurs or the connection closes. 144 func (r *relay) relayFrames(closing chan bool) error { 145 // Shutting down producer-consumers linked by channels is subtle. In this function, the writer 146 // goroutine consumes frames from `r.output`, which are populated by the reader goroutine. If 147 // the writer shuts down before the reader, the reader may deadlock on inserting frames into 148 // `r.output`. The writer therefore has to keep processing until the reader is done. This is 149 // coordinated via `readerDone`. 150 // 151 // A second subtlely is that errors on the writer goroutine should stop the reader goroutine. 152 // This is communicated via `writeErr`. To avoid deadlocks, even after the error occurs, the 153 // writer thread must still wait until `readerDone` has been communicated to stop processing. 154 155 // Communicates to the consuming writer goroutine that the reader (the calling goroutine of this 156 // method) is done. 157 readerDone := make(chan struct{}) 158 defer func() { readerDone <- struct{}{} }() 159 160 // Communicates errors occuring on the writer goroutine to the reader goroutine. 161 writerErr := make(chan error, 1) 162 163 // This writer goroutine consumes the strictly ordered frames in `r.output` and delivers them. 164 go func() { 165 var err error 166 for { 167 select { 168 case f := <-r.output: 169 if err == nil { 170 r.destMu.Lock() 171 err = f.send(r.dest) 172 r.destMu.Unlock() 173 if err != nil { 174 writerErr <- err 175 } 176 } 177 // Once an output error has occurred, the remaining frames are drained from the channel 178 // without sending them. 179 case <-readerDone: 180 return 181 } 182 } 183 }() 184 185 // This channel is buffered to allow the ReadFrame goroutine to drain on closing. 186 frameReady := make(chan struct{}, 1) 187 for { 188 var frame http2.Frame 189 var err error 190 go func() { 191 // ReadFrame is called in its own goroutine to make this function responsive to closing. It 192 // does not need to block here to close. 193 frame, err = r.src.ReadFrame() 194 frameReady <- struct{}{} 195 }() 196 select { 197 case <-frameReady: 198 if err != nil { 199 if err == io.EOF { 200 return nil 201 } 202 return fmt.Errorf("reading frame: %w", err) 203 } 204 if err := r.processFrame(frame); err != nil { 205 return fmt.Errorf("processing frame: %w", err) 206 } 207 if *r.enableDebugLogs { 208 log.Infof("%s--%v-->%s", r.srcLabel, frame, r.destLabel) 209 } 210 case err := <-writerErr: 211 return fmt.Errorf("sending frame: %w", err) 212 case <-closing: 213 // The ReadFrame goroutine is abandoned at this point. It completes as soon as the blocking 214 // ReadFrame call completes, but could potentially leak for an unspecified duration. 215 return nil 216 } 217 } 218 } 219 220 func (r *relay) processFrame(f http2.Frame) error { 221 var err error 222 switch f := f.(type) { 223 case *http2.DataFrame: 224 // The proxy's window increments as soon as it receives data. This assumes that the proxy has 225 // ample resources because it is inteded for testing and development. 226 if err = r.peer.sendWindowUpdates(f); err == nil { 227 err = r.processor(f.StreamID).Data(f.Data(), f.StreamEnded()) 228 } 229 case *http2.HeadersFrame: 230 if !f.HeadersEnded() { 231 r.headerBuffer.Reset() 232 r.headerBuffer.Write(f.HeaderBlockFragment()) 233 r.continuationState = &headerContinuation{f.Priority} 234 } else { 235 var headers []hpack.HeaderField 236 headers, err = r.decodeFull(f.HeaderBlockFragment()) 237 if err != nil { 238 return fmt.Errorf("decoding header %v: %w", f, err) 239 } 240 err = r.processor(f.StreamID).Header(headers, f.StreamEnded(), f.Priority) 241 } 242 case *http2.PriorityFrame: 243 err = r.processor(f.StreamID).Priority(f.PriorityParam) 244 case *http2.RSTStreamFrame: 245 err = r.processor(f.StreamID).RSTStream(f.ErrCode) 246 case *http2.SettingsFrame: 247 if f.IsAck() { 248 r.destMu.Lock() 249 err = r.dest.WriteSettingsAck() 250 r.destMu.Unlock() 251 } else { 252 var settings []http2.Setting 253 if err = f.ForeachSetting(func(s http2.Setting) error { 254 switch s.ID { 255 case http2.SettingHeaderTableSize: 256 r.peer.updateTableSize(s.Val) 257 case http2.SettingInitialWindowSize: 258 r.peer.updateInitialWindowSize(s.Val) 259 case http2.SettingMaxFrameSize: 260 r.peer.updateMaxFrameSize(s.Val) 261 } 262 settings = append(settings, s) 263 return nil 264 }); err == nil { 265 r.destMu.Lock() 266 err = r.dest.WriteSettings(settings...) 267 r.destMu.Unlock() 268 } 269 } 270 case *http2.PushPromiseFrame: 271 if !f.HeadersEnded() { 272 r.headerBuffer.Reset() 273 r.headerBuffer.Write(f.HeaderBlockFragment()) 274 r.continuationState = &pushPromiseContinuation{f.PromiseID} 275 } else { 276 var headers []hpack.HeaderField 277 headers, err = r.decodeFull(f.HeaderBlockFragment()) 278 if err != nil { 279 return fmt.Errorf("decoding push promise %v: %w", f, err) 280 } 281 err = r.processor(f.StreamID).PushPromise(f.PromiseID, headers) 282 } 283 case *http2.PingFrame: 284 r.destMu.Lock() 285 err = r.dest.WritePing(f.IsAck(), f.Data) 286 r.destMu.Unlock() 287 case *http2.GoAwayFrame: 288 r.destMu.Lock() 289 err = r.dest.WriteGoAway(f.LastStreamID, f.ErrCode, f.DebugData()) 290 r.destMu.Unlock() 291 case *http2.WindowUpdateFrame: 292 r.peer.updateWindow(f) 293 case *http2.ContinuationFrame: 294 r.headerBuffer.Write(f.HeaderBlockFragment()) 295 if f.HeadersEnded() { 296 var headers []hpack.HeaderField 297 headers, err = r.decodeFull(r.headerBuffer.Bytes()) 298 if err != nil { 299 return fmt.Errorf("decoding headers for continuation %v: %w", f, err) 300 } 301 err = r.continuationState.complete(r.processor(f.StreamID), headers) 302 } 303 default: 304 err = errors.New("unrecognized frame type") 305 } 306 return err 307 } 308 309 func (r *relay) processor(id uint32) Processor { 310 return r.processors.Get(id, r.dir) 311 } 312 313 func (r *relay) updateTableSize(v uint32) { 314 r.decoderMu.Lock() 315 r.decoder.SetMaxDynamicTableSize(v) 316 r.decoderMu.Unlock() 317 318 r.encoderMu.Lock() 319 r.encoder.SetMaxDynamicTableSize(v) 320 r.encoderMu.Unlock() 321 } 322 323 func (r *relay) updateMaxFrameSize(v uint32) { 324 atomic.StoreUint32(&r.maxFrameSize, v) 325 } 326 327 // updateInitialWindowSize updates the initial window size and updates all stream windows based on 328 // the difference. Note that this should not include the connection window. 329 // See: https://tools.ietf.org/html/rfc7540#section-6.9.2 330 // 331 // This is called by `peer`, so requires a thread-safe implementation. 332 func (r *relay) updateInitialWindowSize(v uint32) { 333 r.flowMu.Lock() 334 delta := int(v) - int(r.initialWindowSize) 335 r.initialWindowSize = v 336 for _, w := range r.outputBuffers { 337 w.windowSize += delta 338 } 339 r.flowMu.Unlock() 340 // Since all the stream windows may be impacted, all the queues need to be checked for newly 341 // eligible frames. 342 r.sendQueuedFramesUnderWindowSize() 343 } 344 345 // updateWindow updates the specified window size and may result in the sending of data frames. 346 func (r *relay) updateWindow(f *http2.WindowUpdateFrame) { 347 if f.StreamID == 0 { 348 // A stream ID of 0 means updating the global connection window size. This may cause any 349 // queued frame belonging to any stream to become eligible for sending. 350 r.flowMu.Lock() 351 r.connectionWindowSize += int(f.Increment) 352 r.flowMu.Unlock() 353 r.sendQueuedFramesUnderWindowSize() 354 } 355 356 r.flowMu.Lock() 357 w := r.outputBuffer(f.StreamID) 358 w.windowSize += int(f.Increment) 359 w.emitEligibleFrames(r.output, &r.connectionWindowSize) 360 r.flowMu.Unlock() 361 } 362 363 func (r *relay) data(id uint32, data []byte, streamEnded bool) error { 364 // This implementation only allows `WriteData` without padding. Padding is used to improve the 365 // security against attacks like CRIME, but this isn't relevant for a development proxy. 366 // 367 // If padding were allowed, this length would need to vary depending on whether the padding 368 // length octet is present. 369 maxPayloadLength := atomic.LoadUint32(&r.maxFrameSize) 370 371 r.flowMu.Lock() 372 w := r.outputBuffer(id) 373 r.flowMu.Unlock() 374 // If data is larger than what would be permitted at the current max frame size setting, the data 375 // is split across multiple frames. 376 for { 377 nextPayloadLength := uint32(len(data)) 378 if nextPayloadLength > maxPayloadLength { 379 nextPayloadLength = maxPayloadLength 380 } 381 nextPayload := make([]byte, nextPayloadLength) 382 copy(nextPayload, data) 383 data = data[nextPayloadLength:] 384 f := &queuedDataFrame{id, streamEnded && len(data) == 0, nextPayload} 385 386 r.flowMu.Lock() 387 w.enqueue(f) 388 w.emitEligibleFrames(r.output, &r.connectionWindowSize) 389 r.flowMu.Unlock() 390 391 // Some protocols send empty data frames with END_STREAM so the check is done here at the end 392 // of the loop instead of at the beginning of the loop. 393 if len(data) == 0 { 394 break 395 } 396 } 397 return nil 398 } 399 400 func (r *relay) header( 401 id uint32, 402 headers []hpack.HeaderField, 403 streamEnded bool, 404 priority http2.PriorityParam, 405 ) error { 406 encoded, err := r.encodeFull(headers) 407 if err != nil { 408 return fmt.Errorf("encoding headers %v: %w", headers, err) 409 } 410 411 maxPayloadLength := atomic.LoadUint32(&r.maxFrameSize) 412 // Padding is not implemented because the extra security is not needed for a development proxy. 413 // If it were used, a single padding length octet should be deducted from the max header fragment 414 // length. 415 maxHeaderFragmentLength := maxPayloadLength 416 if !priority.IsZero() { 417 maxHeaderFragmentLength -= headersPriorityMetadataLength 418 } 419 chunks := splitIntoChunks(int(maxHeaderFragmentLength), int(maxPayloadLength), encoded) 420 421 r.enqueueFrame(&queuedHeaderFrame{ 422 streamID: id, 423 endStream: streamEnded, 424 priority: priority, 425 chunks: chunks, 426 }) 427 return nil 428 } 429 430 func (r *relay) priority(id uint32, priority http2.PriorityParam) { 431 r.enqueueFrame(&queuedPriorityFrame{ 432 streamID: id, 433 priority: priority, 434 }) 435 } 436 437 func (r *relay) rstStream(id uint32, errCode http2.ErrCode) { 438 r.enqueueFrame(&queuedRSTStreamFrame{ 439 streamID: id, 440 errCode: errCode, 441 }) 442 } 443 444 func (r *relay) pushPromise(id, promiseID uint32, headers []hpack.HeaderField) error { 445 encoded, err := r.encodeFull(headers) 446 if err != nil { 447 return fmt.Errorf("encoding push promise headers %v: %w", headers, err) 448 } 449 450 maxPayloadLength := atomic.LoadUint32(&r.maxFrameSize) 451 maxHeaderFragmentLength := maxPayloadLength - pushPromiseMetadataLength 452 chunks := splitIntoChunks(int(maxHeaderFragmentLength), int(maxPayloadLength), encoded) 453 454 r.enqueueFrame(&queuedPushPromiseFrame{ 455 streamID: id, 456 promiseID: promiseID, 457 chunks: chunks, 458 }) 459 return nil 460 } 461 462 func (r *relay) enqueueFrame(f queuedFrame) { 463 // The frame is first added to the appropriate stream. 464 r.flowMu.Lock() 465 w := r.outputBuffer(f.StreamID()) 466 w.enqueue(f) 467 w.emitEligibleFrames(r.output, &r.connectionWindowSize) 468 r.flowMu.Unlock() 469 } 470 471 func (r *relay) sendQueuedFramesUnderWindowSize() { 472 r.flowMu.Lock() 473 for _, w := range r.outputBuffers { 474 w.emitEligibleFrames(r.output, &r.connectionWindowSize) 475 } 476 r.flowMu.Unlock() 477 } 478 479 // outputBuffer returns the outputBuffer instance for the given stream, creating one if needed. 480 // 481 // This method is not thread-safe. The caller should be holding `flowMu`. 482 func (r *relay) outputBuffer(streamID uint32) *outputBuffer { 483 w, ok := r.outputBuffers[streamID] 484 if !ok { 485 w = &outputBuffer{ 486 windowSize: int(r.initialWindowSize), 487 } 488 r.outputBuffers[streamID] = w 489 } 490 return w 491 } 492 493 // sendWindowUpdates sends WINDOW_UPDATE frames effectively acknowledging consumption of the 494 // given data frame. 495 func (r *relay) sendWindowUpdates(f *http2.DataFrame) error { 496 if len(f.Data()) <= 0 { 497 return nil 498 } 499 r.destMu.Lock() 500 defer r.destMu.Unlock() 501 // First updates the connection level window. 502 if err := r.dest.WriteWindowUpdate(0, uint32(len(f.Data()))); err != nil { 503 return err 504 } 505 // Next updates the stream specific window. 506 return r.dest.WriteWindowUpdate(f.StreamID, uint32(len(f.Data()))) 507 } 508 509 func (r *relay) decodeFull(data []byte) ([]hpack.HeaderField, error) { 510 r.decoderMu.Lock() 511 defer r.decoderMu.Unlock() 512 return r.decoder.DecodeFull(data) 513 } 514 515 func (r *relay) encodeFull(headers []hpack.HeaderField) ([]byte, error) { 516 r.encoderMu.Lock() 517 defer r.encoderMu.Unlock() 518 519 r.reencoded.Reset() 520 var buf bytes.Buffer 521 for _, h := range headers { 522 if *r.enableDebugLogs { 523 if h.Name == "content-type" && h.Value == "application/grpc" { 524 fmt.Fprintf(&buf, " \u001b[1;36m%v\u001b[0m\n", h) 525 } else { 526 fmt.Fprintf(&buf, " %v\n", h) 527 } 528 } 529 if err := r.encoder.WriteField(h); err != nil { 530 return nil, fmt.Errorf("reencoding header field %v in %v: %w", h, headers, err) 531 } 532 } 533 if *r.enableDebugLogs { 534 log.Infof("sending headers %s -> %s:\n%s", r.srcLabel, r.destLabel, buf.Bytes()) 535 } 536 return r.reencoded.Bytes(), nil 537 } 538 539 // outputBuffer stores enqueued output frames for a given stream. 540 type outputBuffer struct { 541 // windowSize indicates how much data the receiver is ready to process. 542 windowSize int 543 queue list.List // contains queuedFrame elements 544 } 545 546 // emitEligibleFrames emits frames that would fit under both the stream window size and the 547 // given connection window size. It updates the given connectionWindowSize if applicable. 548 // 549 // This is not thread-safe. The caller should be holding `relay.flowMu`. 550 func (w *outputBuffer) emitEligibleFrames(output chan queuedFrame, connectionWindowSize *int) { 551 for e := w.queue.Front(); e != nil; { 552 f := e.Value.(queuedFrame) 553 if f.flowControlSize() > *connectionWindowSize || f.flowControlSize() > w.windowSize { 554 break 555 } 556 output <- f 557 558 *connectionWindowSize -= f.flowControlSize() 559 w.windowSize -= f.flowControlSize() 560 561 next := e.Next() 562 w.queue.Remove(e) 563 e = next 564 } 565 } 566 567 // enqueue adds the frame to this stream output. This is not thread-safe. The caller must hold 568 // relay.flowMu. 569 func (w *outputBuffer) enqueue(f queuedFrame) { 570 w.queue.PushBack(f) 571 } 572 573 // continuationState holds the context needed to interpret CONTINUATION frames, specifically whether 574 // the parents were HEADERS or PUSH_PROMISE frames. 575 type continuationState interface { 576 complete(s Processor, headers []hpack.HeaderField) error 577 } 578 579 type headerContinuation struct { 580 priority http2.PriorityParam 581 } 582 583 func (h *headerContinuation) complete(s Processor, headers []hpack.HeaderField) error { 584 return s.Header(headers, true, h.priority) 585 } 586 587 type pushPromiseContinuation struct { 588 promiseID uint32 589 } 590 591 func (p *pushPromiseContinuation) complete(s Processor, headers []hpack.HeaderField) error { 592 return s.PushPromise(p.promiseID, headers) 593 } 594 595 // splitIntoChunks splits header payloads into chunks that respect frame size limits. 596 func splitIntoChunks(firstChunkMax, continuationMax int, data []byte) [][]byte { 597 var chunks [][]byte 598 599 firstChunkLength := len(data) 600 if firstChunkLength > firstChunkMax { 601 firstChunkLength = firstChunkMax 602 } 603 buf := make([]byte, firstChunkLength) 604 copy(buf, data[:firstChunkLength]) 605 chunks = append(chunks, buf) 606 remaining := data[firstChunkLength:] 607 for len(remaining) > 0 { 608 nextChunkLength := len(remaining) 609 if nextChunkLength > continuationMax { 610 nextChunkLength = continuationMax 611 } 612 buf = make([]byte, nextChunkLength) 613 copy(buf, remaining[:nextChunkLength]) 614 chunks = append(chunks, buf) 615 remaining = remaining[nextChunkLength:] 616 } 617 return chunks 618 }