github.com/psiphon-labs/psiphon-tunnel-core@v2.0.28+incompatible/psiphon/httpProxy.go (about) 1 /* 2 * Copyright (c) 2016, Psiphon Inc. 3 * All rights reserved. 4 * 5 * This program is free software: you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License as published by 7 * the Free Software Foundation, either version 3 of the License, or 8 * (at your option) any later version. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program. If not, see <http://www.gnu.org/licenses/>. 17 * 18 */ 19 20 package psiphon 21 22 import ( 23 "bytes" 24 "compress/gzip" 25 "crypto/tls" 26 std_errors "errors" 27 "fmt" 28 "io" 29 "io/ioutil" 30 "net" 31 "net/http" 32 "net/url" 33 "path/filepath" 34 "strconv" 35 "strings" 36 "sync" 37 "sync/atomic" 38 "time" 39 40 "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common" 41 "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/errors" 42 "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/parameters" 43 "github.com/grafov/m3u8" 44 ) 45 46 // HttpProxy is a HTTP server that relays HTTP requests through the Psiphon tunnel. 47 // It includes support for HTTP CONNECT. 48 // 49 // This proxy also offers a "URL proxy" mode that relays requests for HTTP or HTTPS 50 // or URLs specified in the proxy request path. This mode relays either through the 51 // Psiphon tunnel, or directly. 52 // 53 // An example use case for tunneled URL proxy relays is to craft proxied URLs to pass to 54 // components that don't support HTTP or SOCKS proxy settings. For example, the 55 // Android Media Player (http://developer.android.com/reference/android/media/MediaPlayer.html). 56 // To make the Media Player use the Psiphon tunnel, construct a URL such as: 57 // "http://127.0.0.1:<proxy-port>/tunneled/<origin media URL>"; and pass this to the player. 58 // The <origin media URL> must be escaped in such a way that it can be used inside a URL query. 59 // 60 // An example use case for direct, untunneled, relaying is to make use of Go's TLS 61 // stack for HTTPS requests in cases where the native TLS stack is lacking (e.g., 62 // WinHTTP on Windows XP). The URL for direct relaying is: 63 // "http://127.0.0.1:<proxy-port>/direct/<origin URL>". 64 // Again, the <origin URL> must be escaped in such a way that it can be used inside a URL query. 65 // 66 // An example use case for tunneled relaying with rewriting (/tunneled-rewrite/) is when the 67 // content of retrieved files contains URLs that also need to be modified to be tunneled. 68 // For example, in iOS 10 the UIWebView media player does not put requests through the 69 // NSURLProtocol, so they are not tunneled. Instead, we rewrite those URLs to use the URL 70 // proxy, and rewrite retrieved playlist files so they also contain proxied URLs. 71 // 72 // The URL proxy offers /tunneled-icy/ which is compatible with both HTTP and ICY protocol 73 // resources. 74 // 75 // Origin URLs must include the scheme prefix ("http://" or "https://") and must be 76 // URL encoded. 77 // 78 type HttpProxy struct { 79 config *Config 80 tunneler Tunneler 81 listener net.Listener 82 serveWaitGroup *sync.WaitGroup 83 httpProxyTunneledRelay *http.Transport 84 urlProxyTunneledRelay *http.Transport 85 urlProxyTunneledClient *http.Client 86 urlProxyDirectRelay *http.Transport 87 urlProxyDirectClient *http.Client 88 responseHeaderTimeout time.Duration 89 openConns *common.Conns 90 stopListeningBroadcast chan struct{} 91 listenIP string 92 listenPort int 93 } 94 95 var _HTTP_PROXY_TYPE = "HTTP" 96 97 // NewHttpProxy initializes and runs a new HTTP proxy server. 98 func NewHttpProxy( 99 config *Config, 100 tunneler Tunneler, 101 listenIP string) (proxy *HttpProxy, err error) { 102 103 listener, portInUse, err := makeLocalProxyListener( 104 listenIP, config.LocalHttpProxyPort) 105 if err != nil { 106 if portInUse { 107 NoticeHttpProxyPortInUse(config.LocalHttpProxyPort) 108 } 109 return nil, errors.Trace(err) 110 } 111 112 tunneledDialer := func(_, addr string) (conn net.Conn, err error) { 113 // downstreamConn is not set in this case, as there is not a fixed 114 // association between a downstream client connection and a particular 115 // tunnel. 116 return tunneler.Dial(addr, nil) 117 } 118 directDialer := func(_, addr string) (conn net.Conn, err error) { 119 return tunneler.DirectDial(addr) 120 } 121 122 p := config.GetParameters().Get() 123 responseHeaderTimeout := p.Duration(parameters.HTTPProxyOriginServerTimeout) 124 maxIdleConnsPerHost := p.Int(parameters.HTTPProxyMaxIdleConnectionsPerHost) 125 126 // TODO: could HTTP proxy share a tunneled transport with URL proxy? 127 // For now, keeping them distinct just to be conservative. 128 httpProxyTunneledRelay := &http.Transport{ 129 Dial: tunneledDialer, 130 MaxIdleConnsPerHost: maxIdleConnsPerHost, 131 ResponseHeaderTimeout: responseHeaderTimeout, 132 } 133 134 // Note: URL proxy relays use http.Client for upstream requests, so 135 // redirects will be followed. HTTP proxy should not follow redirects 136 // and simply uses http.Transport directly. 137 138 urlProxyTunneledRelay := &http.Transport{ 139 Dial: tunneledDialer, 140 MaxIdleConnsPerHost: maxIdleConnsPerHost, 141 ResponseHeaderTimeout: responseHeaderTimeout, 142 } 143 urlProxyTunneledClient := &http.Client{ 144 Transport: urlProxyTunneledRelay, 145 Jar: nil, // TODO: cookie support for URL proxy? 146 147 // Leaving original value in the note below: 148 // Note: don't use this timeout -- it interrupts downloads of large response bodies 149 //Timeout: HTTP_PROXY_ORIGIN_SERVER_TIMEOUT, 150 } 151 152 urlProxyDirectRelay := &http.Transport{ 153 Dial: directDialer, 154 MaxIdleConnsPerHost: maxIdleConnsPerHost, 155 ResponseHeaderTimeout: responseHeaderTimeout, 156 } 157 urlProxyDirectClient := &http.Client{ 158 Transport: urlProxyDirectRelay, 159 Jar: nil, 160 } 161 162 proxyIP, proxyPortString, _ := net.SplitHostPort(listener.Addr().String()) 163 proxyPort, _ := strconv.Atoi(proxyPortString) 164 165 proxy = &HttpProxy{ 166 config: config, 167 tunneler: tunneler, 168 listener: listener, 169 serveWaitGroup: new(sync.WaitGroup), 170 httpProxyTunneledRelay: httpProxyTunneledRelay, 171 urlProxyTunneledRelay: urlProxyTunneledRelay, 172 urlProxyTunneledClient: urlProxyTunneledClient, 173 urlProxyDirectRelay: urlProxyDirectRelay, 174 urlProxyDirectClient: urlProxyDirectClient, 175 responseHeaderTimeout: responseHeaderTimeout, 176 openConns: common.NewConns(), 177 stopListeningBroadcast: make(chan struct{}), 178 listenIP: proxyIP, 179 listenPort: proxyPort, 180 } 181 proxy.serveWaitGroup.Add(1) 182 go proxy.serve() 183 184 // TODO: NoticeListeningHttpProxyPort is emitted after net.Listen 185 // but before go proxy.server() and httpServer.Serve(), and this 186 // appears to cause client connections to the HTTP proxy to fail 187 // (in controller_test.go, only when a tunnel is established very quickly 188 // and NoticeTunnels is emitted and the client makes a request -- all 189 // before the proxy.server() goroutine runs). 190 // This condition doesn't arise in Go 1.4, just in Go tip (pre-1.5). 191 // Note that httpServer.Serve() blocks so the fix can't be to emit 192 // NoticeListeningHttpProxyPort after that call. 193 // Also, check the listen backlog queue length -- shouldn't it be possible 194 // to enqueue pending connections between net.Listen() and httpServer.Serve()? 195 NoticeListeningHttpProxyPort(proxy.listenPort) 196 197 return proxy, nil 198 } 199 200 // Close terminates the HTTP server. 201 func (proxy *HttpProxy) Close() { 202 close(proxy.stopListeningBroadcast) 203 proxy.listener.Close() 204 proxy.serveWaitGroup.Wait() 205 // Close local->proxy persistent connections 206 proxy.openConns.CloseAll() 207 // Close idle proxy->origin persistent connections 208 // TODO: also close active connections 209 proxy.httpProxyTunneledRelay.CloseIdleConnections() 210 proxy.urlProxyTunneledRelay.CloseIdleConnections() 211 proxy.urlProxyDirectRelay.CloseIdleConnections() 212 } 213 214 // ServeHTTP receives HTTP requests and proxies them. CONNECT requests 215 // are hijacked and all data is relayed. Other HTTP requests are proxied 216 // with explicit round trips. In both cases, the tunnel is used for proxied 217 // traffic. 218 // 219 // Implementation is based on: 220 // 221 // https://github.com/justmao945/mallory 222 // Copyright (c) 2014 JianjunMao 223 // The MIT License (MIT) 224 // 225 // https://golang.org/src/pkg/net/http/httputil/reverseproxy.go 226 // Copyright 2011 The Go Authors. All rights reserved. 227 // Use of this source code is governed by a BSD-style 228 // license that can be found in the LICENSE file. 229 // 230 func (proxy *HttpProxy) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) { 231 if request.Method == "CONNECT" { 232 conn := hijack(responseWriter) 233 if conn == nil { 234 // hijack emits an alert notice 235 http.Error(responseWriter, "", http.StatusInternalServerError) 236 return 237 } 238 go func() { 239 err := proxy.httpConnectHandler(conn, request.URL.Host) 240 if err != nil { 241 NoticeWarning("%s", errors.Trace(err)) 242 } 243 }() 244 } else if request.URL.IsAbs() { 245 proxy.httpProxyHandler(responseWriter, request) 246 } else { 247 proxy.urlProxyHandler(responseWriter, request) 248 } 249 } 250 251 func (proxy *HttpProxy) httpConnectHandler(localConn net.Conn, target string) (err error) { 252 defer localConn.Close() 253 defer proxy.openConns.Remove(localConn) 254 proxy.openConns.Add(localConn) 255 // Setting downstreamConn so localConn.Close() will be called when remoteConn.Close() is called. 256 // This ensures that the downstream client (e.g., web browser) doesn't keep waiting on the 257 // open connection for data which will never arrive. 258 remoteConn, err := proxy.tunneler.Dial(target, localConn) 259 if err != nil { 260 return errors.Trace(err) 261 } 262 defer remoteConn.Close() 263 _, err = localConn.Write([]byte("HTTP/1.1 200 OK\r\n\r\n")) 264 if err != nil { 265 return errors.Trace(err) 266 } 267 LocalProxyRelay(proxy.config, _HTTP_PROXY_TYPE, localConn, remoteConn) 268 return nil 269 } 270 271 func (proxy *HttpProxy) httpProxyHandler(responseWriter http.ResponseWriter, request *http.Request) { 272 proxy.relayHTTPRequest(nil, proxy.httpProxyTunneledRelay, request, responseWriter, nil, nil) 273 } 274 275 const ( 276 URL_PROXY_TUNNELED_REQUEST_PATH = "/tunneled/" 277 URL_PROXY_TUNNELED_REWRITE_REQUEST_PATH = "/tunneled-rewrite/" 278 URL_PROXY_TUNNELED_ICY_REQUEST_PATH = "/tunneled-icy/" 279 URL_PROXY_DIRECT_REQUEST_PATH = "/direct/" 280 ) 281 282 func (proxy *HttpProxy) urlProxyHandler(responseWriter http.ResponseWriter, request *http.Request) { 283 284 var client *http.Client 285 var rewriteICYStatus *rewriteICYStatus 286 var originURLString string 287 var err error 288 var rewrites url.Values 289 290 // Request URL should be "/tunneled/<origin URL>" or "/direct/<origin URL>" and the 291 // origin URL must be URL encoded. 292 switch { 293 case strings.HasPrefix(request.URL.RawPath, URL_PROXY_TUNNELED_REQUEST_PATH): 294 originURLString, err = url.QueryUnescape(request.URL.RawPath[len(URL_PROXY_TUNNELED_REQUEST_PATH):]) 295 client = proxy.urlProxyTunneledClient 296 297 case strings.HasPrefix(request.URL.RawPath, URL_PROXY_TUNNELED_REWRITE_REQUEST_PATH): 298 originURLString, err = url.QueryUnescape(request.URL.RawPath[len(URL_PROXY_TUNNELED_REWRITE_REQUEST_PATH):]) 299 client = proxy.urlProxyTunneledClient 300 rewrites = request.URL.Query() 301 302 case strings.HasPrefix(request.URL.RawPath, URL_PROXY_TUNNELED_ICY_REQUEST_PATH): 303 originURLString, err = url.QueryUnescape(request.URL.RawPath[len(URL_PROXY_TUNNELED_ICY_REQUEST_PATH):]) 304 client, rewriteICYStatus = proxy.makeRewriteICYClient() 305 rewrites = request.URL.Query() 306 307 case strings.HasPrefix(request.URL.RawPath, URL_PROXY_DIRECT_REQUEST_PATH): 308 originURLString, err = url.QueryUnescape(request.URL.RawPath[len(URL_PROXY_DIRECT_REQUEST_PATH):]) 309 client = proxy.urlProxyDirectClient 310 311 default: 312 err = std_errors.New("missing origin URL") 313 } 314 if err != nil { 315 NoticeWarning("%s", errors.Trace(common.RedactURLError(err))) 316 forceClose(responseWriter) 317 return 318 } 319 320 // Origin URL must be well-formed, absolute, and have a scheme of "http" or "https" 321 originURL, err := common.SafeParseRequestURI(originURLString) 322 if err != nil { 323 NoticeWarning("%s", errors.Trace(common.RedactURLError(err))) 324 forceClose(responseWriter) 325 return 326 } 327 if !originURL.IsAbs() || (originURL.Scheme != "http" && originURL.Scheme != "https") { 328 NoticeWarning("invalid origin URL") 329 forceClose(responseWriter) 330 return 331 } 332 333 // Transform received request to directly reference the origin URL 334 request.Host = originURL.Host 335 request.URL = originURL 336 337 proxy.relayHTTPRequest(client, nil, request, responseWriter, rewrites, rewriteICYStatus) 338 } 339 340 // rewriteICYConn rewrites an ICY procotol responses to that it may be 341 // consumed by Go's http package. rewriteICYConn expects the ICY response to 342 // be equivalent to HTTP/1.1 with the exception of the protocol name in the 343 // status line, which is the one part that is rewritten. Responses that are 344 // already HTTP are passed through unmodified. 345 type rewriteICYConn struct { 346 net.Conn 347 doneRewriting int32 348 isICY *int32 349 } 350 351 func (conn *rewriteICYConn) Read(b []byte) (int, error) { 352 353 if !atomic.CompareAndSwapInt32(&conn.doneRewriting, 0, 1) { 354 return conn.Conn.Read(b) 355 } 356 357 if len(b) < 3 { 358 // Don't attempt to rewrite the protocol when insufficient 359 // buffer space. This is not expected to happen in practise 360 // when Go's http reads the response, so for now we just 361 // skip the rewrite instead of tracking state accross Reads. 362 return conn.Conn.Read(b) 363 } 364 365 // Expect to read either "ICY" or "HTT". 366 367 n, err := conn.Conn.Read(b[:3]) 368 if err != nil { 369 return n, err 370 } 371 372 if bytes.Equal(b[:3], []byte("ICY")) { 373 atomic.StoreInt32(conn.isICY, 1) 374 protocol := "HTTP/1.0" 375 copy(b, []byte(protocol)) 376 return len(protocol), nil 377 } 378 379 return n, nil 380 } 381 382 type rewriteICYStatus struct { 383 isFirstConnICY int32 384 } 385 386 func (status *rewriteICYStatus) isICY() bool { 387 return atomic.LoadInt32(&status.isFirstConnICY) == 1 388 } 389 390 // makeRewriteICYClient creates an http.Client with a Transport configured to 391 // use rewriteICYConn. Both HTTP and HTTPS are handled. The http.Client is 392 // intended to be used for one single request. The client disables keep alives 393 // as rewriteICYConn can only rewrite the first response in a connection. The 394 // returned rewriteICYStatus indicates whether the first response for the first 395 // request was ICY, allowing the downstream relayed response to replicate the 396 // ICY protocol. 397 func (proxy *HttpProxy) makeRewriteICYClient() (*http.Client, *rewriteICYStatus) { 398 399 rewriteICYStatus := &rewriteICYStatus{} 400 401 tunneledDialer := func(_, addr string) (conn net.Conn, err error) { 402 // See comment in NewHttpProxy regarding downstreamConn 403 return proxy.tunneler.Dial(addr, nil) 404 } 405 406 dial := func(network, address string) (net.Conn, error) { 407 408 conn, err := tunneledDialer(network, address) 409 if err != nil { 410 return nil, errors.Trace(err) 411 } 412 413 return &rewriteICYConn{ 414 Conn: conn, 415 isICY: &rewriteICYStatus.isFirstConnICY, 416 }, nil 417 } 418 419 dialTLS := func(network, address string) (net.Conn, error) { 420 421 conn, err := tunneledDialer(network, address) 422 if err != nil { 423 return nil, errors.Trace(err) 424 } 425 426 serverName, _, err := net.SplitHostPort(address) 427 if err != nil { 428 conn.Close() 429 return nil, errors.Trace(err) 430 } 431 432 tlsConn := tls.Client(conn, &tls.Config{ServerName: serverName}) 433 434 resultChannel := make(chan error, 1) 435 436 timeout := proxy.responseHeaderTimeout 437 afterFunc := time.AfterFunc(timeout, func() { 438 resultChannel <- errors.TraceNew("TLS handshake timeout") 439 }) 440 defer afterFunc.Stop() 441 442 go func() { 443 resultChannel <- tlsConn.Handshake() 444 }() 445 446 err = <-resultChannel 447 if err != nil { 448 conn.Close() 449 return nil, errors.Trace(err) 450 } 451 452 err = tlsConn.VerifyHostname(serverName) 453 if err != nil { 454 conn.Close() 455 return nil, errors.Trace(err) 456 } 457 458 return &rewriteICYConn{ 459 Conn: tlsConn, 460 isICY: &rewriteICYStatus.isFirstConnICY, 461 }, nil 462 463 } 464 465 return &http.Client{ 466 Transport: &http.Transport{ 467 Dial: dial, 468 DialTLS: dialTLS, 469 DisableKeepAlives: true, 470 ResponseHeaderTimeout: proxy.responseHeaderTimeout, 471 }, 472 }, rewriteICYStatus 473 } 474 475 func (proxy *HttpProxy) relayHTTPRequest( 476 client *http.Client, 477 transport *http.Transport, 478 request *http.Request, 479 responseWriter http.ResponseWriter, 480 rewrites url.Values, 481 rewriteICYStatus *rewriteICYStatus) { 482 483 // Transform received request struct before using as input to relayed request 484 request.Close = false 485 request.RequestURI = "" 486 for _, key := range hopHeaders { 487 request.Header.Del(key) 488 } 489 490 // Relay the HTTP request and get the response. Use a client when supplied, 491 // otherwise a transport. A client handles cookies and redirects, and a 492 // transport does not. 493 var response *http.Response 494 var err error 495 if client != nil { 496 response, err = client.Do(request) 497 } else { 498 response, err = transport.RoundTrip(request) 499 } 500 501 if err != nil { 502 NoticeWarning("%s", errors.Trace(common.RedactURLError(err))) 503 forceClose(responseWriter) 504 return 505 } 506 507 defer response.Body.Close() 508 509 // Note: Rewrite functions are responsible for leaving response.Body in 510 // a valid, readable state if there's no error. 511 512 for key := range rewrites { 513 var err error 514 515 switch key { 516 case "m3u8": 517 err = rewriteM3U8(proxy.listenIP, proxy.listenPort, response) 518 } 519 520 if err != nil { 521 NoticeWarning("URL proxy rewrite failed for %s: %s", key, errors.Trace(err)) 522 forceClose(responseWriter) 523 response.Body.Close() 524 return 525 } 526 } 527 528 // Relay the remote response headers 529 530 for _, key := range hopHeaders { 531 response.Header.Del(key) 532 } 533 for key := range responseWriter.Header() { 534 responseWriter.Header().Del(key) 535 } 536 for key, values := range response.Header { 537 for _, value := range values { 538 responseWriter.Header().Add(key, value) 539 } 540 } 541 542 // Send the response downstream 543 544 if rewriteICYStatus != nil && rewriteICYStatus.isICY() { 545 546 // Custom ICY response, using "ICY" as the protocol name 547 // but otherwise equivalent to the HTTP response. 548 549 // As the ICY http.Transport has disabled keep-alives, 550 // hijacking here does not disrupt an otherwise persistent 551 // connection. 552 553 conn := hijack(responseWriter) 554 if conn == nil { 555 // hijack emits an alert notice 556 return 557 } 558 559 _, err := fmt.Fprintf( 560 conn, 561 "ICY %d %s\r\n", 562 response.StatusCode, 563 http.StatusText(response.StatusCode)) 564 if err != nil { 565 NoticeWarning("write status line failed: %s", errors.Trace(err)) 566 conn.Close() 567 return 568 } 569 570 err = responseWriter.Header().Write(conn) 571 if err != nil { 572 NoticeWarning("write headers failed: %s", errors.Trace(err)) 573 conn.Close() 574 return 575 } 576 577 _, err = RelayCopyBuffer(proxy.config, conn, response.Body) 578 if err != nil { 579 NoticeWarning("write body failed: %s", errors.Trace(err)) 580 conn.Close() 581 return 582 } 583 584 } else { 585 586 // Standard HTTP response. 587 588 responseWriter.WriteHeader(response.StatusCode) 589 _, err = RelayCopyBuffer(proxy.config, responseWriter, response.Body) 590 if err != nil { 591 NoticeWarning("%s", errors.Trace(err)) 592 forceClose(responseWriter) 593 return 594 } 595 } 596 } 597 598 // forceClose hijacks and closes persistent connections. This is used 599 // to ensure local persistent connections into the HTTP proxy are closed 600 // when ServeHTTP encounters an error. 601 func forceClose(responseWriter http.ResponseWriter) { 602 conn := hijack(responseWriter) 603 if conn != nil { 604 conn.Close() 605 } 606 } 607 608 func hijack(responseWriter http.ResponseWriter) net.Conn { 609 hijacker, ok := responseWriter.(http.Hijacker) 610 if !ok { 611 NoticeWarning("%s", errors.TraceNew("responseWriter is not an http.Hijacker")) 612 return nil 613 } 614 conn, _, err := hijacker.Hijack() 615 if err != nil { 616 NoticeWarning("%s", errors.Tracef("responseWriter hijack failed: %s", err)) 617 return nil 618 } 619 return conn 620 } 621 622 // From https://golang.org/src/pkg/net/http/httputil/reverseproxy.go: 623 // Hop-by-hop headers. These are removed when sent to the backend. 624 // http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html 625 var hopHeaders = []string{ 626 "Connection", 627 "Keep-Alive", 628 "Proxy-Authenticate", 629 "Proxy-Authorization", 630 "Proxy-Connection", // see: http://homepage.ntlworld.com/jonathan.deboynepollard/FGA/web-proxy-connection-header.html 631 "Te", // canonicalized version of "TE" 632 "Trailers", 633 "Transfer-Encoding", 634 "Upgrade", 635 } 636 637 // httpConnStateCallback is called by http.Server when the state of a local->proxy 638 // connection changes. Open connections are tracked so that all local->proxy persistent 639 // connections can be closed by HttpProxy.Close() 640 // TODO: if the HttpProxy is decoupled from a single Tunnel instance and 641 // instead uses the "current" Tunnel, it may not be necessary to close 642 // local persistent connections when the tunnel reconnects. 643 func (proxy *HttpProxy) httpConnStateCallback(conn net.Conn, connState http.ConnState) { 644 switch connState { 645 case http.StateNew: 646 proxy.openConns.Add(conn) 647 case http.StateActive, http.StateIdle: 648 // No action 649 case http.StateHijacked, http.StateClosed: 650 proxy.openConns.Remove(conn) 651 } 652 } 653 654 func (proxy *HttpProxy) serve() { 655 defer proxy.listener.Close() 656 defer proxy.serveWaitGroup.Done() 657 httpServer := &http.Server{ 658 Handler: proxy, 659 ConnState: proxy.httpConnStateCallback, 660 } 661 // Note: will be interrupted by listener.Close() call made by proxy.Close() 662 err := httpServer.Serve(proxy.listener) 663 // Can't check for the exact error that Close() will cause in Accept(), 664 // (see: https://code.google.com/p/go/issues/detail?id=4373). So using an 665 // explicit stop signal to stop gracefully. 666 select { 667 case <-proxy.stopListeningBroadcast: 668 default: 669 if err != nil { 670 proxy.tunneler.SignalComponentFailure() 671 NoticeLocalProxyError(_HTTP_PROXY_TYPE, errors.Trace(err)) 672 } 673 } 674 NoticeInfo("HTTP proxy stopped") 675 } 676 677 // 678 // Rewrite functions 679 // 680 681 // toAbsoluteURL takes a base URL and a relative URL and constructs an appropriate absolute URL. 682 func toAbsoluteURL(baseURL *url.URL, relativeURLString string) string { 683 relativeURL, err := common.SafeParseURL(relativeURLString) 684 685 if err != nil { 686 return "" 687 } 688 689 if relativeURL.IsAbs() { 690 return relativeURL.String() 691 } 692 693 return baseURL.ResolveReference(relativeURL).String() 694 } 695 696 // proxifyURL takes an absolute URL and rewrites it to go through the local URL proxy. 697 // urlProxy port is the local HTTP proxy port. 698 // 699 // If rewriteParams is nil, then no rewriting will be done. Otherwise, it should contain 700 // supported rewriting flags (like "m3u8"). 701 func proxifyURL(localHTTPProxyIP string, localHTTPProxyPort int, urlString string, rewriteParams []string) string { 702 703 // Note that we need to use the "opaque" form of URL so that it doesn't double-escape the path. See: https://github.com/golang/go/issues/10887 704 705 // TODO: IPv6 support 706 if localHTTPProxyIP == "0.0.0.0" { 707 localHTTPProxyIP = "127.0.0.1" 708 } 709 710 proxyPath := URL_PROXY_TUNNELED_REQUEST_PATH 711 if rewriteParams != nil { 712 proxyPath = URL_PROXY_TUNNELED_REWRITE_REQUEST_PATH 713 } 714 opaqueFormat := fmt.Sprintf("//%%s:%%d%s%%s", proxyPath) 715 716 var proxifiedURL url.URL 717 718 proxifiedURL.Scheme = "http" 719 proxifiedURL.Opaque = fmt.Sprintf(opaqueFormat, localHTTPProxyIP, localHTTPProxyPort, url.QueryEscape(urlString)) 720 721 qp := proxifiedURL.Query() 722 for _, rewrite := range rewriteParams { 723 qp.Set(rewrite, "") 724 } 725 proxifiedURL.RawQuery = qp.Encode() 726 727 return proxifiedURL.String() 728 } 729 730 // Rewrite the contents of the M3U8 file in body to be compatible with URL proxying. 731 // If error is returned, response body may not be valid for reading. 732 func rewriteM3U8(localHTTPProxyIP string, localHTTPProxyPort int, response *http.Response) error { 733 // Check URL path extension 734 extension := filepath.Ext(response.Request.URL.Path) 735 var shouldHandle = (extension == ".m3u8") 736 737 // If not .m3u8 then check content type 738 if !shouldHandle { 739 contentType := strings.ToLower(response.Header.Get("Content-Type")) 740 shouldHandle = (contentType == "application/x-mpegurl" || contentType == "vnd.apple.mpegurl") 741 } 742 743 if !shouldHandle { 744 return nil 745 } 746 747 var reader io.ReadCloser 748 749 switch response.Header.Get("Content-Encoding") { 750 case "gzip": 751 var err error 752 753 reader, err = gzip.NewReader(response.Body) 754 if err != nil { 755 return errors.Trace(err) 756 } 757 758 // Unset Content-Encoding. 759 // There's is no point in deflating the decoded/rewritten content 760 response.Header.Del("Content-Encoding") 761 defer reader.Close() 762 default: 763 reader = response.Body 764 } 765 766 contentBodyBytes, err := ioutil.ReadAll(reader) 767 response.Body.Close() 768 769 if err != nil { 770 return errors.Trace(err) 771 } 772 773 p, listType, err := m3u8.Decode(*bytes.NewBuffer(contentBodyBytes), true) 774 if err != nil { 775 // Don't pass this error up. Just don't change anything. 776 response.Body = ioutil.NopCloser(bytes.NewReader(contentBodyBytes)) 777 response.Header.Set("Content-Length", strconv.FormatInt(int64(len(contentBodyBytes)), 10)) 778 return nil 779 } 780 781 var rewrittenBodyBytes []byte 782 783 switch listType { 784 case m3u8.MEDIA: 785 mediapl := p.(*m3u8.MediaPlaylist) 786 for _, segment := range mediapl.Segments { 787 if segment == nil { 788 break 789 } 790 791 if segment.URI != "" { 792 segment.URI = proxifyURL(localHTTPProxyIP, localHTTPProxyPort, toAbsoluteURL(response.Request.URL, segment.URI), nil) 793 } 794 795 if segment.Key != nil && segment.Key.URI != "" { 796 segment.Key.URI = proxifyURL(localHTTPProxyIP, localHTTPProxyPort, toAbsoluteURL(response.Request.URL, segment.Key.URI), nil) 797 } 798 799 if segment.Map != nil && segment.Map.URI != "" { 800 segment.Map.URI = proxifyURL(localHTTPProxyIP, localHTTPProxyPort, toAbsoluteURL(response.Request.URL, segment.Map.URI), nil) 801 } 802 } 803 rewrittenBodyBytes = []byte(mediapl.String()) 804 case m3u8.MASTER: 805 masterpl := p.(*m3u8.MasterPlaylist) 806 for _, variant := range masterpl.Variants { 807 if variant == nil { 808 break 809 } 810 811 if variant.URI != "" { 812 variant.URI = proxifyURL(localHTTPProxyIP, localHTTPProxyPort, toAbsoluteURL(response.Request.URL, variant.URI), []string{"m3u8"}) 813 } 814 815 for _, alternative := range variant.Alternatives { 816 if alternative == nil { 817 break 818 } 819 820 if alternative.URI != "" { 821 alternative.URI = proxifyURL(localHTTPProxyIP, localHTTPProxyPort, toAbsoluteURL(response.Request.URL, alternative.URI), []string{"m3u8"}) 822 } 823 } 824 } 825 rewrittenBodyBytes = []byte(masterpl.String()) 826 } 827 828 var responseBodyBytes []byte 829 830 if len(rewrittenBodyBytes) == 0 { 831 responseBodyBytes = contentBodyBytes[:] 832 } else { 833 responseBodyBytes = rewrittenBodyBytes[:] 834 // When rewriting the original URL so that it was URL-proxied, we lost the 835 // file extension of it. That means we'd better make sure the Content-Type is set. 836 response.Header.Set("Content-Type", "application/x-mpegurl") 837 } 838 839 response.Header.Set("Content-Length", strconv.FormatInt(int64(len(responseBodyBytes)), 10)) 840 response.Body = ioutil.NopCloser(bytes.NewReader(responseBodyBytes)) 841 842 return nil 843 }