github.com/dbernstein1/tyk@v2.9.0-beta9-dl-apic+incompatible/gateway/reverse_proxy.go (about) 1 // Copyright 2011 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Fork of Go's net/http/httputil/reverseproxy.go with multiple changes, 6 // including: 7 // 8 // * caching 9 // * load balancing 10 // * service discovery 11 12 package gateway 13 14 import ( 15 "bytes" 16 "context" 17 "crypto/tls" 18 "fmt" 19 "io" 20 "io/ioutil" 21 "net" 22 "net/http" 23 "net/url" 24 "strconv" 25 "strings" 26 "sync" 27 "time" 28 29 "github.com/TykTechnologies/tyk/apidef" 30 "github.com/TykTechnologies/tyk/config" 31 "github.com/TykTechnologies/tyk/ctx" 32 "github.com/TykTechnologies/tyk/headers" 33 "github.com/TykTechnologies/tyk/regexp" 34 "github.com/TykTechnologies/tyk/trace" 35 "github.com/TykTechnologies/tyk/user" 36 opentracing "github.com/opentracing/opentracing-go" 37 "github.com/opentracing/opentracing-go/ext" 38 cache "github.com/pmylund/go-cache" 39 "github.com/sirupsen/logrus" 40 "golang.org/x/net/http2" 41 ) 42 43 const defaultUserAgent = "Tyk/" + VERSION 44 45 // Gateway's custom response headers 46 const ( 47 XRateLimitLimit = "X-RateLimit-Limit" 48 XRateLimitRemaining = "X-RateLimit-Remaining" 49 XRateLimitReset = "X-RateLimit-Reset" 50 ) 51 52 var ServiceCache *cache.Cache 53 54 func urlFromService(spec *APISpec) (*apidef.HostList, error) { 55 56 doCacheRefresh := func() (*apidef.HostList, error) { 57 log.Debug("--> Refreshing") 58 spec.ServiceRefreshInProgress = true 59 defer func() { spec.ServiceRefreshInProgress = false }() 60 sd := ServiceDiscovery{} 61 sd.Init(&spec.Proxy.ServiceDiscovery) 62 data, err := sd.Target(spec.Proxy.ServiceDiscovery.QueryEndpoint) 63 if err != nil { 64 return nil, err 65 } 66 spec.HasRun = true 67 // Set the cached value 68 if data.Len() == 0 { 69 log.Warning("[PROXY][SD] Service Discovery returned empty host list! Returning last good set.") 70 71 if spec.LastGoodHostList == nil { 72 log.Warning("[PROXY][SD] Last good host list is nil, returning empty set.") 73 spec.LastGoodHostList = apidef.NewHostList() 74 } 75 76 return spec.LastGoodHostList, nil 77 } 78 79 ServiceCache.Set(spec.APIID, data, cache.DefaultExpiration) 80 // Stash it too 81 spec.LastGoodHostList = data 82 return data, nil 83 } 84 85 // First time? Refresh the cache and return that 86 if !spec.HasRun { 87 log.Debug("First run! Setting cache") 88 return doCacheRefresh() 89 } 90 91 // Not first run - check the cache 92 cachedServiceData, found := ServiceCache.Get(spec.APIID) 93 if !found { 94 if spec.ServiceRefreshInProgress { 95 // Are we already refreshing the cache? skip and return last good conf 96 log.Debug("Cache expired! But service refresh in progress") 97 return spec.LastGoodHostList, nil 98 } 99 // Refresh the spec 100 log.Debug("Cache expired! Refreshing...") 101 return doCacheRefresh() 102 } 103 104 log.Debug("Returning from cache.") 105 return cachedServiceData.(*apidef.HostList), nil 106 } 107 108 // httpScheme matches http://* and https://*, case insensitive 109 var httpScheme = regexp.MustCompile(`^(?i)https?://`) 110 111 func EnsureTransport(host string) string { 112 if httpScheme.MatchString(host) { 113 return host 114 } 115 // no prototcol, assume http 116 return "http://" + host 117 } 118 119 func nextTarget(targetData *apidef.HostList, spec *APISpec) (string, error) { 120 if spec.Proxy.EnableLoadBalancing { 121 log.Debug("[PROXY] [LOAD BALANCING] Load balancer enabled, getting upstream target") 122 // Use a HostList 123 startPos := spec.RoundRobin.WithLen(targetData.Len()) 124 pos := startPos 125 for { 126 gotHost, err := targetData.GetIndex(pos) 127 if err != nil { 128 return "", err 129 } 130 131 host := EnsureTransport(gotHost) 132 133 if !spec.Proxy.CheckHostAgainstUptimeTests { 134 return host, nil // we don't care if it's up 135 } 136 if !GlobalHostChecker.HostDown(host) { 137 return host, nil // we do care and it's up 138 } 139 // if the host is down, keep trying all the rest 140 // in order from where we started. 141 if pos = (pos + 1) % targetData.Len(); pos == startPos { 142 return "", fmt.Errorf("all hosts are down, uptime tests are failing") 143 } 144 } 145 146 } 147 // Use standard target - might still be service data 148 log.Debug("TARGET DATA:", targetData) 149 150 gotHost, err := targetData.GetIndex(0) 151 if err != nil { 152 return "", err 153 } 154 return EnsureTransport(gotHost), nil 155 } 156 157 var ( 158 onceStartAllHostsDown sync.Once 159 160 allHostsDownURL string 161 ) 162 163 // TykNewSingleHostReverseProxy returns a new ReverseProxy that rewrites 164 // URLs to the scheme, host, and base path provided in target. If the 165 // target's path is "/base" and the incoming request was for "/dir", 166 // the target request will be for /base/dir. This version modifies the 167 // stdlib version by also setting the host to the target, this allows 168 // us to work with heroku and other such providers 169 func TykNewSingleHostReverseProxy(target *url.URL, spec *APISpec) *ReverseProxy { 170 onceStartAllHostsDown.Do(func() { 171 handler := func(w http.ResponseWriter, r *http.Request) { 172 http.Error(w, "all hosts are down", http.StatusServiceUnavailable) 173 } 174 listener, err := net.Listen("tcp", "127.0.0.1:0") 175 if err != nil { 176 panic(err) 177 } 178 server := &http.Server{ 179 Handler: http.HandlerFunc(handler), 180 ReadTimeout: 1 * time.Second, 181 WriteTimeout: 1 * time.Second, 182 MaxHeaderBytes: 1 << 20, 183 } 184 allHostsDownURL = "http://" + listener.Addr().String() 185 go func() { 186 panic(server.Serve(listener)) 187 }() 188 }) 189 if spec.Proxy.ServiceDiscovery.UseDiscoveryService { 190 log.Debug("[PROXY] Service discovery enabled") 191 if ServiceCache == nil { 192 log.Debug("[PROXY] Service cache initialising") 193 expiry := 120 194 if spec.Proxy.ServiceDiscovery.CacheTimeout > 0 { 195 expiry = int(spec.Proxy.ServiceDiscovery.CacheTimeout) 196 } else if spec.GlobalConfig.ServiceDiscovery.DefaultCacheTimeout > 0 { 197 expiry = spec.GlobalConfig.ServiceDiscovery.DefaultCacheTimeout 198 } 199 ServiceCache = cache.New(time.Duration(expiry)*time.Second, 15*time.Second) 200 } 201 } 202 203 targetQuery := target.RawQuery 204 director := func(req *http.Request) { 205 hostList := spec.Proxy.StructuredTargetList 206 switch { 207 case spec.Proxy.ServiceDiscovery.UseDiscoveryService: 208 var err error 209 hostList, err = urlFromService(spec) 210 if err != nil { 211 log.Error("[PROXY] [SERVICE DISCOVERY] Failed target lookup: ", err) 212 break 213 } 214 fallthrough // implies load balancing, with replaced host list 215 case spec.Proxy.EnableLoadBalancing: 216 host, err := nextTarget(hostList, spec) 217 if err != nil { 218 log.Error("[PROXY] [LOAD BALANCING] ", err) 219 host = allHostsDownURL 220 } 221 lbRemote, err := url.Parse(host) 222 if err != nil { 223 log.Error("[PROXY] [LOAD BALANCING] Couldn't parse target URL:", err) 224 } else { 225 // Only replace target if everything is OK 226 target = lbRemote 227 targetQuery = target.RawQuery 228 } 229 } 230 231 targetToUse := target 232 233 if spec.URLRewriteEnabled && req.Context().Value(ctx.RetainHost) == true { 234 log.Debug("Detected host rewrite, overriding target") 235 tmpTarget, err := url.Parse(req.URL.String()) 236 if err != nil { 237 log.Error("Failed to parse URL! Err: ", err) 238 } else { 239 // Specifically override with a URL rewrite 240 targetToUse = tmpTarget 241 } 242 } 243 244 // No override, and no load balancing? Use the existing target 245 246 // if this is false, there was an url rewrite, thus we 247 // don't want to do anything to the path - req.URL is 248 // already final. 249 if targetToUse == target { 250 req.URL.Scheme = targetToUse.Scheme 251 req.URL.Host = targetToUse.Host 252 req.URL.Path = singleJoiningSlash(targetToUse.Path, req.URL.Path, spec.Proxy.DisableStripSlash) 253 if req.URL.RawPath != "" { 254 req.URL.RawPath = singleJoiningSlash(targetToUse.Path, req.URL.RawPath, spec.Proxy.DisableStripSlash) 255 } 256 } 257 if !spec.Proxy.PreserveHostHeader { 258 req.Host = targetToUse.Host 259 } 260 if targetQuery == "" || req.URL.RawQuery == "" { 261 req.URL.RawQuery = targetQuery + req.URL.RawQuery 262 } else { 263 req.URL.RawQuery = targetQuery + "&" + req.URL.RawQuery 264 } 265 if _, ok := req.Header[headers.UserAgent]; !ok { 266 // Set Tyk's own default user agent. Without 267 // this line, we would get the net/http default. 268 req.Header.Set(headers.UserAgent, defaultUserAgent) 269 } 270 271 if spec.GlobalConfig.HttpServerOptions.SkipTargetPathEscaping { 272 // force RequestURI to skip escaping if API's proxy is set for this 273 // if we set opaque here it will force URL.RequestURI to skip escaping 274 if req.URL.RawPath != "" { 275 req.URL.Opaque = req.URL.RawPath 276 } 277 } else if req.URL.RawPath == req.URL.Path { 278 // this should force URL to do escaping 279 req.URL.RawPath = "" 280 } 281 } 282 283 proxy := &ReverseProxy{ 284 Director: director, 285 TykAPISpec: spec, 286 FlushInterval: time.Duration(spec.GlobalConfig.HttpServerOptions.FlushInterval) * time.Millisecond, 287 } 288 proxy.ErrorHandler.BaseMiddleware = BaseMiddleware{Spec: spec, Proxy: proxy} 289 return proxy 290 } 291 292 // ReverseProxy is an HTTP Handler that takes an incoming request and 293 // sends it to another server, proxying the response back to the 294 // client. 295 type ReverseProxy struct { 296 // Director must be a function which modifies 297 // the request into a new request to be sent 298 // using Transport. Its response is then copied 299 // back to the original client unmodified. 300 Director func(*http.Request) 301 302 // The transport used to perform proxy requests. 303 // If nil, http.DefaultTransport is used. 304 Transport http.RoundTripper 305 306 // FlushInterval specifies the flush interval 307 // to flush to the client while copying the 308 // response body. 309 // If zero, no periodic flushing is done. 310 FlushInterval time.Duration 311 312 // TLSClientConfig specifies the TLS configuration to use for 'wss'. 313 // If nil, the default configuration is used. 314 TLSClientConfig *tls.Config 315 316 TykAPISpec *APISpec 317 ErrorHandler ErrorHandler 318 } 319 320 func defaultTransport(dialerTimeout float64) *http.Transport { 321 timeout := 30.0 322 if dialerTimeout > 0 { 323 log.Debug("Setting timeout for outbound request to: ", dialerTimeout) 324 timeout = dialerTimeout 325 } 326 327 dialer := &net.Dialer{ 328 Timeout: time.Duration(float64(timeout) * float64(time.Second)), 329 KeepAlive: 30 * time.Second, 330 DualStack: true, 331 } 332 dialContextFunc := dialer.DialContext 333 if dnsCacheManager.IsCacheEnabled() { 334 dialContextFunc = dnsCacheManager.WrapDialer(dialer) 335 } 336 337 return &http.Transport{ 338 DialContext: dialContextFunc, 339 MaxIdleConns: config.Global().MaxIdleConns, 340 MaxIdleConnsPerHost: config.Global().MaxIdleConnsPerHost, // default is 100 341 ResponseHeaderTimeout: time.Duration(dialerTimeout) * time.Second, 342 TLSHandshakeTimeout: 10 * time.Second, 343 } 344 } 345 346 func singleJoiningSlash(a, b string, disableStripSlash bool) string { 347 if disableStripSlash && len(b) == 0 { 348 return a 349 } 350 a = strings.TrimRight(a, "/") 351 b = strings.TrimLeft(b, "/") 352 if len(b) > 0 { 353 return a + "/" + b 354 } 355 return a 356 } 357 358 func copyHeader(dst, src http.Header) { 359 if val := dst.Get(http.CanonicalHeaderKey("Access-Control-Allow-Origin")); len(val) > 0 { 360 src.Del("Access-Control-Allow-Origin") 361 } 362 363 for k, vv := range src { 364 for _, v := range vv { 365 dst.Add(k, v) 366 } 367 } 368 } 369 370 func cloneHeader(h http.Header) http.Header { 371 h2 := make(http.Header, len(h)) 372 for k, vv := range h { 373 vv2 := make([]string, len(vv)) 374 copy(vv2, vv) 375 h2[k] = vv2 376 } 377 return h2 378 } 379 380 // Hop-by-hop headers. These are removed when sent to the backend. 381 // http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html 382 var hopHeaders = []string{ 383 "Connection", 384 "Proxy-Connection", // non-standard but still sent by libcurl and rejected by e.g. google 385 "Keep-Alive", 386 "Proxy-Authenticate", 387 "Proxy-Authorization", 388 "Te", // canonicalized version of "TE" 389 "Trailer", // not Trailers per URL above; http://www.rfc-editor.org/errata_search.php?eid=4522 390 "Transfer-Encoding", 391 "Upgrade", 392 } 393 394 func (p *ReverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) *http.Response { 395 resp := p.WrappedServeHTTP(rw, req, recordDetail(req, config.Global())) 396 397 // make response body to be nopCloser and re-readable before serve it through chain of middlewares 398 nopCloseResponseBody(resp) 399 400 return resp 401 } 402 403 func (p *ReverseProxy) ServeHTTPForCache(rw http.ResponseWriter, req *http.Request) *http.Response { 404 resp := p.WrappedServeHTTP(rw, req, true) 405 406 nopCloseResponseBody(resp) 407 408 return resp 409 } 410 411 func (p *ReverseProxy) CheckHardTimeoutEnforced(spec *APISpec, req *http.Request) (bool, float64) { 412 if !spec.EnforcedTimeoutEnabled { 413 return false, spec.GlobalConfig.ProxyDefaultTimeout 414 } 415 416 _, versionPaths, _, _ := spec.Version(req) 417 found, meta := spec.CheckSpecMatchesStatus(req, versionPaths, HardTimeout) 418 if found { 419 intMeta := meta.(*float64) 420 log.Debug("HARD TIMEOUT ENFORCED: ", *intMeta) 421 return true, *intMeta 422 } 423 424 return false, spec.GlobalConfig.ProxyDefaultTimeout 425 } 426 427 func (p *ReverseProxy) CheckHeaderInRemoveList(hdr string, spec *APISpec, req *http.Request) bool { 428 vInfo, versionPaths, _, _ := spec.Version(req) 429 for _, gdKey := range vInfo.GlobalHeadersRemove { 430 if strings.ToLower(gdKey) == strings.ToLower(hdr) { 431 return true 432 } 433 } 434 435 // Check path config 436 if found, meta := spec.CheckSpecMatchesStatus(req, versionPaths, HeaderInjected); found { 437 hmeta := meta.(*apidef.HeaderInjectionMeta) 438 for _, gdKey := range hmeta.DeleteHeaders { 439 if strings.ToLower(gdKey) == strings.ToLower(hdr) { 440 return true 441 } 442 } 443 } 444 445 return false 446 } 447 448 func (p *ReverseProxy) CheckCircuitBreakerEnforced(spec *APISpec, req *http.Request) (bool, *ExtendedCircuitBreakerMeta) { 449 if !spec.CircuitBreakerEnabled { 450 return false, nil 451 } 452 453 _, versionPaths, _, _ := spec.Version(req) 454 found, meta := spec.CheckSpecMatchesStatus(req, versionPaths, CircuitBreaker) 455 if found { 456 exMeta := meta.(*ExtendedCircuitBreakerMeta) 457 log.Debug("CB Enforced for path: ", *exMeta) 458 return true, exMeta 459 } 460 461 return false, nil 462 } 463 464 func proxyFromAPI(api *APISpec) func(*http.Request) (*url.URL, error) { 465 return func(req *http.Request) (*url.URL, error) { 466 if api != nil && api.Proxy.Transport.ProxyURL != "" { 467 return url.Parse(api.Proxy.Transport.ProxyURL) 468 } 469 return http.ProxyFromEnvironment(req) 470 } 471 } 472 473 func httpTransport(timeOut float64, rw http.ResponseWriter, req *http.Request, p *ReverseProxy) http.RoundTripper { 474 transport := defaultTransport(timeOut) // modifies a newly created transport 475 transport.TLSClientConfig = &tls.Config{} 476 transport.Proxy = proxyFromAPI(p.TykAPISpec) 477 478 if config.Global().ProxySSLInsecureSkipVerify { 479 transport.TLSClientConfig.InsecureSkipVerify = true 480 } 481 482 if p.TykAPISpec.Proxy.Transport.SSLInsecureSkipVerify { 483 transport.TLSClientConfig.InsecureSkipVerify = true 484 } 485 486 // When request routed through the proxy `DialTLS` is not used, and only VerifyPeerCertificate is supported 487 // The reason behind two separate checks is that `DialTLS` supports specifying public keys per hostname, and `VerifyPeerCertificate` only global ones, e.g. `*` 488 if proxyURL, _ := transport.Proxy(req); proxyURL != nil { 489 transport.TLSClientConfig.VerifyPeerCertificate = verifyPeerCertificatePinnedCheck(p.TykAPISpec, transport.TLSClientConfig) 490 } else { 491 transport.DialTLS = dialTLSPinnedCheck(p.TykAPISpec, transport.TLSClientConfig) 492 } 493 494 if p.TykAPISpec.GlobalConfig.ProxySSLMinVersion > 0 { 495 transport.TLSClientConfig.MinVersion = p.TykAPISpec.GlobalConfig.ProxySSLMinVersion 496 } 497 498 if p.TykAPISpec.Proxy.Transport.SSLMinVersion > 0 { 499 transport.TLSClientConfig.MinVersion = p.TykAPISpec.Proxy.Transport.SSLMinVersion 500 } 501 502 if len(p.TykAPISpec.GlobalConfig.ProxySSLCipherSuites) > 0 { 503 transport.TLSClientConfig.CipherSuites = getCipherAliases(p.TykAPISpec.GlobalConfig.ProxySSLCipherSuites) 504 } 505 506 if len(p.TykAPISpec.Proxy.Transport.SSLCipherSuites) > 0 { 507 transport.TLSClientConfig.CipherSuites = getCipherAliases(p.TykAPISpec.Proxy.Transport.SSLCipherSuites) 508 } 509 510 if !config.Global().ProxySSLDisableRenegotiation { 511 transport.TLSClientConfig.Renegotiation = tls.RenegotiateFreelyAsClient 512 } 513 514 transport.DisableKeepAlives = p.TykAPISpec.GlobalConfig.ProxyCloseConnections 515 516 if IsWebsocket(req) { 517 wsTransport := &WSDialer{transport, rw, p.TLSClientConfig} 518 return wsTransport 519 } 520 521 if config.Global().ProxyEnableHttp2 { 522 http2.ConfigureTransport(transport) 523 } 524 525 return transport 526 } 527 528 func (p *ReverseProxy) WrappedServeHTTP(rw http.ResponseWriter, req *http.Request, withCache bool) *http.Response { 529 if trace.IsEnabled() { 530 span, ctx := trace.Span(req.Context(), req.URL.Path) 531 defer span.Finish() 532 ext.SpanKindRPCClient.Set(span) 533 req = req.WithContext(ctx) 534 } 535 outReqIsWebsocket := IsWebsocket(req) 536 var roundTripper http.RoundTripper 537 538 p.TykAPISpec.Lock() 539 if !outReqIsWebsocket { // check if it is a regular HTTP request 540 // create HTTP transport 541 createTransport := p.TykAPISpec.HTTPTransport == nil 542 543 // Check if timeouts are set for this endpoint 544 if !createTransport && config.Global().MaxConnTime != 0 { 545 createTransport = time.Since(p.TykAPISpec.HTTPTransportCreated) > time.Duration(config.Global().MaxConnTime)*time.Second 546 } 547 548 if createTransport { 549 _, timeout := p.CheckHardTimeoutEnforced(p.TykAPISpec, req) 550 p.TykAPISpec.HTTPTransport = httpTransport(timeout, rw, req, p) 551 p.TykAPISpec.HTTPTransportCreated = time.Now() 552 } 553 554 roundTripper = p.TykAPISpec.HTTPTransport 555 } else { // this is NEW WS-connection upgrade request 556 // create WS transport 557 createTransport := p.TykAPISpec.WSTransport == nil 558 559 // Check if timeouts are set for this endpoint 560 if !createTransport && config.Global().MaxConnTime != 0 { 561 createTransport = time.Since(p.TykAPISpec.WSTransportCreated) > time.Duration(config.Global().MaxConnTime)*time.Second 562 } 563 564 if createTransport { 565 _, timeout := p.CheckHardTimeoutEnforced(p.TykAPISpec, req) 566 p.TykAPISpec.WSTransport = httpTransport(timeout, rw, req, p) 567 p.TykAPISpec.WSTransportCreated = time.Now() 568 } 569 570 // overwrite transport's ResponseWriter from previous upgrade request 571 // as it was already hijacked and now is being used for other connection 572 p.TykAPISpec.WSTransport.(*WSDialer).RW = rw 573 574 roundTripper = p.TykAPISpec.WSTransport 575 } 576 p.TykAPISpec.Unlock() 577 578 reqCtx := req.Context() 579 if cn, ok := rw.(http.CloseNotifier); ok { 580 var cancel context.CancelFunc 581 reqCtx, cancel = context.WithCancel(reqCtx) 582 defer cancel() 583 notifyChan := cn.CloseNotify() 584 go func() { 585 select { 586 case <-notifyChan: 587 cancel() 588 case <-reqCtx.Done(): 589 } 590 }() 591 } 592 593 // Do this before we make a shallow copy 594 session := ctxGetSession(req) 595 596 outreq := new(http.Request) 597 logreq := new(http.Request) 598 599 *outreq = *req // includes shallow copies of maps, but okay 600 *logreq = *req 601 // remove context data from the copies 602 setContext(outreq, context.Background()) 603 setContext(logreq, context.Background()) 604 605 log.Debug("UPSTREAM REQUEST URL: ", req.URL) 606 607 // We need to double set the context for the outbound request to reprocess the target 608 if p.TykAPISpec.URLRewriteEnabled && req.Context().Value(ctx.RetainHost) == true { 609 log.Debug("Detected host rewrite, notifying director") 610 setCtxValue(outreq, ctx.RetainHost, true) 611 } 612 613 if req.ContentLength == 0 { 614 outreq.Body = nil // Issue 16036: nil Body for http.Transport retries 615 } 616 outreq = outreq.WithContext(reqCtx) 617 618 outreq.Header = cloneHeader(req.Header) 619 if trace.IsEnabled() { 620 span := opentracing.SpanFromContext(req.Context()) 621 trace.Inject(p.TykAPISpec.Name, span, outreq.Header) 622 } 623 p.Director(outreq) 624 outreq.Close = false 625 626 log.Debug("Outbound Request: ", outreq.URL.String()) 627 628 // Do not modify outbound request headers if they are WS 629 if !outReqIsWebsocket { 630 // Remove hop-by-hop headers listed in the "Connection" header. 631 // See RFC 2616, section 14.10. 632 if c := outreq.Header.Get("Connection"); c != "" { 633 for _, f := range strings.Split(c, ",") { 634 if f = strings.TrimSpace(f); f != "" { 635 outreq.Header.Del(f) 636 } 637 } 638 } 639 // Remove other hop-by-hop headers to the backend. Especially 640 // important is "Connection" because we want a persistent 641 // connection, regardless of what the client sent to us. 642 for _, h := range hopHeaders { 643 hv := outreq.Header.Get(h) 644 if hv == "" { 645 continue 646 } 647 if h == "Te" && hv == "trailers" { 648 continue 649 } 650 outreq.Header.Del(h) 651 logreq.Header.Del(h) 652 } 653 } 654 655 addrs := requestIPHops(req) 656 if !p.CheckHeaderInRemoveList(headers.XForwardFor, p.TykAPISpec, req) { 657 outreq.Header.Set(headers.XForwardFor, addrs) 658 } 659 660 // Circuit breaker 661 breakerEnforced, breakerConf := p.CheckCircuitBreakerEnforced(p.TykAPISpec, req) 662 663 // set up TLS certificates for upstream if needed 664 var tlsCertificates []tls.Certificate 665 if cert := getUpstreamCertificate(outreq.Host, p.TykAPISpec); cert != nil { 666 tlsCertificates = []tls.Certificate{*cert} 667 } 668 669 p.TykAPISpec.Lock() 670 if outReqIsWebsocket { 671 roundTripper.(*WSDialer).TLSClientConfig.Certificates = tlsCertificates 672 } else { 673 roundTripper.(*http.Transport).TLSClientConfig.Certificates = tlsCertificates 674 } 675 p.TykAPISpec.Unlock() 676 677 // do request round trip 678 var res *http.Response 679 var err error 680 if breakerEnforced { 681 if !breakerConf.CB.Ready() { 682 log.Debug("ON REQUEST: Circuit Breaker is in OPEN state") 683 p.ErrorHandler.HandleError(rw, logreq, "Service temporarily unavailable.", 503, true) 684 return nil 685 } 686 log.Debug("ON REQUEST: Circuit Breaker is in CLOSED or HALF-OPEN state") 687 res, err = roundTripper.RoundTrip(outreq) 688 if err != nil || res.StatusCode == http.StatusInternalServerError { 689 breakerConf.CB.Fail() 690 } else { 691 breakerConf.CB.Success() 692 } 693 } else { 694 res, err = roundTripper.RoundTrip(outreq) 695 } 696 697 if err != nil { 698 699 token := ctxGetAuthToken(req) 700 701 var alias string 702 if session != nil { 703 alias = session.Alias 704 } 705 706 log.WithFields(logrus.Fields{ 707 "prefix": "proxy", 708 "user_ip": addrs, 709 "server_name": outreq.Host, 710 "user_id": obfuscateKey(token), 711 "user_name": alias, 712 "org_id": p.TykAPISpec.OrgID, 713 "api_id": p.TykAPISpec.APIID, 714 }).Error("http: proxy error: ", err) 715 716 if strings.Contains(err.Error(), "timeout awaiting response headers") { 717 p.ErrorHandler.HandleError(rw, logreq, "Upstream service reached hard timeout.", http.StatusGatewayTimeout, true) 718 719 if p.TykAPISpec.Proxy.ServiceDiscovery.UseDiscoveryService { 720 if ServiceCache != nil { 721 log.Debug("[PROXY] [SERVICE DISCOVERY] Upstream host failed, refreshing host list") 722 ServiceCache.Delete(p.TykAPISpec.APIID) 723 } 724 } 725 return nil 726 } 727 728 if strings.Contains(err.Error(), "context canceled") { 729 p.ErrorHandler.HandleError(rw, logreq, "Client closed request", 499, true) 730 return nil 731 } 732 733 if strings.Contains(err.Error(), "no such host") { 734 p.ErrorHandler.HandleError(rw, logreq, "Upstream host lookup failed", http.StatusInternalServerError, true) 735 return nil 736 } 737 738 p.ErrorHandler.HandleError(rw, logreq, "There was a problem proxying the request", http.StatusInternalServerError, true) 739 return nil 740 741 } 742 743 if IsWebsocket(req) { 744 return nil 745 } 746 747 ses := new(user.SessionState) 748 if session != nil { 749 ses = session 750 } 751 752 // Middleware chain handling here - very simple, but should do 753 // the trick. Chain can be empty, in which case this is a no-op. 754 if err := handleResponseChain(p.TykAPISpec.ResponseChain, rw, res, req, ses); err != nil { 755 log.Error("Response chain failed! ", err) 756 } 757 758 inres := new(http.Response) 759 if withCache { 760 *inres = *res // includes shallow copies of maps, but okay 761 762 defer res.Body.Close() 763 764 // Buffer body data 765 var bodyBuffer bytes.Buffer 766 bodyBuffer2 := new(bytes.Buffer) 767 768 p.CopyResponse(&bodyBuffer, res.Body) 769 *bodyBuffer2 = bodyBuffer 770 771 // Create new ReadClosers so we can split output 772 res.Body = ioutil.NopCloser(&bodyBuffer) 773 inres.Body = ioutil.NopCloser(bodyBuffer2) 774 } 775 776 // We should at least copy the status code in 777 inres.StatusCode = res.StatusCode 778 inres.ContentLength = res.ContentLength 779 p.HandleResponse(rw, res, ses) 780 return inres 781 } 782 783 func (p *ReverseProxy) HandleResponse(rw http.ResponseWriter, res *http.Response, ses *user.SessionState) error { 784 785 // Remove hop-by-hop headers listed in the 786 // "Connection" header of the response. 787 if c := res.Header.Get(headers.Connection); c != "" { 788 for _, f := range strings.Split(c, ",") { 789 if f = strings.TrimSpace(f); f != "" { 790 res.Header.Del(f) 791 } 792 } 793 } 794 795 for _, h := range hopHeaders { 796 res.Header.Del(h) 797 } 798 defer res.Body.Close() 799 800 // Close connections 801 if config.Global().CloseConnections { 802 res.Header.Set(headers.Connection, "close") 803 } 804 805 // Add resource headers 806 if ses != nil { 807 // We have found a session, lets report back 808 quotaMax, quotaRemaining, _, quotaRenews := ses.GetQuotaLimitByAPIID(p.TykAPISpec.APIID) 809 res.Header.Set(XRateLimitLimit, strconv.Itoa(int(quotaMax))) 810 res.Header.Set(XRateLimitRemaining, strconv.Itoa(int(quotaRemaining))) 811 res.Header.Set(XRateLimitReset, strconv.Itoa(int(quotaRenews))) 812 } 813 814 copyHeader(rw.Header(), res.Header) 815 816 announcedTrailers := len(res.Trailer) 817 if announcedTrailers > 0 { 818 trailerKeys := make([]string, 0, len(res.Trailer)) 819 for k := range res.Trailer { 820 trailerKeys = append(trailerKeys, k) 821 } 822 rw.Header().Add("Trailer", strings.Join(trailerKeys, ", ")) 823 } 824 825 rw.WriteHeader(res.StatusCode) 826 827 if len(res.Trailer) > 0 { 828 // Force chunking if we saw a response trailer. 829 // This prevents net/http from calculating the length for short 830 // bodies and adding a Content-Length. 831 if fl, ok := rw.(http.Flusher); ok { 832 fl.Flush() 833 } 834 } 835 836 p.CopyResponse(rw, res.Body) 837 838 if len(res.Trailer) == announcedTrailers { 839 copyHeader(rw.Header(), res.Trailer) 840 return nil 841 } 842 843 for k, vv := range res.Trailer { 844 k = http.TrailerPrefix + k 845 for _, v := range vv { 846 rw.Header().Add(k, v) 847 } 848 } 849 return nil 850 } 851 852 func (p *ReverseProxy) CopyResponse(dst io.Writer, src io.Reader) { 853 if p.FlushInterval != 0 { 854 if wf, ok := dst.(writeFlusher); ok { 855 mlw := &maxLatencyWriter{ 856 dst: wf, 857 latency: p.FlushInterval, 858 done: make(chan bool), 859 } 860 go mlw.flushLoop() 861 defer mlw.stop() 862 dst = mlw 863 } 864 } 865 866 p.copyBuffer(dst, src, nil) 867 } 868 869 func (p *ReverseProxy) copyBuffer(dst io.Writer, src io.Reader, buf []byte) (int64, error) { 870 if len(buf) == 0 { 871 buf = make([]byte, 32*1024) 872 } 873 var written int64 874 for { 875 nr, rerr := src.Read(buf) 876 if rerr != nil && rerr != io.EOF && rerr != context.Canceled { 877 log.WithFields(logrus.Fields{ 878 "prefix": "proxy", 879 "org_id": p.TykAPISpec.OrgID, 880 "api_id": p.TykAPISpec.APIID, 881 }).Error("http: proxy error during body copy: ", rerr) 882 } 883 if nr > 0 { 884 nw, werr := dst.Write(buf[:nr]) 885 if nw > 0 { 886 written += int64(nw) 887 } 888 if werr != nil { 889 return written, werr 890 } 891 if nr != nw { 892 return written, io.ErrShortWrite 893 } 894 } 895 if rerr != nil { 896 return written, rerr 897 } 898 } 899 } 900 901 type writeFlusher interface { 902 io.Writer 903 http.Flusher 904 } 905 906 type maxLatencyWriter struct { 907 dst writeFlusher 908 latency time.Duration 909 910 mu sync.Mutex // protects Write + Flush 911 done chan bool 912 } 913 914 func (m *maxLatencyWriter) Write(p []byte) (int, error) { 915 m.mu.Lock() 916 defer m.mu.Unlock() 917 return m.dst.Write(p) 918 } 919 920 func (m *maxLatencyWriter) flushLoop() { 921 t := time.NewTicker(m.latency) 922 defer t.Stop() 923 for { 924 select { 925 case <-m.done: 926 return 927 case <-t.C: 928 m.mu.Lock() 929 m.dst.Flush() 930 m.mu.Unlock() 931 } 932 } 933 } 934 935 func (m *maxLatencyWriter) stop() { m.done <- true } 936 937 func requestIPHops(r *http.Request) string { 938 clientIP, _, err := net.SplitHostPort(r.RemoteAddr) 939 if err != nil { 940 return "" 941 } 942 // If we aren't the first proxy retain prior 943 // X-Forwarded-For information as a comma+space 944 // separated list and fold multiple headers into one. 945 if prior, ok := r.Header["X-Forwarded-For"]; ok { 946 clientIP = strings.Join(prior, ", ") + ", " + clientIP 947 } 948 return clientIP 949 } 950 951 // nopCloser is just like ioutil's, but here to let us re-read the same 952 // buffer inside by moving position to the start every time we done with reading 953 type nopCloser struct { 954 io.ReadSeeker 955 } 956 957 // Read just a wrapper around real Read which also moves position to the start if we get EOF 958 // to have it ready for next read-cycle 959 func (n nopCloser) Read(p []byte) (int, error) { 960 num, err := n.ReadSeeker.Read(p) 961 if err == io.EOF { // move to start to have it ready for next read cycle 962 n.Seek(0, io.SeekStart) 963 } 964 return num, err 965 } 966 967 // Close is a no-op Close 968 func (n nopCloser) Close() error { 969 return nil 970 } 971 972 func copyBody(body io.ReadCloser) io.ReadCloser { 973 // check if body was already read and converted into our nopCloser 974 if nc, ok := body.(nopCloser); ok { 975 // seek to the beginning to have it ready for next read 976 nc.Seek(0, io.SeekStart) 977 return body 978 } 979 980 // body is http's io.ReadCloser - let's close it after we read data 981 defer body.Close() 982 983 // body is http's io.ReadCloser - read it up until EOF 984 var bodyRead bytes.Buffer 985 io.Copy(&bodyRead, body) 986 987 // use seek-able reader for further body usage 988 reusableBody := bytes.NewReader(bodyRead.Bytes()) 989 990 return nopCloser{reusableBody} 991 } 992 993 func copyRequest(r *http.Request) *http.Request { 994 if r.Body != nil { 995 r.Body = copyBody(r.Body) 996 } 997 return r 998 } 999 1000 func copyResponse(r *http.Response) *http.Response { 1001 if r.Body != nil { 1002 r.Body = copyBody(r.Body) 1003 } 1004 return r 1005 } 1006 1007 func nopCloseRequestBody(r *http.Request) { 1008 if r == nil { 1009 return 1010 } 1011 1012 copyRequest(r) 1013 } 1014 1015 func nopCloseResponseBody(r *http.Response) { 1016 if r == nil { 1017 return 1018 } 1019 1020 copyResponse(r) 1021 }