github.com/avenga/couper@v1.12.2/server/http_test.go (about)

     1  package server_test
     2  
     3  import (
     4  	"bytes"
     5  	"compress/gzip"
     6  	"context"
     7  	"encoding/json"
     8  	"fmt"
     9  	"io"
    10  	"net"
    11  	"net/http"
    12  	"net/http/httptest"
    13  	"net/url"
    14  	"os"
    15  	"path"
    16  	"regexp"
    17  	"sort"
    18  	"strconv"
    19  	"strings"
    20  	"sync"
    21  	"testing"
    22  	"text/template"
    23  	"time"
    24  
    25  	"github.com/google/go-cmp/cmp"
    26  	"github.com/sirupsen/logrus"
    27  	logrustest "github.com/sirupsen/logrus/hooks/test"
    28  
    29  	"github.com/avenga/couper/cache"
    30  	"github.com/avenga/couper/config/configload"
    31  	"github.com/avenga/couper/config/runtime"
    32  	"github.com/avenga/couper/internal/test"
    33  	"github.com/avenga/couper/logging"
    34  	"github.com/avenga/couper/server"
    35  )
    36  
    37  func TestHTTPServer_ServeHTTP_Files(t *testing.T) {
    38  	helper := test.New(t)
    39  
    40  	currentDir, err := os.Getwd()
    41  	helper.Must(err)
    42  	defer helper.Must(os.Chdir(currentDir))
    43  
    44  	expectedAPIHost := "test.couper.io"
    45  	originBackend := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
    46  		if req.Host != expectedAPIHost {
    47  			rw.WriteHeader(http.StatusBadRequest)
    48  			return
    49  		}
    50  		rw.WriteHeader(http.StatusNoContent)
    51  	}))
    52  	defer originBackend.Close()
    53  
    54  	helper.Must(os.Chdir("testdata/file_serving"))
    55  
    56  	tpl, err := template.ParseFiles("conf_test.hcl")
    57  	helper.Must(err)
    58  
    59  	confBytes := &bytes.Buffer{}
    60  	err = tpl.Execute(confBytes, map[string]string{
    61  		"origin":   "http://" + originBackend.Listener.Addr().String(),
    62  		"hostname": expectedAPIHost,
    63  	})
    64  	helper.Must(err)
    65  
    66  	log, _ := logrustest.NewNullLogger()
    67  	//log.Out = os.Stdout
    68  
    69  	ctx, cancel := context.WithCancel(context.Background())
    70  	defer cancel()
    71  
    72  	conf, err := configload.LoadBytes(confBytes.Bytes(), "conf_test.hcl")
    73  	helper.Must(err)
    74  	conf.Settings.DefaultPort = 0
    75  
    76  	tmpStoreCh := make(chan struct{})
    77  	defer close(tmpStoreCh)
    78  
    79  	logger := log.WithContext(context.TODO())
    80  	tmpMemStore := cache.New(logger, tmpStoreCh)
    81  
    82  	confCTX, confCancel := context.WithCancel(conf.Context)
    83  	conf.Context = confCTX
    84  	defer confCancel()
    85  
    86  	srvConf, err := runtime.NewServerConfiguration(conf, logger, tmpMemStore)
    87  	helper.Must(err)
    88  
    89  	spaContent, err := os.ReadFile(conf.Servers[0].SPAs[0].BootstrapFile)
    90  	helper.Must(err)
    91  
    92  	port := runtime.Port(conf.Settings.DefaultPort)
    93  	gw, err := server.New(ctx, conf.Context, log.WithContext(ctx), conf.Settings, &runtime.DefaultTimings, port, srvConf[port])
    94  	helper.Must(err)
    95  
    96  	gw.Listen()
    97  	defer gw.Close()
    98  
    99  	connectClient := http.Client{Transport: &http.Transport{
   100  		DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
   101  			return net.Dial("tcp4", gw.Addr())
   102  		},
   103  		DisableCompression: true,
   104  	}}
   105  
   106  	for i, testCase := range []struct {
   107  		path           string
   108  		expectedBody   []byte
   109  		expectedStatus int
   110  	}{
   111  		{"/", []byte("<html><body><h1>route not found error: My custom error template</h1></body></html>"), http.StatusNotFound},
   112  		{"/apps/", []byte("<html><body><h1>route not found error: My custom error template</h1></body></html>"), http.StatusNotFound},
   113  		{"/apps/shiny-product/", []byte("<html><body><h1>route not found error: My custom error template</h1></body></html>"), http.StatusNotFound},
   114  		{"/apps/shiny-product/assets/", []byte("<html><body><h1>route not found error: My custom error template</h1></body></html>"), http.StatusNotFound},
   115  		{"/apps/shiny-product/app/", spaContent, http.StatusOK},
   116  		{"/apps/shiny-product/app/sub", spaContent, http.StatusOK},
   117  		{"/apps/shiny-product/api/", nil, http.StatusNoContent},
   118  		{"/apps/shiny-product/api/foo%20bar:%22baz%22", []byte(`{"message": "route not found error" }`), 404},
   119  	} {
   120  		req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://example.com:%s%s", port, testCase.path), nil)
   121  		helper.Must(err)
   122  
   123  		res, err := connectClient.Do(req)
   124  		helper.Must(err)
   125  
   126  		if res.StatusCode != testCase.expectedStatus {
   127  			t.Errorf("%.2d: expected status %d, got %d", i+1, testCase.expectedStatus, res.StatusCode)
   128  		}
   129  
   130  		result, err := io.ReadAll(res.Body)
   131  		helper.Must(err)
   132  		helper.Must(res.Body.Close())
   133  
   134  		if !bytes.Contains(result, testCase.expectedBody) {
   135  			t.Errorf("%.2d: expected body should contain:\n%s\ngot:\n%s", i+1, string(testCase.expectedBody), string(result))
   136  		}
   137  	}
   138  
   139  	helper.Must(os.Chdir(currentDir)) // defer for error cases, would be to late for normal exit
   140  }
   141  
   142  func TestHTTPServer_ServeHTTP_Files2(t *testing.T) {
   143  	helper := test.New(t)
   144  
   145  	currentDir, err := os.Getwd()
   146  	helper.Must(err)
   147  	defer helper.Must(os.Chdir(currentDir))
   148  
   149  	expectedAPIHost := "test.couper.io"
   150  	originBackend := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
   151  		if req.Host != expectedAPIHost {
   152  			rw.WriteHeader(http.StatusBadRequest)
   153  			return
   154  		}
   155  		rw.WriteHeader(http.StatusOK)
   156  		rw.Write([]byte(req.URL.Path))
   157  	}))
   158  	defer originBackend.Close()
   159  
   160  	helper.Must(os.Chdir("testdata/file_serving"))
   161  
   162  	tpl, err := template.ParseFiles("conf_fileserving.hcl")
   163  	helper.Must(err)
   164  
   165  	confBytes := &bytes.Buffer{}
   166  	err = tpl.Execute(confBytes, map[string]string{
   167  		"origin": "http://" + originBackend.Listener.Addr().String(),
   168  	})
   169  	helper.Must(err)
   170  
   171  	log, _ := logrustest.NewNullLogger()
   172  	//log.Out = os.Stdout
   173  
   174  	ctx, cancel := context.WithCancel(context.Background())
   175  	defer cancel()
   176  
   177  	conf, err := configload.LoadBytes(confBytes.Bytes(), "conf_fileserving.hcl")
   178  	helper.Must(err)
   179  
   180  	error404Content := []byte("<html><body><h1>route not found error: My custom error template</h1></body></html>")
   181  	spaContent := []byte("<html><body><h1>vue.js</h1></body></html>")
   182  
   183  	tmpStoreCh := make(chan struct{})
   184  	defer close(tmpStoreCh)
   185  
   186  	logger := log.WithContext(context.TODO())
   187  	tmpMemStore := cache.New(logger, tmpStoreCh)
   188  
   189  	confCTX, confCancel := context.WithCancel(conf.Context)
   190  	conf.Context = confCTX
   191  	defer confCancel()
   192  
   193  	srvConf, err := runtime.NewServerConfiguration(conf, logger, tmpMemStore)
   194  	helper.Must(err)
   195  
   196  	couper, err := server.New(ctx, conf.Context, log.WithContext(ctx), conf.Settings, &runtime.DefaultTimings, runtime.Port(0), srvConf[0])
   197  	helper.Must(err)
   198  
   199  	couper.Listen()
   200  	defer couper.Close()
   201  
   202  	connectClient := http.Client{
   203  		Transport: &http.Transport{
   204  			DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
   205  				return net.Dial("tcp4", couper.Addr())
   206  			},
   207  			DisableCompression: true,
   208  		},
   209  		CheckRedirect: func(req *http.Request, via []*http.Request) error {
   210  			return http.ErrUseLastResponse
   211  		},
   212  	}
   213  
   214  	for i, testCase := range []struct {
   215  		path           string
   216  		expectedBody   []byte
   217  		expectedStatus int
   218  	}{
   219  		// spa path /
   220  		{"/", spaContent, 200},
   221  		// 404 check that spa /dir/** rule doesn't match here
   222  		{"/dirdoesnotexist", error404Content, 404},
   223  		{"/dir:", error404Content, 404},
   224  		{"/dir.txt", error404Content, 404},
   225  		// dir w/ index in files
   226  		{"/another_dir", nil, http.StatusFound},
   227  		{"/another_dir/", []byte("<html>this is another_dir/index.html</html>\n"), http.StatusOK},
   228  		// dir w/ index in files but also a spa mount path
   229  		{"/dir", spaContent, http.StatusOK},
   230  		// dir/ w/ index in files
   231  		{"/dir/", spaContent, 200},
   232  		// dir w/o index in files
   233  		{"/assets/noindex", error404Content, 404},
   234  		{"/assets/noindex/", error404Content, 404},
   235  		{"/assets/noindex/file.txt", []byte("foo\n"), 200},
   236  		// dir w/o index in spa
   237  		{"/dir/noindex", spaContent, 200},
   238  		// file > spa
   239  		{"/dir/noindex/otherfile.txt", []byte("bar\n"), 200},
   240  		{"/robots.txt", []byte("Disallow: /secret\n"), 200},
   241  		{"/foo bar.txt", []byte("foo-and-bar\n"), 200},
   242  		{"/foo%20bar.txt", []byte("foo-and-bar\n"), 200},
   243  		{"/favicon.ico", error404Content, 404},
   244  		{"/app", spaContent, 200},
   245  		{"/app/", spaContent, 200},
   246  		{"/app/bla", spaContent, 200},
   247  		{"/app/bla/foo", spaContent, 200},
   248  		{"/api/foo/bar", []byte("/bar"), 200},
   249  		// spa > file
   250  		{"/my_app", []byte(`<html><body><h1>{"framework":"react.js"}</h1></body></html>`), http.StatusOK},
   251  		{"/my_app/spa.html", []byte(`<html><body><h1>{"framework":"react.js"}</h1></body></html>`), http.StatusOK},
   252  	} {
   253  		req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s%s", couper.Addr(), testCase.path), nil)
   254  		helper.Must(err)
   255  		req.Host = "example.com"
   256  
   257  		res, err := connectClient.Do(req)
   258  		helper.Must(err)
   259  
   260  		if res.StatusCode != testCase.expectedStatus {
   261  			t.Errorf("%.2d: expected status for path %q %d, got %d", i+1, testCase.path, testCase.expectedStatus, res.StatusCode)
   262  		}
   263  
   264  		result, err := io.ReadAll(res.Body)
   265  		helper.Must(err)
   266  		helper.Must(res.Body.Close())
   267  
   268  		if !bytes.Contains(result, testCase.expectedBody) {
   269  			t.Errorf("%.2d: expected body for path %q:\n%s\ngot:\n%s", i+1, testCase.path, string(testCase.expectedBody), string(result))
   270  		}
   271  	}
   272  	helper.Must(os.Chdir(currentDir)) // defer for error cases, would be to late for normal exit
   273  }
   274  
   275  func TestHTTPServer_UUID_Common(t *testing.T) {
   276  	helper := test.New(t)
   277  	client := newClient()
   278  
   279  	confPath := "testdata/settings/02_couper.hcl"
   280  	shutdown, logHook := newCouper(confPath, test.New(t))
   281  	defer shutdown()
   282  
   283  	logHook.Reset()
   284  	req, err := http.NewRequest(http.MethodGet, "http://anyserver:8080/", nil)
   285  	helper.Must(err)
   286  
   287  	_, err = client.Do(req)
   288  	helper.Must(err)
   289  
   290  	// Wait for log
   291  	time.Sleep(300 * time.Millisecond)
   292  
   293  	e := logHook.LastEntry()
   294  	if e == nil {
   295  		t.Fatalf("Missing log line")
   296  	}
   297  
   298  	regexCheck := regexp.MustCompile(`^[0-9a-v]{20}$`)
   299  	if !regexCheck.MatchString(e.Data["uid"].(string)) {
   300  		t.Errorf("Expected a common uid format, got %#v", e.Data["uid"])
   301  	}
   302  }
   303  
   304  func TestHTTPServer_UUID_uuid4(t *testing.T) {
   305  	helper := test.New(t)
   306  	client := newClient()
   307  
   308  	confPath := "testdata/settings/03_couper.hcl"
   309  	shutdown, logHook := newCouper(confPath, test.New(t))
   310  	defer shutdown()
   311  
   312  	logHook.Reset()
   313  	req, err := http.NewRequest(http.MethodGet, "http://anyserver:8080/", nil)
   314  	helper.Must(err)
   315  
   316  	_, err = client.Do(req)
   317  	helper.Must(err)
   318  
   319  	// Wait for log
   320  	time.Sleep(300 * time.Millisecond)
   321  
   322  	e := logHook.LastEntry()
   323  	if e == nil {
   324  		t.Fatalf("Missing log line")
   325  	}
   326  
   327  	regexCheck := regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`)
   328  	if !regexCheck.MatchString(e.Data["uid"].(string)) {
   329  		t.Errorf("Expected a uuid4 uid format, got %#v", e.Data["uid"])
   330  	}
   331  }
   332  
   333  func TestHTTPServer_ServeProxyAbortHandler(t *testing.T) {
   334  	configFile := `
   335  server "zipzip" {
   336  	endpoint "/**" {
   337  		proxy {
   338  			backend {
   339  				origin = "%s"
   340  				set_response_headers = {
   341     					resp = json_encode(backend_responses.default)
   342  				}
   343  			}
   344  		}
   345  	}
   346  }
   347  `
   348  	helper := test.New(t)
   349  
   350  	origin := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
   351  		rw.Header().Set("Content-Encoding", "gzip")
   352  		gzw := gzip.NewWriter(rw)
   353  		defer func() {
   354  			if r.Header.Get("x-close") != "" {
   355  				return // triggers reverseproxy copyBuffer panic due to missing gzip footer
   356  			}
   357  			if e := gzw.Close(); e != nil {
   358  				t.Error(e)
   359  			}
   360  		}()
   361  
   362  		_, err := gzw.Write([]byte(configFile))
   363  		helper.Must(err)
   364  
   365  		err = gzw.Flush() // explicit flush, just the gzip footer is missing
   366  		helper.Must(err)
   367  	}))
   368  	defer origin.Close()
   369  
   370  	shutdown, loghook, err := newCouperWithBytes([]byte(fmt.Sprintf(configFile, origin.URL)), helper)
   371  	helper.Must(err)
   372  	defer shutdown()
   373  
   374  	req, err := http.NewRequest(http.MethodGet, "http://localhost:8080", nil)
   375  	helper.Must(err)
   376  
   377  	res, err := newClient().Do(req)
   378  	helper.Must(err)
   379  
   380  	if res.StatusCode != http.StatusOK {
   381  		t.Errorf("Expected OK, got: %s", res.Status)
   382  		for _, entry := range loghook.AllEntries() {
   383  			t.Log(entry.String())
   384  		}
   385  	}
   386  
   387  	b, err := io.ReadAll(res.Body)
   388  	helper.Must(err)
   389  	helper.Must(res.Body.Close())
   390  
   391  	if string(b) != configFile {
   392  		t.Error("Expected same content")
   393  	}
   394  
   395  	loghook.Reset()
   396  
   397  	// Trigger panic
   398  	req.Header.Set("x-close", "dont")
   399  	_, err = newClient().Do(req)
   400  	helper.Must(err)
   401  
   402  	for _, entry := range loghook.AllEntries() {
   403  		if entry.Level != logrus.ErrorLevel {
   404  			continue
   405  		}
   406  		if strings.HasPrefix(entry.Message, "internal server error: body copy failed") {
   407  			return
   408  		}
   409  	}
   410  	t.Errorf("expected 'body copy failed' log error")
   411  }
   412  
   413  func TestHTTPServer_ServePipedGzip(t *testing.T) {
   414  	configFile := `
   415  server "zipzip" {
   416  	endpoint "/**" {
   417  		proxy {
   418  			backend {
   419  				origin = "%s"
   420  				%s
   421  			}
   422  		}
   423  	}
   424  }
   425  `
   426  	helper := test.New(t)
   427  
   428  	rawPayload, err := os.ReadFile("http.go")
   429  	helper.Must(err)
   430  
   431  	w := &bytes.Buffer{}
   432  	zw, err := gzip.NewWriterLevel(w, gzip.BestCompression)
   433  	helper.Must(err)
   434  
   435  	_, err = zw.Write(rawPayload)
   436  	helper.Must(err)
   437  	helper.Must(zw.Close())
   438  
   439  	compressedPayload := w.Bytes()
   440  
   441  	origin := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
   442  		if r.Header.Get("Accept-Encoding") == "gzip" {
   443  			rw.Header().Set("Content-Encoding", "gzip")
   444  			rw.Header().Set("Content-Length", strconv.Itoa(len(compressedPayload)))
   445  			_, err = rw.Write(compressedPayload)
   446  			helper.Must(err)
   447  			return
   448  		}
   449  		rw.Header().Set("Content-Type", "application/json")
   450  		_, err = rw.Write(rawPayload)
   451  		helper.Must(err)
   452  	}))
   453  	defer origin.Close()
   454  
   455  	for _, testcase := range []struct {
   456  		name           string
   457  		acceptEncoding string
   458  		attributes     string
   459  	}{
   460  		{"piped gzip bytes", "gzip", ""},
   461  		{"read gzip bytes", "", `set_response_headers = {
   462     					resp = json_encode(backend_responses.default.json_body)
   463  				}`},
   464  		{"read and write gzip bytes", "gzip", `set_response_headers = {
   465     					resp = json_encode(backend_responses.default.json_body)
   466  				}`},
   467  	} {
   468  		t.Run(testcase.name, func(st *testing.T) {
   469  			h := test.New(st)
   470  			shutdown, _, err := newCouperWithBytes([]byte(fmt.Sprintf(configFile, origin.URL, testcase.attributes)), h)
   471  			h.Must(err)
   472  			defer shutdown()
   473  
   474  			req, err := http.NewRequest(http.MethodGet, "http://localhost:8080", nil)
   475  			h.Must(err)
   476  			req.Header.Set("Accept-Encoding", testcase.acceptEncoding)
   477  
   478  			res, err := test.NewHTTPClient().Do(req)
   479  			h.Must(err)
   480  
   481  			if res.StatusCode != http.StatusOK {
   482  				st.Errorf("Expected OK, got: %s", res.Status)
   483  				return
   484  			}
   485  
   486  			b, err := io.ReadAll(res.Body)
   487  			h.Must(err)
   488  			h.Must(res.Body.Close())
   489  
   490  			if testcase.acceptEncoding == "gzip" {
   491  				if testcase.attributes == "" && !bytes.Equal(b, compressedPayload) {
   492  					st.Errorf("Expected same content with best compression level, want %d bytes, got %d bytes", len(b), len(compressedPayload))
   493  				}
   494  				if testcase.attributes != "" {
   495  					if bytes.Equal(b, compressedPayload) {
   496  						st.Errorf("Expected different bytes due to compression level")
   497  					}
   498  
   499  					gr, err := gzip.NewReader(bytes.NewReader(b))
   500  					h.Must(err)
   501  					result, err := io.ReadAll(gr)
   502  					h.Must(err)
   503  					if !bytes.Equal(result, rawPayload) {
   504  						st.Error("Expected same (raw) content")
   505  					}
   506  				}
   507  
   508  			} else if testcase.acceptEncoding == "" && !bytes.Equal(b, rawPayload) {
   509  				st.Error("Expected same (raw) content")
   510  			}
   511  		})
   512  	}
   513  }
   514  
   515  func TestHTTPServer_Errors(t *testing.T) {
   516  	helper := test.New(t)
   517  	client := newClient()
   518  
   519  	confPath := "testdata/settings/03_couper.hcl"
   520  	shutdown, logHook := newCouper(confPath, test.New(t))
   521  	defer shutdown()
   522  
   523  	logHook.Reset()
   524  	req, err := http.NewRequest(http.MethodGet, "http://anyserver:8080/", nil)
   525  	helper.Must(err)
   526  
   527  	req.Host = "foo::"
   528  	_, err = client.Do(req)
   529  	helper.Must(err)
   530  
   531  	// Wait for log
   532  	time.Sleep(300 * time.Millisecond)
   533  
   534  	e := logHook.LastEntry()
   535  	if e == nil {
   536  		t.Fatalf("Missing log line")
   537  	}
   538  }
   539  
   540  func TestHTTPServer_RequestID(t *testing.T) {
   541  	client := newClient()
   542  
   543  	const (
   544  		confPath = "testdata/settings/"
   545  		validUID = "0123456789-abc+DEF=@/-"
   546  	)
   547  
   548  	type expectation struct {
   549  		Headers http.Header
   550  	}
   551  
   552  	type testCase struct {
   553  		file         string
   554  		uid          string
   555  		status       int
   556  		expToClient  expectation
   557  		expToBackend expectation
   558  	}
   559  
   560  	for i, tc := range []testCase{
   561  		{"07_couper.hcl", "", http.StatusOK,
   562  			expectation{
   563  				Headers: http.Header{
   564  					"Couper-Client-Request-Id": []string{"{{system-id}}"},
   565  				},
   566  			},
   567  			expectation{
   568  				Headers: http.Header{
   569  					"Couper-Backend-Request-Id": []string{"{{system-id}}"},
   570  				},
   571  			},
   572  		},
   573  		{"07_couper.hcl", "XXX", http.StatusBadRequest,
   574  			expectation{
   575  				Headers: http.Header{
   576  					"Couper-Client-Request-Id": []string{"{{system-id}}"},
   577  					"Couper-Error":             []string{"client request error"},
   578  				},
   579  			},
   580  			expectation{},
   581  		},
   582  		{"07_couper.hcl", validUID, http.StatusOK,
   583  			expectation{
   584  				Headers: http.Header{
   585  					"Couper-Client-Request-Id": []string{validUID},
   586  				},
   587  			},
   588  			expectation{
   589  				Headers: http.Header{
   590  					"Client-Request-Id":         []string{validUID},
   591  					"Couper-Backend-Request-Id": []string{validUID},
   592  				},
   593  			},
   594  		},
   595  		{"08_couper.hcl", validUID, http.StatusOK,
   596  			expectation{
   597  				Headers: http.Header{
   598  					"Couper-Request-Id": []string{validUID},
   599  				},
   600  			},
   601  			expectation{
   602  				Headers: http.Header{
   603  					"Client-Request-Id":   []string{validUID},
   604  					"Couper-Request-Id":   []string{validUID},
   605  					"Request-Id-From-Var": []string{validUID},
   606  				},
   607  			},
   608  		},
   609  		{"08_couper.hcl", "", http.StatusOK,
   610  			expectation{
   611  				Headers: http.Header{
   612  					"Couper-Request-Id": []string{"{{system-id}}"},
   613  				},
   614  			},
   615  			expectation{
   616  				Headers: http.Header{
   617  					"Couper-Request-Id":   []string{"{{system-id}}"},
   618  					"Request-Id-From-Var": []string{"{{system-id}}"},
   619  				},
   620  			},
   621  		},
   622  		{"09_couper.hcl", validUID, http.StatusOK,
   623  			expectation{
   624  				Headers: http.Header{},
   625  			},
   626  			expectation{
   627  				Headers: http.Header{
   628  					"Client-Request-Id":   []string{validUID},
   629  					"Request-ID-From-Var": []string{validUID},
   630  				},
   631  			},
   632  		},
   633  	} {
   634  		t.Run("_"+tc.file, func(subT *testing.T) {
   635  			helper := test.New(subT)
   636  			shutdown, hook := newCouper(path.Join(confPath, tc.file), helper)
   637  			defer shutdown()
   638  
   639  			req, err := http.NewRequest(http.MethodGet, "http://example.com:8080", nil)
   640  			helper.Must(err)
   641  
   642  			if tc.uid != "" {
   643  				req.Header.Set("Client-Request-ID", tc.uid)
   644  			}
   645  
   646  			test.WaitForOpenPort(8080)
   647  
   648  			hook.Reset()
   649  			res, err := client.Do(req)
   650  			helper.Must(err)
   651  
   652  			// Wait for log
   653  			time.Sleep(750 * time.Millisecond)
   654  
   655  			lastLog := hook.LastEntry()
   656  
   657  			getHeaderValue := func(header http.Header, name string) string {
   658  				if lastLog == nil {
   659  					return ""
   660  				}
   661  				return strings.Replace(
   662  					header.Get(name),
   663  					"{{system-id}}",
   664  					lastLog.Data["uid"].(string),
   665  					-1,
   666  				)
   667  			}
   668  
   669  			if tc.status != res.StatusCode {
   670  				subT.Errorf("Unexpected status code given: %d", res.StatusCode)
   671  				return
   672  			}
   673  
   674  			if tc.status == http.StatusOK {
   675  				if lastLog != nil && lastLog.Message != "" {
   676  					subT.Errorf("Unexpected log message given: %s", lastLog.Message)
   677  				}
   678  
   679  				for k := range tc.expToClient.Headers {
   680  					v := getHeaderValue(tc.expToClient.Headers, k)
   681  
   682  					if v != res.Header.Get(k) {
   683  						subT.Errorf("%d: Unexpected response header %q sent: %s, want: %q", i, k, res.Header.Get(k), v)
   684  					}
   685  				}
   686  
   687  				body, err := io.ReadAll(res.Body)
   688  				helper.Must(err)
   689  				helper.Must(res.Body.Close())
   690  
   691  				var jsonResult expectation
   692  				err = json.Unmarshal(body, &jsonResult)
   693  				if err != nil {
   694  					subT.Errorf("unmarshal json: %v: got:\n%s", err, string(body))
   695  				}
   696  
   697  				for k := range tc.expToBackend.Headers {
   698  					v := getHeaderValue(tc.expToBackend.Headers, k)
   699  
   700  					if v != jsonResult.Headers.Get(k) {
   701  						subT.Errorf("%d: Unexpected header %q sent to backend: %q, want: %q", i, k, jsonResult.Headers.Get(k), v)
   702  					}
   703  				}
   704  			} else {
   705  				exp := fmt.Sprintf("client request error: invalid request-id header value: Client-Request-ID: %s", tc.uid)
   706  				if lastLog == nil {
   707  					subT.Errorf("Missing log line")
   708  				} else if lastLog.Message != exp {
   709  					subT.Errorf("\nWant:\t%s\nGot:\t%s", exp, lastLog.Message)
   710  				}
   711  
   712  				for k := range tc.expToClient.Headers {
   713  					v := getHeaderValue(tc.expToClient.Headers, k)
   714  
   715  					if v != res.Header.Get(k) {
   716  						subT.Errorf("Unexpected response header %q: %q, want: %q", k, res.Header.Get(k), v)
   717  					}
   718  				}
   719  			}
   720  		})
   721  	}
   722  }
   723  
   724  func TestHTTPServer_parseDuration(t *testing.T) {
   725  	helper := test.New(t)
   726  	client := newClient()
   727  
   728  	shutdown, logHook := newCouper("testdata/integration/config/16_couper.hcl", test.New(t))
   729  	defer shutdown()
   730  
   731  	logHook.Reset()
   732  	req, err := http.NewRequest(http.MethodGet, "http://anyserver:8080/", nil)
   733  	helper.Must(err)
   734  
   735  	_, err = client.Do(req)
   736  	helper.Must(err)
   737  
   738  	logs := logHook.AllEntries()
   739  
   740  	if logs[0].Message != `using default timing of 0s because an error occurred: timeout: time: invalid duration "xxx"` {
   741  		t.Errorf("%#v", logs[0].Message)
   742  	}
   743  }
   744  
   745  func TestHTTPServer_EnvironmentBlocks(t *testing.T) {
   746  	helper := test.New(t)
   747  	client := newClient()
   748  
   749  	shutdown, _ := newCouper("testdata/integration/environment/01_couper.hcl", test.New(t))
   750  	defer shutdown()
   751  
   752  	req, err := http.NewRequest(http.MethodGet, "http://anyserver:8080/test", nil)
   753  	helper.Must(err)
   754  
   755  	res, err := client.Do(req)
   756  	helper.Must(err)
   757  
   758  	if h := res.Header.Get("X-Test-Env"); h != "test" {
   759  		t.Errorf("Unexpected header given: %q", h)
   760  	}
   761  
   762  	if res.StatusCode != http.StatusOK {
   763  		t.Errorf("Unexpected status code: %d", res.StatusCode)
   764  	}
   765  }
   766  
   767  func TestHTTPServer_RateLimiterFixed(t *testing.T) {
   768  	helper := test.New(t)
   769  	client := newClient()
   770  
   771  	shutdown, hook := newCouper("testdata/integration/ratelimit/01_couper.hcl", test.New(t))
   772  	defer shutdown()
   773  
   774  	hook.Reset()
   775  
   776  	req, err := http.NewRequest(http.MethodGet, "http://anyserver:8080/fixed?", nil)
   777  	helper.Must(err)
   778  	go client.Do(req)
   779  	time.Sleep(1000 * time.Millisecond)
   780  	req, _ = http.NewRequest(http.MethodGet, "http://anyserver:8080/fixed?-", nil)
   781  	go client.Do(req)
   782  	time.Sleep(1000 * time.Millisecond)
   783  	req, _ = http.NewRequest(http.MethodGet, "http://anyserver:8080/fixed?--", nil)
   784  	go client.Do(req)
   785  	time.Sleep(500 * time.Millisecond)
   786  	req, _ = http.NewRequest(http.MethodGet, "http://anyserver:8080/fixed?---", nil)
   787  	go client.Do(req)
   788  
   789  	time.Sleep(700 * time.Millisecond)
   790  
   791  	entries := hook.AllEntries()
   792  	if len(entries) != 8 {
   793  		t.Fatal("Missing log lines")
   794  	}
   795  
   796  	for _, entry := range entries {
   797  		if entry.Data["type"] != "couper_access" {
   798  			continue
   799  		}
   800  
   801  		u := entry.Data["url"].(string)
   802  		cu, err := url.Parse(u)
   803  		helper.Must(err)
   804  		i := len(cu.RawQuery)
   805  
   806  		if total := entry.Data["timings"].(logging.Fields)["total"].(float64); total <= 0 {
   807  			t.Fatal("Something is wrong")
   808  		} else if i < 2 && total > 500 {
   809  			t.Errorf("Request %d time has to be shorter than 0.5 seconds, was %fms", i, total)
   810  		} else if i == 2 && total < 1000 {
   811  			t.Errorf("Request %d time has to be longer than 1 second, was %fms", i, total)
   812  		} else if i > 2 && total < 500 {
   813  			t.Errorf("Request %d time has to be longer than 0.5 seconds, was %fms", i, total)
   814  		}
   815  	}
   816  }
   817  
   818  func TestHTTPServer_RateLimiterSliding(t *testing.T) {
   819  	helper := test.New(t)
   820  	client := newClient()
   821  
   822  	shutdown, hook := newCouper("testdata/integration/ratelimit/01_couper.hcl", test.New(t))
   823  	defer shutdown()
   824  
   825  	hook.Reset()
   826  
   827  	req, err := http.NewRequest(http.MethodGet, "http://anyserver:8080/sliding?", nil)
   828  	helper.Must(err)
   829  	go client.Do(req)
   830  	time.Sleep(1000 * time.Millisecond)
   831  	req, _ = http.NewRequest(http.MethodGet, "http://anyserver:8080/sliding?-", nil)
   832  	go client.Do(req)
   833  	time.Sleep(1000 * time.Millisecond)
   834  	req, _ = http.NewRequest(http.MethodGet, "http://anyserver:8080/sliding?--", nil)
   835  	go client.Do(req)
   836  	time.Sleep(500 * time.Millisecond)
   837  	req, _ = http.NewRequest(http.MethodGet, "http://anyserver:8080/sliding?---", nil)
   838  	go client.Do(req)
   839  
   840  	time.Sleep(1700 * time.Millisecond)
   841  
   842  	entries := hook.AllEntries()
   843  	if len(entries) != 8 {
   844  		t.Fatal("Missing log lines")
   845  	}
   846  
   847  	for _, entry := range entries {
   848  		if entry.Data["type"] != "couper_access" {
   849  			continue
   850  		}
   851  
   852  		u := entry.Data["url"].(string)
   853  		cu, err := url.Parse(u)
   854  		helper.Must(err)
   855  		i := len(cu.RawQuery)
   856  
   857  		if total := entry.Data["timings"].(logging.Fields)["total"].(float64); total <= 0 {
   858  			t.Fatal("Something is wrong")
   859  		} else if i < 2 && total > 500 {
   860  			t.Errorf("Request %d time has to be shorter than 0.5 seconds, was %fms", i, total)
   861  		} else if i == 2 && total < 1000 {
   862  			t.Errorf("Request %d time has to be longer than 1 second, was %fms", i, total)
   863  		} else if i > 2 && total < 1500 {
   864  			t.Errorf("Request %d time has to be longer than 1.5 seconds, was %fms", i, total)
   865  		}
   866  	}
   867  }
   868  
   869  func TestHTTPServer_RateLimiterBlock(t *testing.T) {
   870  	helper := test.New(t)
   871  	client := newClient()
   872  
   873  	shutdown, _ := newCouper("testdata/integration/ratelimit/01_couper.hcl", test.New(t))
   874  	defer shutdown()
   875  
   876  	req, err := http.NewRequest(http.MethodGet, "http://anyserver:8080/block", nil)
   877  	helper.Must(err)
   878  
   879  	var resps [3]*http.Response
   880  	var mu sync.Mutex
   881  
   882  	go func() {
   883  		mu.Lock()
   884  		resps[0], _ = client.Do(req)
   885  		mu.Unlock()
   886  	}()
   887  
   888  	time.Sleep(400 * time.Millisecond)
   889  
   890  	go func() {
   891  		mu.Lock()
   892  		resps[1], _ = client.Do(req)
   893  		mu.Unlock()
   894  	}()
   895  
   896  	time.Sleep(400 * time.Millisecond)
   897  
   898  	go func() {
   899  		mu.Lock()
   900  		resps[2], _ = client.Do(req)
   901  		mu.Unlock()
   902  	}()
   903  
   904  	time.Sleep(400 * time.Millisecond)
   905  
   906  	mu.Lock()
   907  
   908  	if resps[0].StatusCode != 200 {
   909  		t.Errorf("Exp 200, got: %d", resps[0].StatusCode)
   910  	}
   911  	if resps[1].StatusCode != 200 {
   912  		t.Errorf("Exp 200, got: %d", resps[1].StatusCode)
   913  	}
   914  	if resps[2].StatusCode != 429 {
   915  		t.Errorf("Exp 200, got: %d", resps[2].StatusCode)
   916  	}
   917  
   918  	mu.Unlock()
   919  }
   920  
   921  func TestHTTPServer_ServerTiming(t *testing.T) {
   922  	helper := test.New(t)
   923  	client := newClient()
   924  
   925  	confPath1 := "testdata/integration/http/01_couper.hcl"
   926  	shutdown1, _ := newCouper(confPath1, test.New(t))
   927  	defer shutdown1()
   928  
   929  	confPath2 := "testdata/integration/http/02_couper.hcl"
   930  	shutdown2, _ := newCouper(confPath2, test.New(t))
   931  	defer shutdown2()
   932  
   933  	req, err := http.NewRequest(http.MethodGet, "http://anyserver:9090/", nil)
   934  	helper.Must(err)
   935  
   936  	res, err := client.Do(req)
   937  	helper.Must(err)
   938  
   939  	headers := res.Header.Values("Server-Timing")
   940  	if l := len(headers); l != 2 {
   941  		t.Fatalf("Unexpected number of headers: %d", l)
   942  	}
   943  
   944  	dataCouper1 := strings.Split(headers[0], ", ")
   945  	dataCouper2 := strings.Split(headers[1], ", ")
   946  
   947  	sort.Strings(dataCouper1)
   948  	sort.Strings(dataCouper2)
   949  
   950  	if len(dataCouper1) != 4 || len(dataCouper2) != 6 {
   951  		t.Fatal("Unexpected number of metrics")
   952  	}
   953  
   954  	exp1 := regexp.MustCompile(`b1_dns_[0-9a-f]{6};dur=\d+(.\d)* b1_tcp_[0-9a-f]{6};dur=\d+(.\d)* b1_total_[0-9a-f]{6};dur=\d+(.\d)* b1_ttfb_[0-9a-f]{6};dur=\d+(.\d)*`)
   955  	if s := strings.Join(dataCouper1, " "); !exp1.MatchString(s) {
   956  		t.Errorf("Unexpected header from 'first' Couper: %s", s)
   957  	}
   958  
   959  	exp2 := regexp.MustCompile(`b1_tcp;dur=\d+(.\d)* b1_total;dur=\d+(.\d)* b1_ttfb;dur=\d+(.\d)* b2_REQ_tcp;dur=\d+(.\d)* b2_REQ_total;dur=\d+(.\d)* b2_REQ_ttfb;dur=\d+(.\d)*`)
   960  	if s := strings.Join(dataCouper2, " "); !exp2.MatchString(s) {
   961  		t.Errorf("Unexpected header from 'second' Couper: %s", s)
   962  	}
   963  
   964  	req, err = http.NewRequest(http.MethodGet, "http://anyserver:9090/empty", nil)
   965  	helper.Must(err)
   966  
   967  	res, err = client.Do(req)
   968  	helper.Must(err)
   969  
   970  	headers = res.Header.Values("Server-Timing")
   971  	if l := len(headers); l != 0 {
   972  		t.Fatalf("Unexpected number of headers: %d", l)
   973  	}
   974  }
   975  
   976  func TestHTTPServer_CVE_2022_2880(t *testing.T) {
   977  	helper := test.New(t)
   978  	client := newClient()
   979  
   980  	confPath := "testdata/integration/validation/03_couper.hcl"
   981  	shutdown, logHook := newCouper(confPath, test.New(t))
   982  	defer shutdown()
   983  
   984  	logHook.Reset()
   985  
   986  	// See https://github.com/golang/go/issues/54663
   987  	req, err := http.NewRequest(http.MethodGet, "http://anyserver:8080/q?a=%x&b=ok", nil)
   988  	helper.Must(err)
   989  
   990  	_, err = client.Do(req)
   991  	helper.Must(err)
   992  
   993  	// Wait for log
   994  	time.Sleep(300 * time.Millisecond)
   995  
   996  	got := logHook.AllEntries()[0].Data["custom"].(logrus.Fields)["TEST"]
   997  	exp := logrus.Fields{"b": []interface{}{"ok"}}
   998  
   999  	if !cmp.Equal(got, exp) {
  1000  		t.Error(cmp.Diff(got, exp))
  1001  	}
  1002  }
  1003  
  1004  func TestHTTPServer_HealthVsAccessControl(t *testing.T) {
  1005  	helper := test.New(t)
  1006  	client := newClient()
  1007  
  1008  	shutdown, _ := newCouper("testdata/settings/22_couper.hcl", helper)
  1009  	defer shutdown()
  1010  
  1011  	// Call health route
  1012  	req, err := http.NewRequest(http.MethodGet, "http://example.com:8080/healthz", nil)
  1013  	helper.Must(err)
  1014  
  1015  	res, err := client.Do(req)
  1016  	helper.Must(err)
  1017  
  1018  	if res.StatusCode != http.StatusOK {
  1019  		t.Errorf("Expected status 200, got %d", res.StatusCode)
  1020  	}
  1021  
  1022  	// Call other route
  1023  	req, err = http.NewRequest(http.MethodGet, "http://example.com:8080/foo", nil)
  1024  	helper.Must(err)
  1025  
  1026  	res, err = client.Do(req)
  1027  	helper.Must(err)
  1028  
  1029  	if res.StatusCode != http.StatusUnauthorized {
  1030  		t.Errorf("Expected status 401, got %d", res.StatusCode)
  1031  	}
  1032  }