k8s.io/apiserver@v0.31.1/pkg/server/filters/timeout_test.go (about)

     1  /*
     2  Copyright 2016 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package filters
    18  
    19  import (
    20  	"bytes"
    21  	"context"
    22  	"crypto/tls"
    23  	"crypto/x509"
    24  	"encoding/json"
    25  	"fmt"
    26  	"io/ioutil"
    27  	"net"
    28  	"net/http"
    29  	"net/http/httptest"
    30  	"net/http/httptrace"
    31  	"reflect"
    32  	"strings"
    33  	"sync"
    34  	"testing"
    35  	"time"
    36  
    37  	"github.com/google/go-cmp/cmp"
    38  	"golang.org/x/net/http2"
    39  
    40  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    41  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    42  	"k8s.io/apimachinery/pkg/runtime/schema"
    43  	"k8s.io/apimachinery/pkg/util/runtime"
    44  	"k8s.io/apimachinery/pkg/util/sets"
    45  	"k8s.io/apiserver/pkg/endpoints/request"
    46  	"k8s.io/apiserver/pkg/endpoints/responsewriter"
    47  	"k8s.io/klog/v2"
    48  )
    49  
    50  type recorder struct {
    51  	lock  sync.Mutex
    52  	count int
    53  }
    54  
    55  func (r *recorder) Record() {
    56  	r.lock.Lock()
    57  	defer r.lock.Unlock()
    58  	r.count++
    59  }
    60  
    61  func (r *recorder) Count() int {
    62  	r.lock.Lock()
    63  	defer r.lock.Unlock()
    64  	return r.count
    65  }
    66  
    67  func newHandler(responseCh <-chan string, panicCh <-chan interface{}, writeErrCh chan<- error) http.HandlerFunc {
    68  	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
    69  		select {
    70  		case resp := <-responseCh:
    71  			_, err := w.Write([]byte(resp))
    72  			writeErrCh <- err
    73  		case panicReason := <-panicCh:
    74  			panic(panicReason)
    75  		}
    76  	})
    77  }
    78  
    79  func TestTimeout(t *testing.T) {
    80  	origReallyCrash := runtime.ReallyCrash
    81  	runtime.ReallyCrash = false
    82  	defer func() {
    83  		runtime.ReallyCrash = origReallyCrash
    84  	}()
    85  
    86  	sendResponse := make(chan string, 1)
    87  	doPanic := make(chan interface{}, 1)
    88  	writeErrors := make(chan error, 1)
    89  	gotPanic := make(chan interface{}, 1)
    90  	timeout := make(chan time.Time, 1)
    91  	resp := "test response"
    92  	timeoutErr := apierrors.NewServerTimeout(schema.GroupResource{Group: "foo", Resource: "bar"}, "get", 0)
    93  	record := &recorder{}
    94  
    95  	var ctx context.Context
    96  	withDeadline := func(handler http.Handler) http.Handler {
    97  		return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
    98  			req = req.WithContext(ctx)
    99  			handler.ServeHTTP(w, req)
   100  		})
   101  	}
   102  
   103  	handler := newHandler(sendResponse, doPanic, writeErrors)
   104  	ts := httptest.NewServer(withDeadline(withPanicRecovery(
   105  		WithTimeout(handler, func(req *http.Request) (*http.Request, bool, func(), *apierrors.StatusError) {
   106  			return req, false, record.Record, timeoutErr
   107  		}), func(w http.ResponseWriter, req *http.Request, err interface{}) {
   108  			gotPanic <- err
   109  			http.Error(w, "This request caused apiserver to panic. Look in the logs for details.", http.StatusInternalServerError)
   110  		}),
   111  	))
   112  	defer ts.Close()
   113  
   114  	// No timeouts
   115  	ctx = context.Background()
   116  	sendResponse <- resp
   117  	res, err := http.Get(ts.URL)
   118  	if err != nil {
   119  		t.Fatal(err)
   120  	}
   121  	if res.StatusCode != http.StatusOK {
   122  		t.Errorf("got res.StatusCode %d; expected %d", res.StatusCode, http.StatusOK)
   123  	}
   124  	body, _ := ioutil.ReadAll(res.Body)
   125  	if string(body) != resp {
   126  		t.Errorf("got body %q; expected %q", string(body), resp)
   127  	}
   128  	if err := <-writeErrors; err != nil {
   129  		t.Errorf("got unexpected Write error on first request: %v", err)
   130  	}
   131  	if record.Count() != 0 {
   132  		t.Errorf("invoked record method: %#v", record)
   133  	}
   134  
   135  	// Times out
   136  	ctx, cancel := context.WithCancel(context.Background())
   137  	cancel()
   138  	timeout <- time.Time{}
   139  	res, err = http.Get(ts.URL)
   140  	if err != nil {
   141  		t.Fatal(err)
   142  	}
   143  	if res.StatusCode != http.StatusGatewayTimeout {
   144  		t.Errorf("got res.StatusCode %d; expected %d", res.StatusCode, http.StatusGatewayTimeout)
   145  	}
   146  	body, _ = ioutil.ReadAll(res.Body)
   147  	status := &metav1.Status{}
   148  	if err := json.Unmarshal(body, status); err != nil {
   149  		t.Fatal(err)
   150  	}
   151  	if !reflect.DeepEqual(status, &timeoutErr.ErrStatus) {
   152  		t.Errorf("unexpected object: %s", cmp.Diff(&timeoutErr.ErrStatus, status))
   153  	}
   154  	if record.Count() != 1 {
   155  		t.Errorf("did not invoke record method: %#v", record)
   156  	}
   157  
   158  	// Now try to send a response
   159  	ctx = context.Background()
   160  	sendResponse <- resp
   161  	if err := <-writeErrors; err != http.ErrHandlerTimeout {
   162  		t.Errorf("got Write error of %v; expected %v", err, http.ErrHandlerTimeout)
   163  	}
   164  
   165  	// Panics
   166  	doPanic <- "inner handler panics"
   167  	res, err = http.Get(ts.URL)
   168  	if err != nil {
   169  		t.Fatal(err)
   170  	}
   171  	if res.StatusCode != http.StatusInternalServerError {
   172  		t.Errorf("got res.StatusCode %d; expected %d due to panic", res.StatusCode, http.StatusInternalServerError)
   173  	}
   174  	select {
   175  	case err := <-gotPanic:
   176  		msg := fmt.Sprintf("%v", err)
   177  		if !strings.Contains(msg, "newHandler") {
   178  			t.Errorf("expected line with root cause panic in the stack trace, but didn't: %v", err)
   179  		}
   180  	case <-time.After(30 * time.Second):
   181  		t.Fatalf("expected to see a handler panic, but didn't")
   182  	}
   183  
   184  	// Panics with http.ErrAbortHandler
   185  	ctx = context.Background()
   186  	doPanic <- http.ErrAbortHandler
   187  	res, err = http.Get(ts.URL)
   188  	if err != nil {
   189  		t.Fatal(err)
   190  	}
   191  	if res.StatusCode != http.StatusInternalServerError {
   192  		t.Errorf("got res.StatusCode %d; expected %d due to panic", res.StatusCode, http.StatusInternalServerError)
   193  	}
   194  	select {
   195  	case err := <-gotPanic:
   196  		if err != http.ErrAbortHandler {
   197  			t.Errorf("expected unwrapped http.ErrAbortHandler, got %#v", err)
   198  		}
   199  	case <-time.After(30 * time.Second):
   200  		t.Fatalf("expected to see a handler panic, but didn't")
   201  	}
   202  }
   203  
   204  func TestTimeoutHeaders(t *testing.T) {
   205  	origReallyCrash := runtime.ReallyCrash
   206  	runtime.ReallyCrash = false
   207  	defer func() {
   208  		runtime.ReallyCrash = origReallyCrash
   209  	}()
   210  
   211  	ctx, cancel := context.WithCancel(context.Background())
   212  	defer cancel()
   213  
   214  	withDeadline := func(handler http.Handler) http.Handler {
   215  		return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
   216  			handler.ServeHTTP(w, req.WithContext(ctx))
   217  		})
   218  	}
   219  
   220  	postTimeoutCh := make(chan struct{})
   221  	ts := httptest.NewServer(
   222  		withDeadline(
   223  			WithTimeout(
   224  				http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
   225  					h := w.Header()
   226  					// trigger the timeout
   227  					cancel()
   228  					// keep mutating response Headers until the request times out
   229  					for {
   230  						select {
   231  						case <-postTimeoutCh:
   232  							return
   233  						default:
   234  							h.Set("Test", "post")
   235  						}
   236  					}
   237  				}),
   238  				func(req *http.Request) (*http.Request, bool, func(), *apierrors.StatusError) {
   239  					return req, false, func() { close(postTimeoutCh) }, apierrors.NewServerTimeout(schema.GroupResource{Group: "foo", Resource: "bar"}, "get", 0)
   240  				},
   241  			),
   242  		),
   243  	)
   244  	defer ts.Close()
   245  
   246  	res, err := http.Get(ts.URL)
   247  	if err != nil {
   248  		t.Fatal(err)
   249  	}
   250  	if res.StatusCode != http.StatusGatewayTimeout {
   251  		t.Errorf("got res.StatusCode %d; expected %d", res.StatusCode, http.StatusGatewayTimeout)
   252  	}
   253  	res.Body.Close()
   254  }
   255  
   256  func TestTimeoutRequestHeaders(t *testing.T) {
   257  	origReallyCrash := runtime.ReallyCrash
   258  	runtime.ReallyCrash = false
   259  	defer func() {
   260  		runtime.ReallyCrash = origReallyCrash
   261  	}()
   262  
   263  	ctx, cancel := context.WithCancel(context.Background())
   264  	defer cancel()
   265  
   266  	// Add dummy request info, otherwise we skip postTimeoutFn
   267  	ctx = request.WithRequestInfo(ctx, &request.RequestInfo{})
   268  
   269  	withDeadline := func(handler http.Handler) http.Handler {
   270  		return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
   271  			handler.ServeHTTP(w, req.WithContext(ctx))
   272  		})
   273  	}
   274  
   275  	testDone := make(chan struct{})
   276  	defer close(testDone)
   277  	ts := httptest.NewServer(
   278  		withDeadline(
   279  			WithTimeoutForNonLongRunningRequests(
   280  				http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
   281  					// trigger the timeout
   282  					cancel()
   283  					// mutate request Headers
   284  					// Authorization filter does it for example
   285  					for {
   286  						select {
   287  						case <-testDone:
   288  							return
   289  						default:
   290  							req.Header.Set("Test", "post")
   291  						}
   292  					}
   293  				}),
   294  				func(r *http.Request, requestInfo *request.RequestInfo) bool {
   295  					return false
   296  				},
   297  			),
   298  		),
   299  	)
   300  	defer ts.Close()
   301  
   302  	client := &http.Client{}
   303  	req, err := http.NewRequest(http.MethodPatch, ts.URL, nil)
   304  	if err != nil {
   305  		t.Fatal(err)
   306  	}
   307  	res, err := client.Do(req)
   308  	if err != nil {
   309  		t.Fatal(err)
   310  	}
   311  	if actual, expected := res.StatusCode, http.StatusGatewayTimeout; actual != expected {
   312  		t.Errorf("got status code %d; expected %d", actual, expected)
   313  	}
   314  	res.Body.Close()
   315  }
   316  
   317  func TestTimeoutWithLogging(t *testing.T) {
   318  	origReallyCrash := runtime.ReallyCrash
   319  	runtime.ReallyCrash = false
   320  	defer func() {
   321  		runtime.ReallyCrash = origReallyCrash
   322  	}()
   323  
   324  	ctx, cancel := context.WithCancel(context.Background())
   325  	defer cancel()
   326  
   327  	withDeadline := func(handler http.Handler) http.Handler {
   328  		return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
   329  			handler.ServeHTTP(w, req.WithContext(ctx))
   330  		})
   331  	}
   332  
   333  	testDone := make(chan struct{})
   334  	defer close(testDone)
   335  	ts := httptest.NewServer(
   336  		WithHTTPLogging(
   337  			withDeadline(
   338  				WithTimeoutForNonLongRunningRequests(
   339  					http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
   340  						// trigger the timeout
   341  						cancel()
   342  						// mutate request Headers
   343  						// Authorization filter does it for example
   344  						for {
   345  							select {
   346  							case <-testDone:
   347  								return
   348  							default:
   349  								req.Header.Set("Test", "post")
   350  							}
   351  						}
   352  					}),
   353  					func(r *http.Request, requestInfo *request.RequestInfo) bool {
   354  						return false
   355  					},
   356  				),
   357  			),
   358  		),
   359  	)
   360  	defer ts.Close()
   361  
   362  	client := &http.Client{}
   363  	req, err := http.NewRequest(http.MethodPatch, ts.URL, nil)
   364  	if err != nil {
   365  		t.Fatal(err)
   366  	}
   367  	res, err := client.Do(req)
   368  	if err != nil {
   369  		t.Fatal(err)
   370  	}
   371  	if actual, expected := res.StatusCode, http.StatusGatewayTimeout; actual != expected {
   372  		t.Errorf("got status code %d; expected %d", actual, expected)
   373  	}
   374  	res.Body.Close()
   375  }
   376  
   377  func TestErrConnKilled(t *testing.T) {
   378  	var buf bytes.Buffer
   379  	klog.SetOutput(&buf)
   380  	klog.LogToStderr(false)
   381  	defer klog.LogToStderr(true)
   382  
   383  	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
   384  		// this error must be ignored by the WithPanicRecovery handler
   385  		// it is thrown by WithTimeoutForNonLongRunningRequests handler when a response has been already sent to the client and the handler timed out
   386  		// panicking with http.ErrAbortHandler also suppresses logging of a stack trace to the server's error log and closes the underlying connection
   387  		w.Write([]byte("hello from the handler"))
   388  		panic(http.ErrAbortHandler)
   389  	})
   390  	resolver := &request.RequestInfoFactory{
   391  		APIPrefixes:          sets.NewString("api", "apis"),
   392  		GrouplessAPIPrefixes: sets.NewString("api"),
   393  	}
   394  
   395  	ts := httptest.NewServer(WithPanicRecovery(handler, resolver))
   396  	defer ts.Close()
   397  
   398  	_, err := http.Get(ts.URL)
   399  	if err == nil {
   400  		t.Fatal("expected to receive an error")
   401  	}
   402  
   403  	klog.Flush()
   404  	klog.SetOutput(&bytes.Buffer{}) // prevent further writes into buf
   405  	capturedOutput := buf.String()
   406  
   407  	// We don't expect stack trace from the panic to be included in the log.
   408  	if isStackTraceLoggedByRuntime(capturedOutput) {
   409  		t.Errorf("unexpected stack trace in log, actual = %v", capturedOutput)
   410  	}
   411  	// For the sake of simplicity and clarity this matches the full log line.
   412  	// This is not part of the Kubernetes API and could change.
   413  	if !strings.Contains(capturedOutput, `"Timeout or abort while handling" logger="UnhandledError" method="GET" URI="/" auditID=""`) {
   414  		t.Errorf("unexpected output captured actual = %v", capturedOutput)
   415  	}
   416  }
   417  
   418  type panicOnNonReuseTransport struct {
   419  	Transport   http.RoundTripper
   420  	gotConnSeen bool
   421  }
   422  
   423  func (t *panicOnNonReuseTransport) RoundTrip(req *http.Request) (*http.Response, error) {
   424  	return t.Transport.RoundTrip(req)
   425  }
   426  
   427  func (t *panicOnNonReuseTransport) GotConn(info httptrace.GotConnInfo) {
   428  	if !t.gotConnSeen {
   429  		t.gotConnSeen = true
   430  		return
   431  	}
   432  	if !info.Reused {
   433  		panic(fmt.Sprintf("expected the connection to be reused, info %#v", info))
   434  	}
   435  }
   436  
   437  // TestErrConnKilledHTTP2 check if HTTP/2 connection is not closed when an HTTP handler panics
   438  // The net/http library recovers the panic and sends an HTTP/2 RST_STREAM.
   439  func TestErrConnKilledHTTP2(t *testing.T) {
   440  	var buf bytes.Buffer
   441  	klog.SetOutput(&buf)
   442  	klog.LogToStderr(false)
   443  	defer klog.LogToStderr(true)
   444  
   445  	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
   446  		// this error must be ignored by the WithPanicRecovery handler
   447  		// it is thrown by WithTimeoutForNonLongRunningRequests handler when a response has been already sent to the client and the handler timed out
   448  		// panicking with http.ErrAbortHandler also suppresses logging of a stack trace to the server's error log and closes the underlying connection
   449  		w.Write([]byte("hello from the handler"))
   450  		panic(http.ErrAbortHandler)
   451  	})
   452  	resolver := &request.RequestInfoFactory{
   453  		APIPrefixes:          sets.NewString("api", "apis"),
   454  		GrouplessAPIPrefixes: sets.NewString("api"),
   455  	}
   456  
   457  	// test server
   458  	ts := httptest.NewUnstartedServer(WithPanicRecovery(handler, resolver))
   459  	tsCert, err := tls.X509KeyPair(tsCrt, tsKey)
   460  	if err != nil {
   461  		t.Fatalf("backend: invalid x509/key pair: %v", err)
   462  	}
   463  	ts.TLS = &tls.Config{
   464  		Certificates: []tls.Certificate{tsCert},
   465  		NextProtos:   []string{http2.NextProtoTLS},
   466  	}
   467  	ts.StartTLS()
   468  	defer ts.Close()
   469  
   470  	newServerRequest := func(tr *panicOnNonReuseTransport) *http.Request {
   471  		req, _ := http.NewRequest("GET", fmt.Sprintf("https://127.0.0.1:%d", ts.Listener.Addr().(*net.TCPAddr).Port), nil)
   472  		trace := &httptrace.ClientTrace{
   473  			GotConn: tr.GotConn,
   474  		}
   475  		return req.WithContext(httptrace.WithClientTrace(req.Context(), trace))
   476  	}
   477  
   478  	// client
   479  	clientCACertPool := x509.NewCertPool()
   480  	clientCACertPool.AppendCertsFromPEM(tsCrt)
   481  	clientTLSConfig := &tls.Config{
   482  		RootCAs:    clientCACertPool,
   483  		NextProtos: []string{http2.NextProtoTLS},
   484  	}
   485  	tr := &panicOnNonReuseTransport{}
   486  	client := &http.Client{}
   487  	tr.Transport = &http2.Transport{
   488  		TLSClientConfig: clientTLSConfig,
   489  	}
   490  	client.Transport = tr
   491  
   492  	// act
   493  	_, err = client.Do(newServerRequest(tr))
   494  	if err == nil {
   495  		t.Fatal("expected to receive an error")
   496  	}
   497  
   498  	klog.Flush()
   499  	klog.SetOutput(&bytes.Buffer{}) // prevent further writes into buf
   500  	capturedOutput := buf.String()
   501  
   502  	// We don't expect stack trace from the panic to be included in the log.
   503  	if isStackTraceLoggedByRuntime(capturedOutput) {
   504  		t.Errorf("unexpected stack trace in log, actual = %v", capturedOutput)
   505  	}
   506  	// For the sake of simplicity and clarity this matches the full log line.
   507  	// This is not part of the Kubernetes API and could change.
   508  	if !strings.Contains(capturedOutput, `"Timeout or abort while handling" logger="UnhandledError" method="GET" URI="/" auditID=""`) {
   509  		t.Errorf("unexpected output captured actual = %v", capturedOutput)
   510  	}
   511  
   512  	// make another req to the server
   513  	// the connection should be reused
   514  	// the client uses a custom transport that checks and panics when the con wasn't reused.
   515  	_, err = client.Do(newServerRequest(tr))
   516  	if err == nil {
   517  		t.Fatal("expected to receive an error")
   518  	}
   519  }
   520  
   521  func TestResponseWriterDecorator(t *testing.T) {
   522  	decorator := &baseTimeoutWriter{
   523  		w: &responsewriter.FakeResponseWriter{},
   524  	}
   525  	var w http.ResponseWriter = decorator
   526  
   527  	if inner := w.(responsewriter.UserProvidedDecorator).Unwrap(); inner != decorator.w {
   528  		t.Errorf("Expected the decorator to return the inner http.ResponseWriter object")
   529  	}
   530  }
   531  
   532  func isStackTraceLoggedByRuntime(message string) bool {
   533  	// Check the captured output for the following patterns to find out if the
   534  	// stack trace is included in the log:
   535  	// - 'Observed a panic' (apimachinery runtime.go logs panic with this message)
   536  	// - 'goroutine 44 [running]:' (stack trace always starts with this)
   537  	if strings.Contains(message, "Observed a panic") &&
   538  		strings.Contains(message, "goroutine") &&
   539  		strings.Contains(message, "[running]:") {
   540  		return true
   541  	}
   542  
   543  	return false
   544  }
   545  
   546  var tsCrt = []byte(`-----BEGIN CERTIFICATE-----
   547  MIIDTjCCAjagAwIBAgIJAJdcQEBN2CjoMA0GCSqGSIb3DQEBCwUAMFAxCzAJBgNV
   548  BAYTAlBMMQ8wDQYDVQQIDAZQb2xhbmQxDzANBgNVBAcMBkdkYW5zazELMAkGA1UE
   549  CgwCU0sxEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0yMDA5MjgxMTU1MjhaFw0zMDA5
   550  MjYxMTU1MjhaMFAxCzAJBgNVBAYTAlBMMQ8wDQYDVQQIDAZQb2xhbmQxDzANBgNV
   551  BAcMBkdkYW5zazELMAkGA1UECgwCU0sxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIw
   552  DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMr6b/uTHkIDEd88x3t3jnroOVwh
   553  jWMwZ6qXN2NV/If1L9FNvtoZzZi6yCDE1uLdD1kWZ0R2XOPEwUPn+Z8A/lg9kF8J
   554  GloLCF8q+XeYp8aWRKzwtdi+MPaKFf0wsuxEEHU4pypFrszNY0yLRbWAbMtgBFy0
   555  KhyNGahFO9V69cRHUj6EJ9kSBg0nG5bsypon2rinzKpUrzAEl2MbM3F34Zit5yOv
   556  rYQcbDME+9XmOJPD97XBvMZCbmPnmpst3tX7ZhdKgSKtIjoYt+d//wtPMXOhrRzM
   557  xcc6HuIHAovtB4kvZl5wvVU8ra8DKZviYyjfW36kQHo+yFwP3XXZFWezZi0CAwEA
   558  AaMrMCkwCQYDVR0TBAIwADALBgNVHQ8EBAMCBaAwDwYDVR0RBAgwBocEfwAAATAN
   559  BgkqhkiG9w0BAQsFAAOCAQEAMoAlhZ6yRwPmFL2ql9ZYWqaPu2NC4dXYV6kUIXUA
   560  pG3IqFWb3L4ePkgYBMDlJGpAJcXlSSgEGiLj+3qQojawIMzzxmqr2KX888S5Mr+9
   561  I1qseRwcftwYqV/ewJSWE90HJ21pb1ixA6bSRJLV7DyxO6zKsdVJ4xIvehZtGbux
   562  0RTf+8zUx8z2Goy1GUztOIqfMRt1P1hlQG0uvYsGQM84HO4+YhFwejrGaj8ajpgF
   563  uo3B8BVHeh57FNGE6C45NkFGHq3tkNLMdAa32Az8DDvPmsJuycf6vgIfBEQxLZSF
   564  OUKrKmtfdFv4XrInqFUYBYp5GkL8SGM2wmv6aSw9Aju4lA==
   565  -----END CERTIFICATE-----`)
   566  
   567  var tsKey = []byte(`-----BEGIN PRIVATE KEY-----
   568  MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDK+m/7kx5CAxHf
   569  PMd7d4566DlcIY1jMGeqlzdjVfyH9S/RTb7aGc2YusggxNbi3Q9ZFmdEdlzjxMFD
   570  5/mfAP5YPZBfCRpaCwhfKvl3mKfGlkSs8LXYvjD2ihX9MLLsRBB1OKcqRa7MzWNM
   571  i0W1gGzLYARctCocjRmoRTvVevXER1I+hCfZEgYNJxuW7MqaJ9q4p8yqVK8wBJdj
   572  GzNxd+GYrecjr62EHGwzBPvV5jiTw/e1wbzGQm5j55qbLd7V+2YXSoEirSI6GLfn
   573  f/8LTzFzoa0czMXHOh7iBwKL7QeJL2ZecL1VPK2vAymb4mMo31t+pEB6PshcD911
   574  2RVns2YtAgMBAAECggEAA2Qx0MtBeyrf9pHmZ1q1B7qvkqmA2kJpyQDjzQYXxRHE
   575  rcOVx8EcnUupolqHmJzG798e9JbhsHCOJhtPIWf71++XZO8bAJwklKp8JpJnYzsJ
   576  hLY0450x5jyiZ2uT4by1Za//owYtCID6AsJk9MZjivZcvEvKVFXLMvONL2DxkEj1
   577  KaGQJh6/GT4jtNX07bW9+5w069KAAf+BNuqv8+Y/FseV3ovlpLTKjMV9xCCp9i62
   578  PJs/hs5CW2X+JCE7OCLsAiu0JTpXYyHcLwYwnCONdvj6veiMWjRyNDr3ew5NeZNf
   579  nGU4WX7mXjPd/1OvzJy6iyrBlAA63ZfFZYjWQnfsIQKBgQDmo3AMIr+9rE79NnaD
   580  woQyO539YSO45KSM39/Xrp/NJVpOxtzgZrYo7O6f6kQ3S5zQOddy9Oj7gN3RXhZ7
   581  Vi+Oja78ig7KUrqxcBiBGRsKZGm5CGdZ0EFd3rIEh4Qb+f+2c4f+6NWANb4kwvfq
   582  K24c1o71+77lEVlzE2/L33K+mQKBgQDhTFr/f2e9gnRNX9bjF4p7DQI0RsFADjx0
   583  jgJYHfm/lCIdH9vf6SmmvJv2E76Bqx9XVilhav/egqKO/wzJWHyNo2RFBXNqfwoF
   584  UxRZKgqhcU52y2LKAYoTYfodktatZk74rinMDLmA6arnlAWQELk3Mx48DlND43Zc
   585  DUHTKcJEtQKBgQDYdL1c9mPjnEqJxMqXwEAXcPJG8hr3lMaGXDoVjxL1EsBdvK9h
   586  f6QoZq1RsiiRiMpEdnSotAfQutHzhA0vdeSuMnTvGJbm9Zu3mc+1oZ1KNJEwkh2F
   587  Ijmm4rFKJPEs3IVMc8NHzrdJW6b3k2/e+yGduRR08e7nx0+e+7fpq+1hyQKBgHY9
   588  l4h9+hkYjSdKhEG8yh3Ybu62r5eJoSremNZcLQXhnaHBZaj2+rgaRpP4OsRc5d71
   589  RlRtTood72iy7KgDO6MuPGKJANDEiaLPvl8pVFj0WWS5S0iPVELl6dl5hheNGSck
   590  aKVBjF3exKYzJlQ8oqgYuOZ18jcv+p9HCePkB6P9AoGBAJSYpkNDc/lnCpfIlxVw
   591  n+VroX6QDIMZzC7BGiUSrmVsu6xEbI+8/C7ecN2oCZZLMj96EXe6j+np4zmkQezc
   592  c1EwB7fNAiS0fWyE2RU6QAOZJ71bDpzQa4q4DxbOkYSybGPM/nqDRwovdjUnWeuM
   593  +vrJUjAZAPHJcvos0iylnc8E
   594  -----END PRIVATE KEY-----`)