k8s.io/apiserver@v0.31.1/pkg/server/filters/timeout_test.go (about) 1 /* 2 Copyright 2016 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package filters 18 19 import ( 20 "bytes" 21 "context" 22 "crypto/tls" 23 "crypto/x509" 24 "encoding/json" 25 "fmt" 26 "io/ioutil" 27 "net" 28 "net/http" 29 "net/http/httptest" 30 "net/http/httptrace" 31 "reflect" 32 "strings" 33 "sync" 34 "testing" 35 "time" 36 37 "github.com/google/go-cmp/cmp" 38 "golang.org/x/net/http2" 39 40 apierrors "k8s.io/apimachinery/pkg/api/errors" 41 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 42 "k8s.io/apimachinery/pkg/runtime/schema" 43 "k8s.io/apimachinery/pkg/util/runtime" 44 "k8s.io/apimachinery/pkg/util/sets" 45 "k8s.io/apiserver/pkg/endpoints/request" 46 "k8s.io/apiserver/pkg/endpoints/responsewriter" 47 "k8s.io/klog/v2" 48 ) 49 50 type recorder struct { 51 lock sync.Mutex 52 count int 53 } 54 55 func (r *recorder) Record() { 56 r.lock.Lock() 57 defer r.lock.Unlock() 58 r.count++ 59 } 60 61 func (r *recorder) Count() int { 62 r.lock.Lock() 63 defer r.lock.Unlock() 64 return r.count 65 } 66 67 func newHandler(responseCh <-chan string, panicCh <-chan interface{}, writeErrCh chan<- error) http.HandlerFunc { 68 return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 69 select { 70 case resp := <-responseCh: 71 _, err := w.Write([]byte(resp)) 72 writeErrCh <- err 73 case panicReason := <-panicCh: 74 panic(panicReason) 75 } 76 }) 77 } 78 79 func TestTimeout(t *testing.T) { 80 origReallyCrash := runtime.ReallyCrash 81 runtime.ReallyCrash = false 82 defer func() { 83 runtime.ReallyCrash = origReallyCrash 84 }() 85 86 sendResponse := make(chan string, 1) 87 doPanic := make(chan interface{}, 1) 88 writeErrors := make(chan error, 1) 89 gotPanic := make(chan interface{}, 1) 90 timeout := make(chan time.Time, 1) 91 resp := "test response" 92 timeoutErr := apierrors.NewServerTimeout(schema.GroupResource{Group: "foo", Resource: "bar"}, "get", 0) 93 record := &recorder{} 94 95 var ctx context.Context 96 withDeadline := func(handler http.Handler) http.Handler { 97 return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 98 req = req.WithContext(ctx) 99 handler.ServeHTTP(w, req) 100 }) 101 } 102 103 handler := newHandler(sendResponse, doPanic, writeErrors) 104 ts := httptest.NewServer(withDeadline(withPanicRecovery( 105 WithTimeout(handler, func(req *http.Request) (*http.Request, bool, func(), *apierrors.StatusError) { 106 return req, false, record.Record, timeoutErr 107 }), func(w http.ResponseWriter, req *http.Request, err interface{}) { 108 gotPanic <- err 109 http.Error(w, "This request caused apiserver to panic. Look in the logs for details.", http.StatusInternalServerError) 110 }), 111 )) 112 defer ts.Close() 113 114 // No timeouts 115 ctx = context.Background() 116 sendResponse <- resp 117 res, err := http.Get(ts.URL) 118 if err != nil { 119 t.Fatal(err) 120 } 121 if res.StatusCode != http.StatusOK { 122 t.Errorf("got res.StatusCode %d; expected %d", res.StatusCode, http.StatusOK) 123 } 124 body, _ := ioutil.ReadAll(res.Body) 125 if string(body) != resp { 126 t.Errorf("got body %q; expected %q", string(body), resp) 127 } 128 if err := <-writeErrors; err != nil { 129 t.Errorf("got unexpected Write error on first request: %v", err) 130 } 131 if record.Count() != 0 { 132 t.Errorf("invoked record method: %#v", record) 133 } 134 135 // Times out 136 ctx, cancel := context.WithCancel(context.Background()) 137 cancel() 138 timeout <- time.Time{} 139 res, err = http.Get(ts.URL) 140 if err != nil { 141 t.Fatal(err) 142 } 143 if res.StatusCode != http.StatusGatewayTimeout { 144 t.Errorf("got res.StatusCode %d; expected %d", res.StatusCode, http.StatusGatewayTimeout) 145 } 146 body, _ = ioutil.ReadAll(res.Body) 147 status := &metav1.Status{} 148 if err := json.Unmarshal(body, status); err != nil { 149 t.Fatal(err) 150 } 151 if !reflect.DeepEqual(status, &timeoutErr.ErrStatus) { 152 t.Errorf("unexpected object: %s", cmp.Diff(&timeoutErr.ErrStatus, status)) 153 } 154 if record.Count() != 1 { 155 t.Errorf("did not invoke record method: %#v", record) 156 } 157 158 // Now try to send a response 159 ctx = context.Background() 160 sendResponse <- resp 161 if err := <-writeErrors; err != http.ErrHandlerTimeout { 162 t.Errorf("got Write error of %v; expected %v", err, http.ErrHandlerTimeout) 163 } 164 165 // Panics 166 doPanic <- "inner handler panics" 167 res, err = http.Get(ts.URL) 168 if err != nil { 169 t.Fatal(err) 170 } 171 if res.StatusCode != http.StatusInternalServerError { 172 t.Errorf("got res.StatusCode %d; expected %d due to panic", res.StatusCode, http.StatusInternalServerError) 173 } 174 select { 175 case err := <-gotPanic: 176 msg := fmt.Sprintf("%v", err) 177 if !strings.Contains(msg, "newHandler") { 178 t.Errorf("expected line with root cause panic in the stack trace, but didn't: %v", err) 179 } 180 case <-time.After(30 * time.Second): 181 t.Fatalf("expected to see a handler panic, but didn't") 182 } 183 184 // Panics with http.ErrAbortHandler 185 ctx = context.Background() 186 doPanic <- http.ErrAbortHandler 187 res, err = http.Get(ts.URL) 188 if err != nil { 189 t.Fatal(err) 190 } 191 if res.StatusCode != http.StatusInternalServerError { 192 t.Errorf("got res.StatusCode %d; expected %d due to panic", res.StatusCode, http.StatusInternalServerError) 193 } 194 select { 195 case err := <-gotPanic: 196 if err != http.ErrAbortHandler { 197 t.Errorf("expected unwrapped http.ErrAbortHandler, got %#v", err) 198 } 199 case <-time.After(30 * time.Second): 200 t.Fatalf("expected to see a handler panic, but didn't") 201 } 202 } 203 204 func TestTimeoutHeaders(t *testing.T) { 205 origReallyCrash := runtime.ReallyCrash 206 runtime.ReallyCrash = false 207 defer func() { 208 runtime.ReallyCrash = origReallyCrash 209 }() 210 211 ctx, cancel := context.WithCancel(context.Background()) 212 defer cancel() 213 214 withDeadline := func(handler http.Handler) http.Handler { 215 return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 216 handler.ServeHTTP(w, req.WithContext(ctx)) 217 }) 218 } 219 220 postTimeoutCh := make(chan struct{}) 221 ts := httptest.NewServer( 222 withDeadline( 223 WithTimeout( 224 http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 225 h := w.Header() 226 // trigger the timeout 227 cancel() 228 // keep mutating response Headers until the request times out 229 for { 230 select { 231 case <-postTimeoutCh: 232 return 233 default: 234 h.Set("Test", "post") 235 } 236 } 237 }), 238 func(req *http.Request) (*http.Request, bool, func(), *apierrors.StatusError) { 239 return req, false, func() { close(postTimeoutCh) }, apierrors.NewServerTimeout(schema.GroupResource{Group: "foo", Resource: "bar"}, "get", 0) 240 }, 241 ), 242 ), 243 ) 244 defer ts.Close() 245 246 res, err := http.Get(ts.URL) 247 if err != nil { 248 t.Fatal(err) 249 } 250 if res.StatusCode != http.StatusGatewayTimeout { 251 t.Errorf("got res.StatusCode %d; expected %d", res.StatusCode, http.StatusGatewayTimeout) 252 } 253 res.Body.Close() 254 } 255 256 func TestTimeoutRequestHeaders(t *testing.T) { 257 origReallyCrash := runtime.ReallyCrash 258 runtime.ReallyCrash = false 259 defer func() { 260 runtime.ReallyCrash = origReallyCrash 261 }() 262 263 ctx, cancel := context.WithCancel(context.Background()) 264 defer cancel() 265 266 // Add dummy request info, otherwise we skip postTimeoutFn 267 ctx = request.WithRequestInfo(ctx, &request.RequestInfo{}) 268 269 withDeadline := func(handler http.Handler) http.Handler { 270 return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 271 handler.ServeHTTP(w, req.WithContext(ctx)) 272 }) 273 } 274 275 testDone := make(chan struct{}) 276 defer close(testDone) 277 ts := httptest.NewServer( 278 withDeadline( 279 WithTimeoutForNonLongRunningRequests( 280 http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 281 // trigger the timeout 282 cancel() 283 // mutate request Headers 284 // Authorization filter does it for example 285 for { 286 select { 287 case <-testDone: 288 return 289 default: 290 req.Header.Set("Test", "post") 291 } 292 } 293 }), 294 func(r *http.Request, requestInfo *request.RequestInfo) bool { 295 return false 296 }, 297 ), 298 ), 299 ) 300 defer ts.Close() 301 302 client := &http.Client{} 303 req, err := http.NewRequest(http.MethodPatch, ts.URL, nil) 304 if err != nil { 305 t.Fatal(err) 306 } 307 res, err := client.Do(req) 308 if err != nil { 309 t.Fatal(err) 310 } 311 if actual, expected := res.StatusCode, http.StatusGatewayTimeout; actual != expected { 312 t.Errorf("got status code %d; expected %d", actual, expected) 313 } 314 res.Body.Close() 315 } 316 317 func TestTimeoutWithLogging(t *testing.T) { 318 origReallyCrash := runtime.ReallyCrash 319 runtime.ReallyCrash = false 320 defer func() { 321 runtime.ReallyCrash = origReallyCrash 322 }() 323 324 ctx, cancel := context.WithCancel(context.Background()) 325 defer cancel() 326 327 withDeadline := func(handler http.Handler) http.Handler { 328 return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 329 handler.ServeHTTP(w, req.WithContext(ctx)) 330 }) 331 } 332 333 testDone := make(chan struct{}) 334 defer close(testDone) 335 ts := httptest.NewServer( 336 WithHTTPLogging( 337 withDeadline( 338 WithTimeoutForNonLongRunningRequests( 339 http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 340 // trigger the timeout 341 cancel() 342 // mutate request Headers 343 // Authorization filter does it for example 344 for { 345 select { 346 case <-testDone: 347 return 348 default: 349 req.Header.Set("Test", "post") 350 } 351 } 352 }), 353 func(r *http.Request, requestInfo *request.RequestInfo) bool { 354 return false 355 }, 356 ), 357 ), 358 ), 359 ) 360 defer ts.Close() 361 362 client := &http.Client{} 363 req, err := http.NewRequest(http.MethodPatch, ts.URL, nil) 364 if err != nil { 365 t.Fatal(err) 366 } 367 res, err := client.Do(req) 368 if err != nil { 369 t.Fatal(err) 370 } 371 if actual, expected := res.StatusCode, http.StatusGatewayTimeout; actual != expected { 372 t.Errorf("got status code %d; expected %d", actual, expected) 373 } 374 res.Body.Close() 375 } 376 377 func TestErrConnKilled(t *testing.T) { 378 var buf bytes.Buffer 379 klog.SetOutput(&buf) 380 klog.LogToStderr(false) 381 defer klog.LogToStderr(true) 382 383 handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 384 // this error must be ignored by the WithPanicRecovery handler 385 // it is thrown by WithTimeoutForNonLongRunningRequests handler when a response has been already sent to the client and the handler timed out 386 // panicking with http.ErrAbortHandler also suppresses logging of a stack trace to the server's error log and closes the underlying connection 387 w.Write([]byte("hello from the handler")) 388 panic(http.ErrAbortHandler) 389 }) 390 resolver := &request.RequestInfoFactory{ 391 APIPrefixes: sets.NewString("api", "apis"), 392 GrouplessAPIPrefixes: sets.NewString("api"), 393 } 394 395 ts := httptest.NewServer(WithPanicRecovery(handler, resolver)) 396 defer ts.Close() 397 398 _, err := http.Get(ts.URL) 399 if err == nil { 400 t.Fatal("expected to receive an error") 401 } 402 403 klog.Flush() 404 klog.SetOutput(&bytes.Buffer{}) // prevent further writes into buf 405 capturedOutput := buf.String() 406 407 // We don't expect stack trace from the panic to be included in the log. 408 if isStackTraceLoggedByRuntime(capturedOutput) { 409 t.Errorf("unexpected stack trace in log, actual = %v", capturedOutput) 410 } 411 // For the sake of simplicity and clarity this matches the full log line. 412 // This is not part of the Kubernetes API and could change. 413 if !strings.Contains(capturedOutput, `"Timeout or abort while handling" logger="UnhandledError" method="GET" URI="/" auditID=""`) { 414 t.Errorf("unexpected output captured actual = %v", capturedOutput) 415 } 416 } 417 418 type panicOnNonReuseTransport struct { 419 Transport http.RoundTripper 420 gotConnSeen bool 421 } 422 423 func (t *panicOnNonReuseTransport) RoundTrip(req *http.Request) (*http.Response, error) { 424 return t.Transport.RoundTrip(req) 425 } 426 427 func (t *panicOnNonReuseTransport) GotConn(info httptrace.GotConnInfo) { 428 if !t.gotConnSeen { 429 t.gotConnSeen = true 430 return 431 } 432 if !info.Reused { 433 panic(fmt.Sprintf("expected the connection to be reused, info %#v", info)) 434 } 435 } 436 437 // TestErrConnKilledHTTP2 check if HTTP/2 connection is not closed when an HTTP handler panics 438 // The net/http library recovers the panic and sends an HTTP/2 RST_STREAM. 439 func TestErrConnKilledHTTP2(t *testing.T) { 440 var buf bytes.Buffer 441 klog.SetOutput(&buf) 442 klog.LogToStderr(false) 443 defer klog.LogToStderr(true) 444 445 handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 446 // this error must be ignored by the WithPanicRecovery handler 447 // it is thrown by WithTimeoutForNonLongRunningRequests handler when a response has been already sent to the client and the handler timed out 448 // panicking with http.ErrAbortHandler also suppresses logging of a stack trace to the server's error log and closes the underlying connection 449 w.Write([]byte("hello from the handler")) 450 panic(http.ErrAbortHandler) 451 }) 452 resolver := &request.RequestInfoFactory{ 453 APIPrefixes: sets.NewString("api", "apis"), 454 GrouplessAPIPrefixes: sets.NewString("api"), 455 } 456 457 // test server 458 ts := httptest.NewUnstartedServer(WithPanicRecovery(handler, resolver)) 459 tsCert, err := tls.X509KeyPair(tsCrt, tsKey) 460 if err != nil { 461 t.Fatalf("backend: invalid x509/key pair: %v", err) 462 } 463 ts.TLS = &tls.Config{ 464 Certificates: []tls.Certificate{tsCert}, 465 NextProtos: []string{http2.NextProtoTLS}, 466 } 467 ts.StartTLS() 468 defer ts.Close() 469 470 newServerRequest := func(tr *panicOnNonReuseTransport) *http.Request { 471 req, _ := http.NewRequest("GET", fmt.Sprintf("https://127.0.0.1:%d", ts.Listener.Addr().(*net.TCPAddr).Port), nil) 472 trace := &httptrace.ClientTrace{ 473 GotConn: tr.GotConn, 474 } 475 return req.WithContext(httptrace.WithClientTrace(req.Context(), trace)) 476 } 477 478 // client 479 clientCACertPool := x509.NewCertPool() 480 clientCACertPool.AppendCertsFromPEM(tsCrt) 481 clientTLSConfig := &tls.Config{ 482 RootCAs: clientCACertPool, 483 NextProtos: []string{http2.NextProtoTLS}, 484 } 485 tr := &panicOnNonReuseTransport{} 486 client := &http.Client{} 487 tr.Transport = &http2.Transport{ 488 TLSClientConfig: clientTLSConfig, 489 } 490 client.Transport = tr 491 492 // act 493 _, err = client.Do(newServerRequest(tr)) 494 if err == nil { 495 t.Fatal("expected to receive an error") 496 } 497 498 klog.Flush() 499 klog.SetOutput(&bytes.Buffer{}) // prevent further writes into buf 500 capturedOutput := buf.String() 501 502 // We don't expect stack trace from the panic to be included in the log. 503 if isStackTraceLoggedByRuntime(capturedOutput) { 504 t.Errorf("unexpected stack trace in log, actual = %v", capturedOutput) 505 } 506 // For the sake of simplicity and clarity this matches the full log line. 507 // This is not part of the Kubernetes API and could change. 508 if !strings.Contains(capturedOutput, `"Timeout or abort while handling" logger="UnhandledError" method="GET" URI="/" auditID=""`) { 509 t.Errorf("unexpected output captured actual = %v", capturedOutput) 510 } 511 512 // make another req to the server 513 // the connection should be reused 514 // the client uses a custom transport that checks and panics when the con wasn't reused. 515 _, err = client.Do(newServerRequest(tr)) 516 if err == nil { 517 t.Fatal("expected to receive an error") 518 } 519 } 520 521 func TestResponseWriterDecorator(t *testing.T) { 522 decorator := &baseTimeoutWriter{ 523 w: &responsewriter.FakeResponseWriter{}, 524 } 525 var w http.ResponseWriter = decorator 526 527 if inner := w.(responsewriter.UserProvidedDecorator).Unwrap(); inner != decorator.w { 528 t.Errorf("Expected the decorator to return the inner http.ResponseWriter object") 529 } 530 } 531 532 func isStackTraceLoggedByRuntime(message string) bool { 533 // Check the captured output for the following patterns to find out if the 534 // stack trace is included in the log: 535 // - 'Observed a panic' (apimachinery runtime.go logs panic with this message) 536 // - 'goroutine 44 [running]:' (stack trace always starts with this) 537 if strings.Contains(message, "Observed a panic") && 538 strings.Contains(message, "goroutine") && 539 strings.Contains(message, "[running]:") { 540 return true 541 } 542 543 return false 544 } 545 546 var tsCrt = []byte(`-----BEGIN CERTIFICATE----- 547 MIIDTjCCAjagAwIBAgIJAJdcQEBN2CjoMA0GCSqGSIb3DQEBCwUAMFAxCzAJBgNV 548 BAYTAlBMMQ8wDQYDVQQIDAZQb2xhbmQxDzANBgNVBAcMBkdkYW5zazELMAkGA1UE 549 CgwCU0sxEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0yMDA5MjgxMTU1MjhaFw0zMDA5 550 MjYxMTU1MjhaMFAxCzAJBgNVBAYTAlBMMQ8wDQYDVQQIDAZQb2xhbmQxDzANBgNV 551 BAcMBkdkYW5zazELMAkGA1UECgwCU0sxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIw 552 DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMr6b/uTHkIDEd88x3t3jnroOVwh 553 jWMwZ6qXN2NV/If1L9FNvtoZzZi6yCDE1uLdD1kWZ0R2XOPEwUPn+Z8A/lg9kF8J 554 GloLCF8q+XeYp8aWRKzwtdi+MPaKFf0wsuxEEHU4pypFrszNY0yLRbWAbMtgBFy0 555 KhyNGahFO9V69cRHUj6EJ9kSBg0nG5bsypon2rinzKpUrzAEl2MbM3F34Zit5yOv 556 rYQcbDME+9XmOJPD97XBvMZCbmPnmpst3tX7ZhdKgSKtIjoYt+d//wtPMXOhrRzM 557 xcc6HuIHAovtB4kvZl5wvVU8ra8DKZviYyjfW36kQHo+yFwP3XXZFWezZi0CAwEA 558 AaMrMCkwCQYDVR0TBAIwADALBgNVHQ8EBAMCBaAwDwYDVR0RBAgwBocEfwAAATAN 559 BgkqhkiG9w0BAQsFAAOCAQEAMoAlhZ6yRwPmFL2ql9ZYWqaPu2NC4dXYV6kUIXUA 560 pG3IqFWb3L4ePkgYBMDlJGpAJcXlSSgEGiLj+3qQojawIMzzxmqr2KX888S5Mr+9 561 I1qseRwcftwYqV/ewJSWE90HJ21pb1ixA6bSRJLV7DyxO6zKsdVJ4xIvehZtGbux 562 0RTf+8zUx8z2Goy1GUztOIqfMRt1P1hlQG0uvYsGQM84HO4+YhFwejrGaj8ajpgF 563 uo3B8BVHeh57FNGE6C45NkFGHq3tkNLMdAa32Az8DDvPmsJuycf6vgIfBEQxLZSF 564 OUKrKmtfdFv4XrInqFUYBYp5GkL8SGM2wmv6aSw9Aju4lA== 565 -----END CERTIFICATE-----`) 566 567 var tsKey = []byte(`-----BEGIN PRIVATE KEY----- 568 MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDK+m/7kx5CAxHf 569 PMd7d4566DlcIY1jMGeqlzdjVfyH9S/RTb7aGc2YusggxNbi3Q9ZFmdEdlzjxMFD 570 5/mfAP5YPZBfCRpaCwhfKvl3mKfGlkSs8LXYvjD2ihX9MLLsRBB1OKcqRa7MzWNM 571 i0W1gGzLYARctCocjRmoRTvVevXER1I+hCfZEgYNJxuW7MqaJ9q4p8yqVK8wBJdj 572 GzNxd+GYrecjr62EHGwzBPvV5jiTw/e1wbzGQm5j55qbLd7V+2YXSoEirSI6GLfn 573 f/8LTzFzoa0czMXHOh7iBwKL7QeJL2ZecL1VPK2vAymb4mMo31t+pEB6PshcD911 574 2RVns2YtAgMBAAECggEAA2Qx0MtBeyrf9pHmZ1q1B7qvkqmA2kJpyQDjzQYXxRHE 575 rcOVx8EcnUupolqHmJzG798e9JbhsHCOJhtPIWf71++XZO8bAJwklKp8JpJnYzsJ 576 hLY0450x5jyiZ2uT4by1Za//owYtCID6AsJk9MZjivZcvEvKVFXLMvONL2DxkEj1 577 KaGQJh6/GT4jtNX07bW9+5w069KAAf+BNuqv8+Y/FseV3ovlpLTKjMV9xCCp9i62 578 PJs/hs5CW2X+JCE7OCLsAiu0JTpXYyHcLwYwnCONdvj6veiMWjRyNDr3ew5NeZNf 579 nGU4WX7mXjPd/1OvzJy6iyrBlAA63ZfFZYjWQnfsIQKBgQDmo3AMIr+9rE79NnaD 580 woQyO539YSO45KSM39/Xrp/NJVpOxtzgZrYo7O6f6kQ3S5zQOddy9Oj7gN3RXhZ7 581 Vi+Oja78ig7KUrqxcBiBGRsKZGm5CGdZ0EFd3rIEh4Qb+f+2c4f+6NWANb4kwvfq 582 K24c1o71+77lEVlzE2/L33K+mQKBgQDhTFr/f2e9gnRNX9bjF4p7DQI0RsFADjx0 583 jgJYHfm/lCIdH9vf6SmmvJv2E76Bqx9XVilhav/egqKO/wzJWHyNo2RFBXNqfwoF 584 UxRZKgqhcU52y2LKAYoTYfodktatZk74rinMDLmA6arnlAWQELk3Mx48DlND43Zc 585 DUHTKcJEtQKBgQDYdL1c9mPjnEqJxMqXwEAXcPJG8hr3lMaGXDoVjxL1EsBdvK9h 586 f6QoZq1RsiiRiMpEdnSotAfQutHzhA0vdeSuMnTvGJbm9Zu3mc+1oZ1KNJEwkh2F 587 Ijmm4rFKJPEs3IVMc8NHzrdJW6b3k2/e+yGduRR08e7nx0+e+7fpq+1hyQKBgHY9 588 l4h9+hkYjSdKhEG8yh3Ybu62r5eJoSremNZcLQXhnaHBZaj2+rgaRpP4OsRc5d71 589 RlRtTood72iy7KgDO6MuPGKJANDEiaLPvl8pVFj0WWS5S0iPVELl6dl5hheNGSck 590 aKVBjF3exKYzJlQ8oqgYuOZ18jcv+p9HCePkB6P9AoGBAJSYpkNDc/lnCpfIlxVw 591 n+VroX6QDIMZzC7BGiUSrmVsu6xEbI+8/C7ecN2oCZZLMj96EXe6j+np4zmkQezc 592 c1EwB7fNAiS0fWyE2RU6QAOZJ71bDpzQa4q4DxbOkYSybGPM/nqDRwovdjUnWeuM 593 +vrJUjAZAPHJcvos0iylnc8E 594 -----END PRIVATE KEY-----`)