github.com/arthur-befumo/witchcraft-go-server@v1.12.0/integration/health_test.go (about) 1 // Copyright (c) 2018 Palantir Technologies. All rights reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package integration 16 17 import ( 18 "context" 19 "encoding/json" 20 "errors" 21 "fmt" 22 "io" 23 "io/ioutil" 24 "net/http" 25 "sync" 26 "testing" 27 "time" 28 29 "github.com/palantir/pkg/httpserver" 30 "github.com/palantir/witchcraft-go-server/config" 31 "github.com/palantir/witchcraft-go-server/conjure/witchcraft/api/health" 32 "github.com/palantir/witchcraft-go-server/status" 33 "github.com/palantir/witchcraft-go-server/status/health/periodic" 34 "github.com/palantir/witchcraft-go-server/status/reporter" 35 "github.com/palantir/witchcraft-go-server/witchcraft" 36 "github.com/palantir/witchcraft-go-server/witchcraft/refreshable" 37 "github.com/stretchr/testify/assert" 38 "github.com/stretchr/testify/require" 39 ) 40 41 // TestAddHealthCheckSources verifies that custom health check sources report via the health endpoint. 42 func TestAddHealthCheckSources(t *testing.T) { 43 port, err := httpserver.AvailablePort() 44 require.NoError(t, err) 45 server, serverErr, cleanup := createAndRunCustomTestServer(t, port, port, nil, ioutil.Discard, func(t *testing.T, initFn witchcraft.InitFunc, installCfg config.Install, logOutputBuffer io.Writer) *witchcraft.Server { 46 return createTestServer(t, initFn, installCfg, logOutputBuffer).WithHealth(healthCheckWithType{typ: "FOO"}, healthCheckWithType{typ: "BAR"}) 47 }) 48 49 defer func() { 50 require.NoError(t, server.Close()) 51 }() 52 defer cleanup() 53 54 resp, err := testServerClient().Get(fmt.Sprintf("https://localhost:%d/%s/%s", port, basePath, status.HealthEndpoint)) 55 require.NoError(t, err) 56 57 bytes, err := ioutil.ReadAll(resp.Body) 58 require.NoError(t, err) 59 60 var healthResults health.HealthStatus 61 err = json.Unmarshal(bytes, &healthResults) 62 require.NoError(t, err) 63 assert.Equal(t, health.HealthStatus{ 64 Checks: map[health.CheckType]health.HealthCheckResult{ 65 health.CheckType("FOO"): { 66 Type: health.CheckType("FOO"), 67 State: health.HealthStateHealthy, 68 Message: nil, 69 Params: make(map[string]interface{}), 70 }, 71 health.CheckType("BAR"): { 72 Type: health.CheckType("BAR"), 73 State: health.HealthStateHealthy, 74 Message: nil, 75 Params: make(map[string]interface{}), 76 }, 77 health.CheckType("CONFIG_RELOAD"): { 78 Type: health.CheckType("CONFIG_RELOAD"), 79 State: health.HealthStateHealthy, 80 Params: make(map[string]interface{}), 81 }, 82 health.CheckType("SERVER_STATUS"): { 83 Type: health.CheckType("SERVER_STATUS"), 84 State: health.HealthStateHealthy, 85 Message: nil, 86 Params: make(map[string]interface{}), 87 }, 88 }, 89 }, healthResults) 90 91 select { 92 case err := <-serverErr: 93 require.NoError(t, err) 94 default: 95 } 96 } 97 98 // TestHealthReporter verifies the behavior of the reporter package. 99 // We create 4 health components, flip their health/unhealthy states, and ensure the aggregated states 100 // returned by the health endpoint reflect what we have set. 101 func TestHealthReporter(t *testing.T) { 102 healthReporter := reporter.NewHealthReporter() 103 104 port, err := httpserver.AvailablePort() 105 require.NoError(t, err) 106 server, serverErr, cleanup := createAndRunCustomTestServer(t, port, port, nil, ioutil.Discard, func(t *testing.T, initFn witchcraft.InitFunc, installCfg config.Install, logOutputBuffer io.Writer) *witchcraft.Server { 107 return createTestServer(t, initFn, installCfg, logOutputBuffer).WithHealth(healthReporter) 108 }) 109 110 defer func() { 111 require.NoError(t, server.Close()) 112 }() 113 defer cleanup() 114 115 // Initialize health components and set their health 116 healthyComponents := []string{"COMPONENT_A", "COMPONENT_B"} 117 unhealthyComponents := []string{"COMPONENT_C", "COMPONENT_D"} 118 errString := "Something failed" 119 var wg sync.WaitGroup 120 wg.Add(len(healthyComponents) + len(unhealthyComponents)) 121 for _, n := range healthyComponents { 122 go func(healthReporter reporter.HealthReporter, name string) { 123 defer wg.Done() 124 component, err := healthReporter.InitializeHealthComponent(name) 125 if err != nil { 126 panic(fmt.Errorf("failed to initialize %s health reporter: %v", name, err)) 127 } 128 if component.Status() != reporter.StartingState { 129 panic(fmt.Errorf("expected reporter to be in REPAIRING before being marked healthy, got %s", component.Status())) 130 } 131 component.Healthy() 132 if component.Status() != reporter.HealthyState { 133 panic(fmt.Errorf("expected reporter to be in HEALTHY after being marked healthy, got %s", component.Status())) 134 } 135 }(healthReporter, n) 136 } 137 for _, n := range unhealthyComponents { 138 go func(healthReporter reporter.HealthReporter, name string) { 139 defer wg.Done() 140 component, err := healthReporter.InitializeHealthComponent(name) 141 if err != nil { 142 panic(fmt.Errorf("failed to initialize %s health reporter: %v", name, err)) 143 } 144 if component.Status() != reporter.StartingState { 145 panic(fmt.Errorf("expected reporter to be in REPAIRING before being marked healthy, got %s", component.Status())) 146 } 147 component.Error(errors.New(errString)) 148 if component.Status() != reporter.ErrorState { 149 panic(fmt.Errorf("expected reporter to be in ERROR after being marked with error, got %s", component.Status())) 150 } 151 }(healthReporter, n) 152 } 153 wg.Wait() 154 155 // Validate GetHealthComponent 156 component, ok := healthReporter.GetHealthComponent(healthyComponents[0]) 157 assert.True(t, ok) 158 assert.Equal(t, reporter.HealthyState, component.Status()) 159 160 // Validate health 161 resp, err := testServerClient().Get(fmt.Sprintf("https://localhost:%d/%s/%s", port, basePath, status.HealthEndpoint)) 162 require.NoError(t, err) 163 164 bytes, err := ioutil.ReadAll(resp.Body) 165 require.NoError(t, err) 166 167 var healthResults health.HealthStatus 168 err = json.Unmarshal(bytes, &healthResults) 169 require.NoError(t, err) 170 assert.Equal(t, health.HealthStatus{ 171 Checks: map[health.CheckType]health.HealthCheckResult{ 172 health.CheckType("COMPONENT_A"): { 173 Type: health.CheckType("COMPONENT_A"), 174 State: reporter.HealthyState, 175 Message: nil, 176 Params: make(map[string]interface{}), 177 }, 178 health.CheckType("COMPONENT_B"): { 179 Type: health.CheckType("COMPONENT_B"), 180 State: reporter.HealthyState, 181 Message: nil, 182 Params: make(map[string]interface{}), 183 }, 184 health.CheckType("COMPONENT_C"): { 185 Type: health.CheckType("COMPONENT_C"), 186 State: reporter.ErrorState, 187 Message: &errString, 188 Params: make(map[string]interface{}), 189 }, 190 health.CheckType("COMPONENT_D"): { 191 Type: health.CheckType("COMPONENT_D"), 192 State: reporter.ErrorState, 193 Message: &errString, 194 Params: make(map[string]interface{}), 195 }, 196 health.CheckType("CONFIG_RELOAD"): { 197 Type: health.CheckType("CONFIG_RELOAD"), 198 State: health.HealthStateHealthy, 199 Params: make(map[string]interface{}), 200 }, 201 health.CheckType("SERVER_STATUS"): { 202 Type: health.CheckType("SERVER_STATUS"), 203 State: reporter.HealthyState, 204 Params: make(map[string]interface{}), 205 }, 206 }, 207 }, healthResults) 208 209 select { 210 case err := <-serverErr: 211 require.NoError(t, err) 212 default: 213 } 214 } 215 216 // TestPeriodicHealthSource tests that basic periodic healthcheck wiring works properly. Unit testing covers the grace 217 // period logic - this test covers the plumbing. 218 func TestPeriodicHealthSource(t *testing.T) { 219 inputSource := periodic.Source{ 220 Checks: map[health.CheckType]periodic.CheckFunc{ 221 "HEALTHY_CHECK": func(ctx context.Context) *health.HealthCheckResult { 222 return &health.HealthCheckResult{ 223 Type: "HEALTHY_CHECK", 224 State: health.HealthStateHealthy, 225 } 226 }, 227 "ERROR_CHECK": func(ctx context.Context) *health.HealthCheckResult { 228 return &health.HealthCheckResult{ 229 Type: "ERROR_CHECK", 230 State: health.HealthStateError, 231 Message: stringPtr("something went wrong"), 232 Params: map[string]interface{}{"foo": "bar"}, 233 } 234 }, 235 }, 236 } 237 expectedStatus := health.HealthStatus{Checks: map[health.CheckType]health.HealthCheckResult{ 238 "HEALTHY_CHECK": { 239 Type: "HEALTHY_CHECK", 240 State: health.HealthStateHealthy, 241 Message: nil, 242 Params: make(map[string]interface{}), 243 }, 244 "ERROR_CHECK": { 245 Type: "ERROR_CHECK", 246 State: health.HealthStateError, 247 Message: stringPtr("No successful checks during 1m0s grace period: something went wrong"), 248 Params: map[string]interface{}{"foo": "bar"}, 249 }, 250 health.CheckType("CONFIG_RELOAD"): { 251 Type: health.CheckType("CONFIG_RELOAD"), 252 State: health.HealthStateHealthy, 253 Params: make(map[string]interface{}), 254 }, 255 health.CheckType("SERVER_STATUS"): { 256 Type: health.CheckType("SERVER_STATUS"), 257 State: health.HealthStateHealthy, 258 Message: nil, 259 Params: make(map[string]interface{}), 260 }, 261 }} 262 periodicHealthCheckSource := periodic.FromHealthCheckSource(context.Background(), time.Second*60, time.Millisecond*1, inputSource) 263 264 port, err := httpserver.AvailablePort() 265 require.NoError(t, err) 266 server, serverErr, cleanup := createAndRunCustomTestServer(t, port, port, nil, ioutil.Discard, func(t *testing.T, initFn witchcraft.InitFunc, installCfg config.Install, logOutputBuffer io.Writer) *witchcraft.Server { 267 return createTestServer(t, initFn, installCfg, logOutputBuffer).WithHealth(periodicHealthCheckSource) 268 }) 269 270 defer func() { 271 require.NoError(t, server.Close()) 272 }() 273 defer cleanup() 274 275 // Wait for checks to run at least once 276 time.Sleep(5 * time.Millisecond) 277 278 resp, err := testServerClient().Get(fmt.Sprintf("https://localhost:%d/%s/%s", port, basePath, status.HealthEndpoint)) 279 require.NoError(t, err) 280 281 bytes, err := ioutil.ReadAll(resp.Body) 282 require.NoError(t, err) 283 284 var healthResults health.HealthStatus 285 err = json.Unmarshal(bytes, &healthResults) 286 require.NoError(t, err) 287 assert.Equal(t, expectedStatus, healthResults) 288 289 select { 290 case err := <-serverErr: 291 require.NoError(t, err) 292 default: 293 } 294 } 295 296 // TestHealthSharedSecret verifies that a non-empty health check shared secret is required by the endpoint when configured. 297 // If the secret is not provided or is incorrect, the endpoint returns 401 Unauthorized. 298 func TestHealthSharedSecret(t *testing.T) { 299 port, err := httpserver.AvailablePort() 300 require.NoError(t, err) 301 server, serverErr, cleanup := createAndRunCustomTestServer(t, port, port, nil, ioutil.Discard, func(t *testing.T, initFn witchcraft.InitFunc, installCfg config.Install, logOutputBuffer io.Writer) *witchcraft.Server { 302 return createTestServer(t, initFn, installCfg, logOutputBuffer). 303 WithHealth(emptyHealthCheckSource{}). 304 WithDisableGoRuntimeMetrics(). 305 WithRuntimeConfig(config.Runtime{ 306 HealthChecks: config.HealthChecksConfig{ 307 SharedSecret: "top-secret", 308 }, 309 }) 310 }) 311 312 defer func() { 313 require.NoError(t, server.Close()) 314 }() 315 defer cleanup() 316 317 client := testServerClient() 318 request, err := http.NewRequest(http.MethodGet, fmt.Sprintf("https://localhost:%d/%s/%s", port, basePath, status.HealthEndpoint), nil) 319 require.NoError(t, err) 320 request.Header.Set("Authorization", "Bearer top-secret") 321 resp, err := client.Do(request) 322 require.NoError(t, err) 323 require.Equal(t, http.StatusOK, resp.StatusCode) 324 325 bytes, err := ioutil.ReadAll(resp.Body) 326 require.NoError(t, err) 327 328 var healthResults health.HealthStatus 329 err = json.Unmarshal(bytes, &healthResults) 330 require.NoError(t, err) 331 assert.Equal(t, health.HealthStatus{ 332 Checks: map[health.CheckType]health.HealthCheckResult{ 333 health.CheckType("CONFIG_RELOAD"): { 334 Type: health.CheckType("CONFIG_RELOAD"), 335 State: health.HealthStateHealthy, 336 Params: make(map[string]interface{}), 337 }, 338 health.CheckType("SERVER_STATUS"): { 339 Type: health.CheckType("SERVER_STATUS"), 340 State: reporter.HealthyState, 341 Params: make(map[string]interface{}), 342 }, 343 }, 344 }, healthResults) 345 346 // bad header should return 401 347 request.Header.Set("Authorization", fmt.Sprintf("Bearer %s", "bad-secret")) 348 resp, err = client.Do(request) 349 require.NoError(t, err) 350 require.Equal(t, http.StatusUnauthorized, resp.StatusCode) 351 352 select { 353 case err := <-serverErr: 354 require.NoError(t, err) 355 default: 356 } 357 } 358 359 // TestRuntimeConfigReloadHealth verifies that runtime configuration that is invalid when strict unmarshal mode is true 360 // does not produces an error health check if strict unmarshal mode is not specified (since default value is false). 361 func TestRuntimeConfigReloadHealthWithStrictUnmarshalFalse(t *testing.T) { 362 port, err := httpserver.AvailablePort() 363 require.NoError(t, err) 364 365 validCfgYML := `logging: 366 level: info 367 ` 368 invalidCfgYML := ` 369 invalid-key: invalid-value 370 ` 371 runtimeConfigRefreshable := refreshable.NewDefaultRefreshable([]byte(validCfgYML)) 372 server, serverErr, cleanup := createAndRunCustomTestServer(t, port, port, nil, ioutil.Discard, func(t *testing.T, initFn witchcraft.InitFunc, installCfg config.Install, logOutputBuffer io.Writer) *witchcraft.Server { 373 return createTestServer(t, initFn, installCfg, logOutputBuffer). 374 WithRuntimeConfigProvider(runtimeConfigRefreshable). 375 WithDisableGoRuntimeMetrics() 376 }) 377 378 defer func() { 379 require.NoError(t, server.Close()) 380 }() 381 defer cleanup() 382 383 client := testServerClient() 384 request, err := http.NewRequest(http.MethodGet, fmt.Sprintf("https://localhost:%d/%s/%s", port, basePath, status.HealthEndpoint), nil) 385 require.NoError(t, err) 386 resp, err := client.Do(request) 387 require.NoError(t, err) 388 389 bytes, err := ioutil.ReadAll(resp.Body) 390 require.NoError(t, err) 391 392 var healthResults health.HealthStatus 393 err = json.Unmarshal(bytes, &healthResults) 394 require.NoError(t, err) 395 assert.Equal(t, health.HealthStatus{ 396 Checks: map[health.CheckType]health.HealthCheckResult{ 397 health.CheckType("CONFIG_RELOAD"): { 398 Type: health.CheckType("CONFIG_RELOAD"), 399 State: health.HealthStateHealthy, 400 Params: make(map[string]interface{}), 401 }, 402 health.CheckType("SERVER_STATUS"): { 403 Type: health.CheckType("SERVER_STATUS"), 404 State: reporter.HealthyState, 405 Params: make(map[string]interface{}), 406 }, 407 }, 408 }, healthResults) 409 410 // write invalid runtime config and observe health check go unhealthy 411 err = runtimeConfigRefreshable.Update([]byte(invalidCfgYML)) 412 require.NoError(t, err) 413 time.Sleep(500 * time.Millisecond) 414 415 request, err = http.NewRequest(http.MethodGet, fmt.Sprintf("https://localhost:%d/%s/%s", port, basePath, status.HealthEndpoint), nil) 416 require.NoError(t, err) 417 resp, err = client.Do(request) 418 require.NoError(t, err) 419 420 bytes, err = ioutil.ReadAll(resp.Body) 421 require.NoError(t, err) 422 423 err = json.Unmarshal(bytes, &healthResults) 424 require.NoError(t, err) 425 assert.Equal(t, health.HealthStatus{ 426 Checks: map[health.CheckType]health.HealthCheckResult{ 427 health.CheckType("CONFIG_RELOAD"): { 428 Type: health.CheckType("CONFIG_RELOAD"), 429 State: health.HealthStateHealthy, 430 Params: make(map[string]interface{}), 431 }, 432 health.CheckType("SERVER_STATUS"): { 433 Type: health.CheckType("SERVER_STATUS"), 434 State: reporter.HealthyState, 435 Params: make(map[string]interface{}), 436 }, 437 }, 438 }, healthResults) 439 440 select { 441 case err := <-serverErr: 442 require.NoError(t, err) 443 default: 444 } 445 } 446 447 // TestRuntimeConfigReloadHealth verifies that runtime configuration that is invalid when strict unmarshal mode is true 448 // produces an error health check. 449 func TestRuntimeConfigReloadHealthWithStrictUnmarshalTrue(t *testing.T) { 450 port, err := httpserver.AvailablePort() 451 require.NoError(t, err) 452 453 validCfgYML := `logging: 454 level: info 455 ` 456 invalidCfgYML := ` 457 invalid-key: invalid-value 458 ` 459 runtimeConfigRefreshable := refreshable.NewDefaultRefreshable([]byte(validCfgYML)) 460 server, serverErr, cleanup := createAndRunCustomTestServer(t, port, port, nil, ioutil.Discard, func(t *testing.T, initFn witchcraft.InitFunc, installCfg config.Install, logOutputBuffer io.Writer) *witchcraft.Server { 461 return createTestServer(t, initFn, installCfg, logOutputBuffer). 462 WithRuntimeConfigProvider(runtimeConfigRefreshable). 463 WithDisableGoRuntimeMetrics(). 464 WithStrictUnmarshalConfig() 465 }) 466 467 defer func() { 468 require.NoError(t, server.Close()) 469 }() 470 defer cleanup() 471 472 client := testServerClient() 473 request, err := http.NewRequest(http.MethodGet, fmt.Sprintf("https://localhost:%d/%s/%s", port, basePath, status.HealthEndpoint), nil) 474 require.NoError(t, err) 475 resp, err := client.Do(request) 476 require.NoError(t, err) 477 478 bytes, err := ioutil.ReadAll(resp.Body) 479 require.NoError(t, err) 480 481 var healthResults health.HealthStatus 482 err = json.Unmarshal(bytes, &healthResults) 483 require.NoError(t, err) 484 assert.Equal(t, health.HealthStatus{ 485 Checks: map[health.CheckType]health.HealthCheckResult{ 486 health.CheckType("CONFIG_RELOAD"): { 487 Type: health.CheckType("CONFIG_RELOAD"), 488 State: health.HealthStateHealthy, 489 Params: make(map[string]interface{}), 490 }, 491 health.CheckType("SERVER_STATUS"): { 492 Type: health.CheckType("SERVER_STATUS"), 493 State: reporter.HealthyState, 494 Params: make(map[string]interface{}), 495 }, 496 }, 497 }, healthResults) 498 499 // write invalid runtime config and observe health check go unhealthy 500 err = runtimeConfigRefreshable.Update([]byte(invalidCfgYML)) 501 require.NoError(t, err) 502 time.Sleep(500 * time.Millisecond) 503 504 request, err = http.NewRequest(http.MethodGet, fmt.Sprintf("https://localhost:%d/%s/%s", port, basePath, status.HealthEndpoint), nil) 505 require.NoError(t, err) 506 resp, err = client.Do(request) 507 require.NoError(t, err) 508 509 bytes, err = ioutil.ReadAll(resp.Body) 510 require.NoError(t, err) 511 512 err = json.Unmarshal(bytes, &healthResults) 513 require.NoError(t, err) 514 assert.Equal(t, health.HealthStatus{ 515 Checks: map[health.CheckType]health.HealthCheckResult{ 516 health.CheckType("CONFIG_RELOAD"): { 517 Type: health.CheckType("CONFIG_RELOAD"), 518 State: health.HealthStateError, 519 Params: make(map[string]interface{}), 520 Message: stringPtr("Refreshable validation failed, please look at service logs for more information."), 521 }, 522 health.CheckType("SERVER_STATUS"): { 523 Type: health.CheckType("SERVER_STATUS"), 524 State: reporter.HealthyState, 525 Params: make(map[string]interface{}), 526 }, 527 }, 528 }, healthResults) 529 530 select { 531 case err := <-serverErr: 532 require.NoError(t, err) 533 default: 534 } 535 } 536 537 type emptyHealthCheckSource struct{} 538 539 func (emptyHealthCheckSource) HealthStatus(ctx context.Context) health.HealthStatus { 540 return health.HealthStatus{} 541 } 542 543 type healthCheckWithType struct { 544 typ health.CheckType 545 } 546 547 func (cwt healthCheckWithType) HealthStatus(_ context.Context) health.HealthStatus { 548 return health.HealthStatus{ 549 Checks: map[health.CheckType]health.HealthCheckResult{ 550 cwt.typ: { 551 Type: cwt.typ, 552 State: health.HealthStateHealthy, 553 Message: nil, 554 Params: make(map[string]interface{}), 555 }, 556 }, 557 } 558 } 559 560 func stringPtr(s string) *string { 561 return &s 562 }