github.com/hernad/nomad@v1.6.112/nomad/job_endpoint_hook_expose_check_test.go (about) 1 // Copyright (c) HashiCorp, Inc. 2 // SPDX-License-Identifier: MPL-2.0 3 4 package nomad 5 6 import ( 7 "testing" 8 9 "github.com/hernad/nomad/ci" 10 "github.com/hernad/nomad/nomad/structs" 11 "github.com/stretchr/testify/require" 12 ) 13 14 func TestJobExposeCheckHook_Name(t *testing.T) { 15 ci.Parallel(t) 16 17 require.Equal(t, "expose-check", new(jobExposeCheckHook).Name()) 18 } 19 20 func TestJobExposeCheckHook_tgUsesExposeCheck(t *testing.T) { 21 ci.Parallel(t) 22 23 t.Run("no check.expose", func(t *testing.T) { 24 require.False(t, tgUsesExposeCheck(&structs.TaskGroup{ 25 Services: []*structs.Service{{ 26 Checks: []*structs.ServiceCheck{{ 27 Expose: false, 28 }}, 29 }}, 30 })) 31 }) 32 33 t.Run("with check.expose", func(t *testing.T) { 34 require.True(t, tgUsesExposeCheck(&structs.TaskGroup{ 35 Services: []*structs.Service{{ 36 Checks: []*structs.ServiceCheck{{ 37 Expose: false, 38 }, { 39 Expose: true, 40 }}, 41 }}, 42 })) 43 }) 44 } 45 46 func TestJobExposeCheckHook_tgValidateUseOfBridgeMode(t *testing.T) { 47 ci.Parallel(t) 48 49 s1 := &structs.Service{ 50 Name: "s1", 51 Checks: []*structs.ServiceCheck{{ 52 Name: "s1-check1", 53 Type: "http", 54 PortLabel: "health", 55 Expose: true, 56 }}, 57 } 58 59 t.Run("no networks but no use of expose", func(t *testing.T) { 60 require.Nil(t, tgValidateUseOfBridgeMode(&structs.TaskGroup{ 61 Networks: make(structs.Networks, 0), 62 })) 63 }) 64 65 t.Run("no networks and uses expose", func(t *testing.T) { 66 require.EqualError(t, tgValidateUseOfBridgeMode(&structs.TaskGroup{ 67 Name: "g1", 68 Networks: make(structs.Networks, 0), 69 Services: []*structs.Service{s1}, 70 }), `group "g1" must specify one bridge network for exposing service check(s)`) 71 }) 72 73 t.Run("non-bridge network and uses expose", func(t *testing.T) { 74 require.EqualError(t, tgValidateUseOfBridgeMode(&structs.TaskGroup{ 75 Name: "g1", 76 Networks: structs.Networks{{ 77 Mode: "host", 78 }}, 79 Services: []*structs.Service{s1}, 80 }), `group "g1" must use bridge network for exposing service check(s)`) 81 }) 82 83 t.Run("bridge network uses expose", func(t *testing.T) { 84 require.Nil(t, tgValidateUseOfBridgeMode(&structs.TaskGroup{ 85 Name: "g1", 86 Networks: structs.Networks{{ 87 Mode: "bridge", 88 }}, 89 Services: []*structs.Service{s1}, 90 })) 91 }) 92 } 93 94 func TestJobExposeCheckHook_tgValidateUseOfCheckExpose(t *testing.T) { 95 ci.Parallel(t) 96 97 withCustomProxyTask := &structs.Service{ 98 Name: "s1", 99 Connect: &structs.ConsulConnect{ 100 SidecarTask: &structs.SidecarTask{Name: "custom"}, 101 }, 102 Checks: []*structs.ServiceCheck{{ 103 Name: "s1-check1", 104 Type: "http", 105 PortLabel: "health", 106 Expose: true, 107 }}, 108 } 109 110 t.Run("group-service uses custom proxy", func(t *testing.T) { 111 require.EqualError(t, tgValidateUseOfCheckExpose(&structs.TaskGroup{ 112 Name: "g1", 113 Services: []*structs.Service{withCustomProxyTask}, 114 }), `exposed service check g1->s1->s1-check1 requires use of sidecar_proxy`) 115 }) 116 117 t.Run("group-service uses custom proxy but no expose", func(t *testing.T) { 118 withCustomProxyTaskNoExpose := *withCustomProxyTask 119 withCustomProxyTask.Checks[0].Expose = false 120 require.Nil(t, tgValidateUseOfCheckExpose(&structs.TaskGroup{ 121 Name: "g1", 122 Services: []*structs.Service{&withCustomProxyTaskNoExpose}, 123 })) 124 }) 125 126 t.Run("task-service sets expose", func(t *testing.T) { 127 require.EqualError(t, tgValidateUseOfCheckExpose(&structs.TaskGroup{ 128 Name: "g1", 129 Tasks: []*structs.Task{{ 130 Name: "t1", 131 Services: []*structs.Service{{ 132 Name: "s2", 133 Checks: []*structs.ServiceCheck{{ 134 Name: "check1", 135 Type: "http", 136 Expose: true, 137 }}, 138 }}, 139 }}, 140 }), `exposed service check g1[t1]->s2->check1 is not a task-group service`) 141 }) 142 } 143 144 func TestJobExposeCheckHook_Validate(t *testing.T) { 145 ci.Parallel(t) 146 147 s1 := &structs.Service{ 148 Name: "s1", 149 Checks: []*structs.ServiceCheck{{ 150 Name: "s1-check1", 151 Type: "http", 152 Expose: true, 153 }}, 154 } 155 156 t.Run("double network", func(t *testing.T) { 157 warnings, err := new(jobExposeCheckHook).Validate(&structs.Job{ 158 TaskGroups: []*structs.TaskGroup{{ 159 Name: "g1", 160 Networks: structs.Networks{{ 161 Mode: "bridge", 162 }, { 163 Mode: "bridge", 164 }}, 165 Services: []*structs.Service{s1}, 166 }}, 167 }) 168 require.Empty(t, warnings) 169 require.EqualError(t, err, `group "g1" must specify one bridge network for exposing service check(s)`) 170 }) 171 172 t.Run("expose in service check", func(t *testing.T) { 173 warnings, err := new(jobExposeCheckHook).Validate(&structs.Job{ 174 TaskGroups: []*structs.TaskGroup{{ 175 Name: "g1", 176 Networks: structs.Networks{{ 177 Mode: "bridge", 178 }}, 179 Tasks: []*structs.Task{{ 180 Name: "t1", 181 Services: []*structs.Service{{ 182 Name: "s2", 183 Checks: []*structs.ServiceCheck{{ 184 Name: "s2-check1", 185 Type: "http", 186 Expose: true, 187 }}, 188 }}, 189 }}, 190 }}, 191 }) 192 require.Empty(t, warnings) 193 require.EqualError(t, err, `exposed service check g1[t1]->s2->s2-check1 is not a task-group service`) 194 }) 195 196 t.Run("ok", func(t *testing.T) { 197 warnings, err := new(jobExposeCheckHook).Validate(&structs.Job{ 198 TaskGroups: []*structs.TaskGroup{{ 199 Name: "g1", 200 Networks: structs.Networks{{ 201 Mode: "bridge", 202 }}, 203 Services: []*structs.Service{{ 204 Name: "s1", 205 Connect: &structs.ConsulConnect{ 206 SidecarService: &structs.ConsulSidecarService{}, 207 }, 208 Checks: []*structs.ServiceCheck{{ 209 Name: "check1", 210 Type: "http", 211 Expose: true, 212 }}, 213 }}, 214 Tasks: []*structs.Task{{ 215 Name: "t1", 216 Services: []*structs.Service{{ 217 Name: "s2", 218 Checks: []*structs.ServiceCheck{{ 219 Name: "s2-check1", 220 Type: "http", 221 Expose: false, 222 }}, 223 }}, 224 }}, 225 }}, 226 }) 227 require.Empty(t, warnings) 228 require.Nil(t, err) 229 }) 230 } 231 232 func TestJobExposeCheckHook_exposePathForCheck(t *testing.T) { 233 ci.Parallel(t) 234 235 const checkIdx = 0 236 237 t.Run("not expose compatible", func(t *testing.T) { 238 c := &structs.ServiceCheck{ 239 Type: "tcp", // not expose compatible 240 } 241 s := &structs.Service{ 242 Checks: []*structs.ServiceCheck{c}, 243 } 244 ePath, err := exposePathForCheck(&structs.TaskGroup{ 245 Services: []*structs.Service{s}, 246 }, s, c, checkIdx) 247 require.NoError(t, err) 248 require.Nil(t, ePath) 249 }) 250 251 t.Run("direct port", func(t *testing.T) { 252 c := &structs.ServiceCheck{ 253 Name: "check1", 254 Type: "http", 255 Path: "/health", 256 PortLabel: "hcPort", 257 } 258 s := &structs.Service{ 259 Name: "service1", 260 PortLabel: "4000", 261 Checks: []*structs.ServiceCheck{c}, 262 } 263 ePath, err := exposePathForCheck(&structs.TaskGroup{ 264 Name: "group1", 265 Services: []*structs.Service{s}, 266 }, s, c, checkIdx) 267 require.NoError(t, err) 268 require.Equal(t, &structs.ConsulExposePath{ 269 Path: "/health", 270 Protocol: "", // often blank, consul does the Right Thing 271 LocalPathPort: 4000, 272 ListenerPort: "hcPort", 273 }, ePath) 274 }) 275 276 t.Run("labeled port", func(t *testing.T) { 277 c := &structs.ServiceCheck{ 278 Name: "check1", 279 Type: "http", 280 Path: "/health", 281 PortLabel: "hcPort", 282 } 283 s := &structs.Service{ 284 Name: "service1", 285 PortLabel: "sPort", // port label indirection 286 Checks: []*structs.ServiceCheck{c}, 287 } 288 ePath, err := exposePathForCheck(&structs.TaskGroup{ 289 Name: "group1", 290 Services: []*structs.Service{s}, 291 Networks: structs.Networks{{ 292 Mode: "bridge", 293 DynamicPorts: []structs.Port{ 294 {Label: "sPort", Value: 4000}, 295 }, 296 }}, 297 }, s, c, checkIdx) 298 require.NoError(t, err) 299 require.Equal(t, &structs.ConsulExposePath{ 300 Path: "/health", 301 Protocol: "", 302 LocalPathPort: 4000, 303 ListenerPort: "hcPort", 304 }, ePath) 305 }) 306 307 t.Run("missing port", func(t *testing.T) { 308 c := &structs.ServiceCheck{ 309 Name: "check1", 310 Type: "http", 311 Path: "/health", 312 PortLabel: "hcPort", 313 } 314 s := &structs.Service{ 315 Name: "service1", 316 PortLabel: "sPort", // port label indirection 317 Checks: []*structs.ServiceCheck{c}, 318 } 319 _, err := exposePathForCheck(&structs.TaskGroup{ 320 Name: "group1", 321 Services: []*structs.Service{s}, 322 Networks: structs.Networks{{ 323 Mode: "bridge", 324 DynamicPorts: []structs.Port{ 325 // service declares "sPort", but does not exist 326 }, 327 }}, 328 }, s, c, checkIdx) 329 require.EqualError(t, err, `unable to determine local service port for service check group1->service1->check1`) 330 }) 331 332 t.Run("empty check port", func(t *testing.T) { 333 setup := func() (*structs.TaskGroup, *structs.Service, *structs.ServiceCheck) { 334 c := &structs.ServiceCheck{ 335 Name: "check1", 336 Type: "http", 337 Path: "/health", 338 } 339 s := &structs.Service{ 340 Name: "service1", 341 PortLabel: "9999", 342 Checks: []*structs.ServiceCheck{c}, 343 } 344 tg := &structs.TaskGroup{ 345 Name: "group1", 346 Services: []*structs.Service{s}, 347 Networks: structs.Networks{{ 348 Mode: "bridge", 349 DynamicPorts: []structs.Port{}, 350 }}, 351 } 352 return tg, s, c 353 } 354 355 tg, s, c := setup() 356 ePath, err := exposePathForCheck(tg, s, c, checkIdx) 357 require.NoError(t, err) 358 require.Len(t, tg.Networks[0].DynamicPorts, 1) 359 require.Equal(t, "default", tg.Networks[0].DynamicPorts[0].HostNetwork) 360 require.Equal(t, "svc_", tg.Networks[0].DynamicPorts[0].Label[0:4]) 361 require.Equal(t, &structs.ConsulExposePath{ 362 Path: "/health", 363 Protocol: "", 364 LocalPathPort: 9999, 365 ListenerPort: tg.Networks[0].DynamicPorts[0].Label, 366 }, ePath) 367 368 t.Run("deterministic generated port label", func(t *testing.T) { 369 tg2, s2, c2 := setup() 370 ePath2, err2 := exposePathForCheck(tg2, s2, c2, checkIdx) 371 require.NoError(t, err2) 372 require.Equal(t, ePath, ePath2) 373 }) 374 375 t.Run("unique on check index", func(t *testing.T) { 376 tg3, s3, c3 := setup() 377 ePath3, err3 := exposePathForCheck(tg3, s3, c3, checkIdx+1) 378 require.NoError(t, err3) 379 require.NotEqual(t, ePath.ListenerPort, ePath3.ListenerPort) 380 }) 381 }) 382 383 t.Run("missing network with no service check port label", func(t *testing.T) { 384 // this test ensures we do not try to manipulate the group network 385 // to inject an expose port if the group network does not exist 386 c := &structs.ServiceCheck{ 387 Name: "check1", 388 Type: "http", 389 Path: "/health", 390 PortLabel: "", // not set 391 Expose: true, // will require a service check port label 392 } 393 s := &structs.Service{ 394 Name: "service1", 395 Checks: []*structs.ServiceCheck{c}, 396 } 397 tg := &structs.TaskGroup{ 398 Name: "group1", 399 Services: []*structs.Service{s}, 400 Networks: nil, // not set, should cause validation error 401 } 402 ePath, err := exposePathForCheck(tg, s, c, checkIdx) 403 require.EqualError(t, err, `group "group1" must specify one bridge network for exposing service check(s)`) 404 require.Nil(t, ePath) 405 }) 406 } 407 408 func TestJobExposeCheckHook_containsExposePath(t *testing.T) { 409 ci.Parallel(t) 410 411 t.Run("contains path", func(t *testing.T) { 412 require.True(t, containsExposePath([]structs.ConsulExposePath{{ 413 Path: "/v2/health", 414 Protocol: "grpc", 415 LocalPathPort: 8080, 416 ListenerPort: "v2Port", 417 }, { 418 Path: "/health", 419 Protocol: "http", 420 LocalPathPort: 8080, 421 ListenerPort: "hcPort", 422 }}, structs.ConsulExposePath{ 423 Path: "/health", 424 Protocol: "http", 425 LocalPathPort: 8080, 426 ListenerPort: "hcPort", 427 })) 428 }) 429 430 t.Run("no such path", func(t *testing.T) { 431 require.False(t, containsExposePath([]structs.ConsulExposePath{{ 432 Path: "/v2/health", 433 Protocol: "grpc", 434 LocalPathPort: 8080, 435 ListenerPort: "v2Port", 436 }, { 437 Path: "/health", 438 Protocol: "http", 439 LocalPathPort: 8080, 440 ListenerPort: "hcPort", 441 }}, structs.ConsulExposePath{ 442 Path: "/v3/health", 443 Protocol: "http", 444 LocalPathPort: 8080, 445 ListenerPort: "hcPort", 446 })) 447 }) 448 } 449 450 func TestJobExposeCheckHook_serviceExposeConfig(t *testing.T) { 451 ci.Parallel(t) 452 453 t.Run("proxy is nil", func(t *testing.T) { 454 require.NotNil(t, serviceExposeConfig(&structs.Service{ 455 Connect: &structs.ConsulConnect{ 456 SidecarService: &structs.ConsulSidecarService{}, 457 }, 458 })) 459 }) 460 461 t.Run("expose is nil", func(t *testing.T) { 462 require.NotNil(t, serviceExposeConfig(&structs.Service{ 463 Connect: &structs.ConsulConnect{ 464 SidecarService: &structs.ConsulSidecarService{ 465 Proxy: &structs.ConsulProxy{}, 466 }, 467 }, 468 })) 469 }) 470 471 t.Run("expose pre-existing", func(t *testing.T) { 472 exposeConfig := serviceExposeConfig(&structs.Service{ 473 Connect: &structs.ConsulConnect{ 474 SidecarService: &structs.ConsulSidecarService{ 475 Proxy: &structs.ConsulProxy{ 476 Expose: &structs.ConsulExposeConfig{ 477 Paths: []structs.ConsulExposePath{{ 478 Path: "/health", 479 }}, 480 }, 481 }, 482 }, 483 }, 484 }) 485 require.NotNil(t, exposeConfig) 486 require.Equal(t, []structs.ConsulExposePath{{ 487 Path: "/health", 488 }}, exposeConfig.Paths) 489 }) 490 491 t.Run("append to paths is safe", func(t *testing.T) { 492 // double check that serviceExposeConfig(s).Paths can be appended to 493 // from a derived pointer without fear of the original underlying array 494 // pointer being lost 495 496 s := &structs.Service{ 497 Connect: &structs.ConsulConnect{ 498 SidecarService: &structs.ConsulSidecarService{ 499 Proxy: &structs.ConsulProxy{ 500 Expose: &structs.ConsulExposeConfig{ 501 Paths: []structs.ConsulExposePath{{ 502 Path: "/one", 503 }}, 504 }, 505 }, 506 }, 507 }, 508 } 509 510 exposeConfig := serviceExposeConfig(s) 511 exposeConfig.Paths = append(exposeConfig.Paths, 512 structs.ConsulExposePath{Path: "/two"}, 513 structs.ConsulExposePath{Path: "/three"}, 514 structs.ConsulExposePath{Path: "/four"}, 515 structs.ConsulExposePath{Path: "/five"}, 516 structs.ConsulExposePath{Path: "/six"}, 517 structs.ConsulExposePath{Path: "/seven"}, 518 structs.ConsulExposePath{Path: "/eight"}, 519 structs.ConsulExposePath{Path: "/nine"}, 520 ) 521 522 // works, because exposeConfig.Paths gets re-assigned into exposeConfig 523 // which is a pointer, meaning the field is modified also from the 524 // service struct's perspective 525 require.Equal(t, 9, len(s.Connect.SidecarService.Proxy.Expose.Paths)) 526 }) 527 } 528 529 func TestJobExposeCheckHook_checkIsExposable(t *testing.T) { 530 ci.Parallel(t) 531 532 t.Run("grpc", func(t *testing.T) { 533 require.True(t, checkIsExposable(&structs.ServiceCheck{ 534 Type: "grpc", 535 Path: "/health", 536 })) 537 require.True(t, checkIsExposable(&structs.ServiceCheck{ 538 Type: "gRPC", 539 Path: "/health", 540 })) 541 }) 542 543 t.Run("http", func(t *testing.T) { 544 require.True(t, checkIsExposable(&structs.ServiceCheck{ 545 Type: "http", 546 Path: "/health", 547 })) 548 require.True(t, checkIsExposable(&structs.ServiceCheck{ 549 Type: "HTTP", 550 Path: "/health", 551 })) 552 }) 553 554 t.Run("tcp", func(t *testing.T) { 555 require.False(t, checkIsExposable(&structs.ServiceCheck{ 556 Type: "tcp", 557 Path: "/health", 558 })) 559 }) 560 561 t.Run("no path slash prefix", func(t *testing.T) { 562 require.False(t, checkIsExposable(&structs.ServiceCheck{ 563 Type: "http", 564 Path: "health", 565 })) 566 }) 567 } 568 569 func TestJobExposeCheckHook_Mutate(t *testing.T) { 570 ci.Parallel(t) 571 572 t.Run("typical", func(t *testing.T) { 573 result, warnings, err := new(jobExposeCheckHook).Mutate(&structs.Job{ 574 TaskGroups: []*structs.TaskGroup{{ 575 Name: "group0", 576 Networks: structs.Networks{{ 577 Mode: "host", 578 }}, 579 }, { 580 Name: "group1", 581 Networks: structs.Networks{{ 582 Mode: "bridge", 583 }}, 584 Services: []*structs.Service{{ 585 Name: "service1", 586 PortLabel: "8000", 587 Checks: []*structs.ServiceCheck{{ 588 Name: "check1", 589 Type: "tcp", 590 PortLabel: "8100", 591 }, { 592 Name: "check2", 593 Type: "http", 594 PortLabel: "health", 595 Path: "/health", 596 Expose: true, 597 }, { 598 Name: "check3", 599 Type: "grpc", 600 PortLabel: "health", 601 Path: "/v2/health", 602 Expose: true, 603 }}, 604 Connect: &structs.ConsulConnect{ 605 SidecarService: &structs.ConsulSidecarService{ 606 Proxy: &structs.ConsulProxy{ 607 Expose: &structs.ConsulExposeConfig{ 608 Paths: []structs.ConsulExposePath{{ 609 Path: "/pre-existing", 610 Protocol: "http", 611 LocalPathPort: 9000, 612 ListenerPort: "otherPort", 613 }}}}}}}, { 614 Name: "service2", 615 PortLabel: "3000", 616 Checks: []*structs.ServiceCheck{{ 617 Name: "check1", 618 Type: "grpc", 619 Protocol: "http2", 620 Path: "/ok", 621 PortLabel: "health", 622 Expose: true, 623 }}, 624 Connect: &structs.ConsulConnect{ 625 SidecarService: &structs.ConsulSidecarService{ 626 Proxy: &structs.ConsulProxy{}, 627 }, 628 }, 629 }}}}, 630 }) 631 632 require.NoError(t, err) 633 require.Empty(t, warnings) 634 require.Equal(t, []structs.ConsulExposePath{{ 635 Path: "/pre-existing", 636 LocalPathPort: 9000, 637 Protocol: "http", 638 ListenerPort: "otherPort", 639 }, { 640 Path: "/health", 641 LocalPathPort: 8000, 642 ListenerPort: "health", 643 }, { 644 Path: "/v2/health", 645 LocalPathPort: 8000, 646 ListenerPort: "health", 647 }}, result.TaskGroups[1].Services[0].Connect.SidecarService.Proxy.Expose.Paths) 648 require.Equal(t, []structs.ConsulExposePath{{ 649 Path: "/ok", 650 LocalPathPort: 3000, 651 Protocol: "http2", 652 ListenerPort: "health", 653 }}, result.TaskGroups[1].Services[1].Connect.SidecarService.Proxy.Expose.Paths) 654 }) 655 }