github.com/cilium/cilium@v1.16.2/pkg/bgpv1/manager/reconciler/service_test.go (about) 1 // SPDX-License-Identifier: Apache-2.0 2 // Copyright Authors of Cilium 3 4 package reconciler 5 6 import ( 7 "context" 8 "net/netip" 9 "testing" 10 11 "github.com/stretchr/testify/require" 12 meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 13 "k8s.io/utils/ptr" 14 15 "github.com/cilium/cilium/pkg/bgpv1/manager/instance" 16 "github.com/cilium/cilium/pkg/bgpv1/manager/store" 17 "github.com/cilium/cilium/pkg/bgpv1/types" 18 cmtypes "github.com/cilium/cilium/pkg/clustermesh/types" 19 "github.com/cilium/cilium/pkg/k8s" 20 v2api "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2" 21 v2alpha1api "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2alpha1" 22 "github.com/cilium/cilium/pkg/k8s/resource" 23 slim_corev1 "github.com/cilium/cilium/pkg/k8s/slim/k8s/api/core/v1" 24 slim_metav1 "github.com/cilium/cilium/pkg/k8s/slim/k8s/apis/meta/v1" 25 ) 26 27 func TestServiceReconcilerWithLoadBalancer(t *testing.T) { 28 blueSelector := slim_metav1.LabelSelector{MatchLabels: map[string]string{"color": "blue"}} 29 redSelector := slim_metav1.LabelSelector{MatchLabels: map[string]string{"color": "red"}} 30 svc1Name := resource.Key{Name: "svc-1", Namespace: "default"} 31 svc1NonDefaultName := resource.Key{Name: "svc-1", Namespace: "non-default"} 32 svc2NonDefaultName := resource.Key{Name: "svc-2", Namespace: "non-default"} 33 ingressV4 := "192.168.0.1" 34 ingressV4_2 := "192.168.0.2" 35 ingressV4Prefix := ingressV4 + "/32" 36 ingressV4Prefix_2 := ingressV4_2 + "/32" 37 ingressV6 := "fd00:192:168::1" 38 ingressV6Prefix := ingressV6 + "/128" 39 40 svc1 := &slim_corev1.Service{ 41 ObjectMeta: slim_metav1.ObjectMeta{ 42 Name: svc1Name.Name, 43 Namespace: svc1Name.Namespace, 44 Labels: blueSelector.MatchLabels, 45 }, 46 Spec: slim_corev1.ServiceSpec{ 47 Type: slim_corev1.ServiceTypeLoadBalancer, 48 }, 49 Status: slim_corev1.ServiceStatus{ 50 LoadBalancer: slim_corev1.LoadBalancerStatus{ 51 Ingress: []slim_corev1.LoadBalancerIngress{ 52 { 53 IP: ingressV4, 54 }, 55 }, 56 }, 57 }, 58 } 59 60 svc1TwoIngress := svc1.DeepCopy() 61 svc1TwoIngress.Status.LoadBalancer.Ingress = 62 append(svc1TwoIngress.Status.LoadBalancer.Ingress, 63 slim_corev1.LoadBalancerIngress{IP: ingressV6}) 64 65 svc1RedLabel := svc1.DeepCopy() 66 svc1RedLabel.ObjectMeta.Labels = redSelector.MatchLabels 67 68 svc1NonDefault := svc1.DeepCopy() 69 svc1NonDefault.Namespace = svc1NonDefaultName.Namespace 70 svc1NonDefault.Status.LoadBalancer.Ingress[0] = slim_corev1.LoadBalancerIngress{IP: ingressV4_2} 71 72 svc1NonLB := svc1.DeepCopy() 73 svc1NonLB.Spec.Type = slim_corev1.ServiceTypeClusterIP 74 75 svc1ETPLocal := svc1.DeepCopy() 76 svc1ETPLocal.Spec.ExternalTrafficPolicy = slim_corev1.ServiceExternalTrafficPolicyLocal 77 78 svc1ETPLocalTwoIngress := svc1TwoIngress.DeepCopy() 79 svc1ETPLocalTwoIngress.Spec.ExternalTrafficPolicy = slim_corev1.ServiceExternalTrafficPolicyLocal 80 81 svc1IPv6ETPLocal := svc1.DeepCopy() 82 svc1IPv6ETPLocal.Status.LoadBalancer.Ingress[0] = slim_corev1.LoadBalancerIngress{IP: ingressV6} 83 svc1IPv6ETPLocal.Spec.ExternalTrafficPolicy = slim_corev1.ServiceExternalTrafficPolicyLocal 84 85 svc1LbClass := svc1.DeepCopy() 86 svc1LbClass.Spec.LoadBalancerClass = ptr.To[string](v2alpha1api.BGPLoadBalancerClass) 87 88 svc1UnsupportedClass := svc1LbClass.DeepCopy() 89 svc1UnsupportedClass.Spec.LoadBalancerClass = ptr.To[string]("io.vendor/unsupported-class") 90 91 svc2NonDefault := &slim_corev1.Service{ 92 ObjectMeta: slim_metav1.ObjectMeta{ 93 Name: svc2NonDefaultName.Name, 94 Namespace: svc2NonDefaultName.Namespace, 95 Labels: blueSelector.MatchLabels, 96 }, 97 Spec: slim_corev1.ServiceSpec{ 98 Type: slim_corev1.ServiceTypeLoadBalancer, 99 }, 100 Status: slim_corev1.ServiceStatus{ 101 LoadBalancer: slim_corev1.LoadBalancerStatus{ 102 Ingress: []slim_corev1.LoadBalancerIngress{ 103 { 104 IP: ingressV4_2, 105 }, 106 }, 107 }, 108 }, 109 } 110 111 eps1IPv4Local := &k8s.Endpoints{ 112 ObjectMeta: slim_metav1.ObjectMeta{ 113 Name: "svc-1-ipv4", 114 Namespace: "default", 115 }, 116 EndpointSliceID: k8s.EndpointSliceID{ 117 ServiceID: k8s.ServiceID{ 118 Name: "svc-1", 119 Namespace: "default", 120 }, 121 EndpointSliceName: "svc-1-ipv4", 122 }, 123 Backends: map[cmtypes.AddrCluster]*k8s.Backend{ 124 cmtypes.MustParseAddrCluster("10.0.0.1"): { 125 NodeName: "node1", 126 }, 127 }, 128 } 129 130 eps1IPv4LocalTerminating := &k8s.Endpoints{ 131 ObjectMeta: slim_metav1.ObjectMeta{ 132 Name: "svc-1-ipv4", 133 Namespace: "default", 134 }, 135 EndpointSliceID: k8s.EndpointSliceID{ 136 ServiceID: k8s.ServiceID{ 137 Name: "svc-1", 138 Namespace: "default", 139 }, 140 EndpointSliceName: "svc-1-ipv4", 141 }, 142 Backends: map[cmtypes.AddrCluster]*k8s.Backend{ 143 cmtypes.MustParseAddrCluster("10.0.0.1"): { 144 NodeName: "node1", 145 Terminating: true, 146 }, 147 }, 148 } 149 150 eps1IPv4Remote := &k8s.Endpoints{ 151 ObjectMeta: slim_metav1.ObjectMeta{ 152 Name: "svc-1-ipv4", 153 Namespace: "default", 154 }, 155 EndpointSliceID: k8s.EndpointSliceID{ 156 ServiceID: k8s.ServiceID{ 157 Name: "svc-1", 158 Namespace: "default", 159 }, 160 EndpointSliceName: "svc-1-ipv4", 161 }, 162 Backends: map[cmtypes.AddrCluster]*k8s.Backend{ 163 cmtypes.MustParseAddrCluster("10.0.0.2"): { 164 NodeName: "node2", 165 }, 166 }, 167 } 168 169 eps1IPv4Mixed := &k8s.Endpoints{ 170 ObjectMeta: slim_metav1.ObjectMeta{ 171 Name: "svc-1-ipv4", 172 Namespace: "default", 173 }, 174 EndpointSliceID: k8s.EndpointSliceID{ 175 ServiceID: k8s.ServiceID{ 176 Name: "svc-1", 177 Namespace: "default", 178 }, 179 EndpointSliceName: "svc-1-ipv4", 180 }, 181 Backends: map[cmtypes.AddrCluster]*k8s.Backend{ 182 cmtypes.MustParseAddrCluster("10.0.0.1"): { 183 NodeName: "node1", 184 }, 185 cmtypes.MustParseAddrCluster("10.0.0.2"): { 186 NodeName: "node2", 187 }, 188 }, 189 } 190 191 eps1IPv6Local := &k8s.Endpoints{ 192 ObjectMeta: slim_metav1.ObjectMeta{ 193 Name: "svc-1-ipv6", 194 Namespace: "default", 195 }, 196 EndpointSliceID: k8s.EndpointSliceID{ 197 ServiceID: k8s.ServiceID{ 198 Name: "svc-1", 199 Namespace: "default", 200 }, 201 EndpointSliceName: "svc-1-ipv6", 202 }, 203 Backends: map[cmtypes.AddrCluster]*k8s.Backend{ 204 cmtypes.MustParseAddrCluster("fd00:10::1"): { 205 NodeName: "node1", 206 }, 207 }, 208 } 209 210 eps1IPv6Remote := &k8s.Endpoints{ 211 ObjectMeta: slim_metav1.ObjectMeta{ 212 Name: "svc-1-ipv6", 213 Namespace: "default", 214 }, 215 EndpointSliceID: k8s.EndpointSliceID{ 216 ServiceID: k8s.ServiceID{ 217 Name: "svc-1", 218 Namespace: "default", 219 }, 220 EndpointSliceName: "svc-1-ipv6", 221 }, 222 Backends: map[cmtypes.AddrCluster]*k8s.Backend{ 223 cmtypes.MustParseAddrCluster("fd00:10::2"): { 224 NodeName: "node2", 225 }, 226 }, 227 } 228 229 eps1IPv6Mixed := &k8s.Endpoints{ 230 ObjectMeta: slim_metav1.ObjectMeta{ 231 Name: "svc-1-ipv4", 232 Namespace: "default", 233 }, 234 EndpointSliceID: k8s.EndpointSliceID{ 235 ServiceID: k8s.ServiceID{ 236 Name: "svc-1", 237 Namespace: "default", 238 }, 239 EndpointSliceName: "svc-1-ipv4", 240 }, 241 Backends: map[cmtypes.AddrCluster]*k8s.Backend{ 242 cmtypes.MustParseAddrCluster("fd00:10::1"): { 243 NodeName: "node1", 244 }, 245 cmtypes.MustParseAddrCluster("fd00:10::2"): { 246 NodeName: "node2", 247 }, 248 }, 249 } 250 251 var table = []struct { 252 // name of the test case 253 name string 254 // The service selector of the vRouter 255 oldServiceSelector *slim_metav1.LabelSelector 256 // The service selector of the vRouter 257 newServiceSelector *slim_metav1.LabelSelector 258 // the advertised PodCIDR blocks the test begins with 259 advertised map[resource.Key][]string 260 // the services which will be "upserted" in the diffstore 261 upsertedServices []*slim_corev1.Service 262 // the services which will be "deleted" in the diffstore 263 deletedServices []resource.Key 264 // the endpoints which will be "upserted" in the diffstore 265 upsertedEndpoints []*k8s.Endpoints 266 // the updated PodCIDR blocks to reconcile, these are string encoded 267 // for the convenience of attaching directly to the NodeSpec.PodCIDRs 268 // field. 269 updated map[resource.Key][]string 270 // error nil or not 271 err error 272 }{ 273 // Add 1 ingress 274 { 275 name: "lb-svc-1-ingress", 276 oldServiceSelector: &blueSelector, 277 newServiceSelector: &blueSelector, 278 advertised: make(map[resource.Key][]string), 279 upsertedServices: []*slim_corev1.Service{svc1}, 280 updated: map[resource.Key][]string{ 281 svc1Name: { 282 ingressV4Prefix, 283 }, 284 }, 285 }, 286 // Add 2 ingress 287 { 288 name: "lb-svc-2-ingress", 289 oldServiceSelector: &blueSelector, 290 newServiceSelector: &blueSelector, 291 advertised: make(map[resource.Key][]string), 292 upsertedServices: []*slim_corev1.Service{svc1TwoIngress}, 293 updated: map[resource.Key][]string{ 294 svc1Name: { 295 ingressV4Prefix, 296 ingressV6Prefix, 297 }, 298 }, 299 }, 300 // Delete service 301 { 302 name: "delete-svc", 303 oldServiceSelector: &blueSelector, 304 newServiceSelector: &blueSelector, 305 advertised: map[resource.Key][]string{ 306 svc1Name: { 307 ingressV4Prefix, 308 }, 309 }, 310 deletedServices: []resource.Key{ 311 svc1Name, 312 }, 313 updated: map[resource.Key][]string{}, 314 }, 315 // Update service to no longer match 316 { 317 name: "update-service-no-match", 318 oldServiceSelector: &blueSelector, 319 newServiceSelector: &blueSelector, 320 advertised: map[resource.Key][]string{ 321 svc1Name: { 322 ingressV4Prefix, 323 }, 324 }, 325 upsertedServices: []*slim_corev1.Service{svc1RedLabel}, 326 updated: map[resource.Key][]string{}, 327 }, 328 // Update vRouter to no longer match 329 { 330 name: "update-vrouter-selector", 331 oldServiceSelector: &blueSelector, 332 newServiceSelector: &redSelector, 333 advertised: map[resource.Key][]string{ 334 svc1Name: { 335 ingressV4Prefix, 336 }, 337 }, 338 upsertedServices: []*slim_corev1.Service{svc1}, 339 updated: map[resource.Key][]string{}, 340 }, 341 // 1 -> 2 ingress 342 { 343 name: "update-1-to-2-ingress", 344 oldServiceSelector: &blueSelector, 345 newServiceSelector: &blueSelector, 346 advertised: map[resource.Key][]string{ 347 svc1Name: { 348 ingressV4Prefix, 349 }, 350 }, 351 upsertedServices: []*slim_corev1.Service{svc1TwoIngress}, 352 updated: map[resource.Key][]string{ 353 svc1Name: { 354 ingressV4Prefix, 355 ingressV6Prefix, 356 }, 357 }, 358 }, 359 // No selector 360 { 361 name: "no-selector", 362 oldServiceSelector: nil, 363 newServiceSelector: nil, 364 advertised: map[resource.Key][]string{}, 365 upsertedServices: []*slim_corev1.Service{svc1}, 366 updated: map[resource.Key][]string{}, 367 }, 368 // Namespace selector 369 { 370 name: "svc-namespace-selector", 371 oldServiceSelector: &slim_metav1.LabelSelector{MatchLabels: map[string]string{"io.kubernetes.service.namespace": "default"}}, 372 newServiceSelector: &slim_metav1.LabelSelector{MatchLabels: map[string]string{"io.kubernetes.service.namespace": "default"}}, 373 advertised: map[resource.Key][]string{}, 374 upsertedServices: []*slim_corev1.Service{ 375 svc1, 376 svc2NonDefault, 377 }, 378 updated: map[resource.Key][]string{ 379 svc1Name: { 380 ingressV4Prefix, 381 }, 382 }, 383 }, 384 // Service name selector 385 { 386 name: "svc-name-selector", 387 oldServiceSelector: &slim_metav1.LabelSelector{MatchLabels: map[string]string{"io.kubernetes.service.name": "svc-1"}}, 388 newServiceSelector: &slim_metav1.LabelSelector{MatchLabels: map[string]string{"io.kubernetes.service.name": "svc-1"}}, 389 advertised: map[resource.Key][]string{}, 390 upsertedServices: []*slim_corev1.Service{ 391 svc1, 392 svc1NonDefault, 393 }, 394 updated: map[resource.Key][]string{ 395 svc1Name: { 396 ingressV4Prefix, 397 }, 398 svc1NonDefaultName: { 399 ingressV4Prefix_2, 400 }, 401 }, 402 }, 403 // BGP load balancer class with matching selectors for service. 404 { 405 name: "lb-class-and-selectors", 406 oldServiceSelector: &blueSelector, 407 newServiceSelector: &blueSelector, 408 advertised: map[resource.Key][]string{}, 409 upsertedServices: []*slim_corev1.Service{svc1LbClass}, 410 updated: map[resource.Key][]string{ 411 svc1Name: { 412 ingressV4Prefix, 413 }, 414 }, 415 }, 416 // BGP load balancer class with no selectors for service. 417 { 418 name: "lb-class-no-selectors", 419 oldServiceSelector: nil, 420 newServiceSelector: nil, 421 advertised: map[resource.Key][]string{}, 422 upsertedServices: []*slim_corev1.Service{svc1LbClass}, 423 updated: map[resource.Key][]string{}, 424 }, 425 // BGP load balancer class with selectors for a different service. 426 { 427 name: "lb-class-with-diff-selectors", 428 oldServiceSelector: &slim_metav1.LabelSelector{MatchLabels: map[string]string{"io.kubernetes.service.name": "svc-2"}}, 429 newServiceSelector: &slim_metav1.LabelSelector{MatchLabels: map[string]string{"io.kubernetes.service.name": "svc-2"}}, 430 advertised: map[resource.Key][]string{}, 431 upsertedServices: []*slim_corev1.Service{svc1LbClass}, 432 updated: map[resource.Key][]string{}, 433 }, 434 // Unsupported load balancer class with matching selectors for service. 435 { 436 name: "unsupported-lb-class-with-selectors", 437 oldServiceSelector: &blueSelector, 438 newServiceSelector: &blueSelector, 439 advertised: map[resource.Key][]string{}, 440 upsertedServices: []*slim_corev1.Service{svc1UnsupportedClass}, 441 updated: map[resource.Key][]string{}, 442 }, 443 // Unsupported load balancer class with no matching selectors for service. 444 { 445 name: "unsupported-lb-class-with-no-selectors", 446 oldServiceSelector: nil, 447 newServiceSelector: nil, 448 advertised: map[resource.Key][]string{}, 449 upsertedServices: []*slim_corev1.Service{svc1UnsupportedClass}, 450 updated: map[resource.Key][]string{}, 451 }, 452 // No-LB service 453 { 454 name: "non-lb svc", 455 oldServiceSelector: &blueSelector, 456 newServiceSelector: &blueSelector, 457 advertised: map[resource.Key][]string{}, 458 upsertedServices: []*slim_corev1.Service{svc1NonLB}, 459 updated: map[resource.Key][]string{}, 460 }, 461 // Service without endpoints 462 { 463 name: "etp-local-no-endpoints", 464 oldServiceSelector: &blueSelector, 465 newServiceSelector: &blueSelector, 466 advertised: map[resource.Key][]string{}, 467 upsertedServices: []*slim_corev1.Service{svc1ETPLocal}, 468 upsertedEndpoints: []*k8s.Endpoints{}, 469 updated: map[resource.Key][]string{}, 470 }, 471 // Service with terminating endpoint 472 { 473 name: "etp-local-terminating-endpoint", 474 oldServiceSelector: &blueSelector, 475 newServiceSelector: &blueSelector, 476 advertised: map[resource.Key][]string{}, 477 upsertedServices: []*slim_corev1.Service{svc1ETPLocal}, 478 upsertedEndpoints: []*k8s.Endpoints{eps1IPv4LocalTerminating}, 479 updated: map[resource.Key][]string{}, 480 }, 481 // externalTrafficPolicy=Local && IPv4 && single slice && local endpoint 482 { 483 name: "etp-local-ipv4-single-slice-local", 484 oldServiceSelector: &blueSelector, 485 newServiceSelector: &blueSelector, 486 advertised: map[resource.Key][]string{}, 487 upsertedServices: []*slim_corev1.Service{svc1ETPLocal}, 488 upsertedEndpoints: []*k8s.Endpoints{eps1IPv4Local}, 489 updated: map[resource.Key][]string{ 490 svc1Name: { 491 ingressV4Prefix, 492 }, 493 }, 494 }, 495 // externalTrafficPolicy=Local && IPv4 && single slice && remote endpoint 496 { 497 name: "etp-local-ipv4-single-slice-remote", 498 oldServiceSelector: &blueSelector, 499 newServiceSelector: &blueSelector, 500 advertised: map[resource.Key][]string{}, 501 upsertedServices: []*slim_corev1.Service{svc1ETPLocal}, 502 upsertedEndpoints: []*k8s.Endpoints{eps1IPv4Remote}, 503 updated: map[resource.Key][]string{}, 504 }, 505 // externalTrafficPolicy=Local && IPv4 && single slice && mixed endpoint 506 { 507 name: "etp-local-ipv4-single-slice-mixed", 508 oldServiceSelector: &blueSelector, 509 newServiceSelector: &blueSelector, 510 advertised: map[resource.Key][]string{}, 511 upsertedServices: []*slim_corev1.Service{svc1ETPLocal}, 512 upsertedEndpoints: []*k8s.Endpoints{eps1IPv4Mixed}, 513 updated: map[resource.Key][]string{ 514 svc1Name: { 515 ingressV4Prefix, 516 }, 517 }, 518 }, 519 // externalTrafficPolicy=Local && IPv6 && single slice && local endpoint 520 { 521 name: "etp-local-ipv6-single-slice-local", 522 oldServiceSelector: &blueSelector, 523 newServiceSelector: &blueSelector, 524 advertised: map[resource.Key][]string{}, 525 upsertedServices: []*slim_corev1.Service{svc1IPv6ETPLocal}, 526 upsertedEndpoints: []*k8s.Endpoints{eps1IPv6Local}, 527 updated: map[resource.Key][]string{ 528 svc1Name: { 529 ingressV6Prefix, 530 }, 531 }, 532 }, 533 // externalTrafficPolicy=Local && IPv6 && single slice && remote endpoint 534 { 535 name: "etp-local-ipv6-single-slice-remote", 536 oldServiceSelector: &blueSelector, 537 newServiceSelector: &blueSelector, 538 advertised: map[resource.Key][]string{}, 539 upsertedServices: []*slim_corev1.Service{svc1IPv6ETPLocal}, 540 upsertedEndpoints: []*k8s.Endpoints{eps1IPv6Remote}, 541 updated: map[resource.Key][]string{}, 542 }, 543 // externalTrafficPolicy=Local && IPv6 && single slice && mixed endpoint 544 { 545 name: "etp-local-ipv6-single-slice-mixed", 546 oldServiceSelector: &blueSelector, 547 newServiceSelector: &blueSelector, 548 advertised: map[resource.Key][]string{}, 549 upsertedServices: []*slim_corev1.Service{svc1IPv6ETPLocal}, 550 upsertedEndpoints: []*k8s.Endpoints{eps1IPv6Mixed}, 551 updated: map[resource.Key][]string{ 552 svc1Name: { 553 ingressV6Prefix, 554 }, 555 }, 556 }, 557 // externalTrafficPolicy=Local && Dual && two slices && local endpoint 558 { 559 name: "etp-local-dual-two-slices-local", 560 oldServiceSelector: &blueSelector, 561 newServiceSelector: &blueSelector, 562 advertised: map[resource.Key][]string{}, 563 upsertedServices: []*slim_corev1.Service{svc1ETPLocalTwoIngress}, 564 upsertedEndpoints: []*k8s.Endpoints{ 565 eps1IPv4Local, 566 eps1IPv6Local, 567 }, 568 updated: map[resource.Key][]string{ 569 svc1Name: { 570 ingressV4Prefix, 571 ingressV6Prefix, 572 }, 573 }, 574 }, 575 // externalTrafficPolicy=Local && Dual && two slices && remote endpoint 576 { 577 name: "etp-local-dual-two-slices-remote", 578 oldServiceSelector: &blueSelector, 579 newServiceSelector: &blueSelector, 580 advertised: map[resource.Key][]string{}, 581 upsertedServices: []*slim_corev1.Service{svc1ETPLocalTwoIngress}, 582 upsertedEndpoints: []*k8s.Endpoints{ 583 eps1IPv4Remote, 584 eps1IPv6Remote, 585 }, 586 updated: map[resource.Key][]string{ 587 svc1Name: {}, 588 }, 589 }, 590 // externalTrafficPolicy=Local && Dual && two slices && mixed endpoint 591 { 592 name: "etp-local-dual-two-slices-mixed", 593 oldServiceSelector: &blueSelector, 594 newServiceSelector: &blueSelector, 595 advertised: map[resource.Key][]string{}, 596 upsertedServices: []*slim_corev1.Service{svc1ETPLocalTwoIngress}, 597 upsertedEndpoints: []*k8s.Endpoints{ 598 eps1IPv4Mixed, 599 eps1IPv6Mixed, 600 }, 601 updated: map[resource.Key][]string{ 602 svc1Name: { 603 ingressV4Prefix, 604 ingressV6Prefix, 605 }, 606 }, 607 }, 608 } 609 for _, tt := range table { 610 t.Run(tt.name, func(t *testing.T) { 611 // setup our test server, create a BgpServer, advertise the tt.advertised 612 // networks, and store each returned Advertisement in testSC.PodCIDRAnnouncements 613 srvParams := types.ServerParameters{ 614 Global: types.BGPGlobal{ 615 ASN: 64125, 616 RouterID: "127.0.0.1", 617 ListenPort: -1, 618 }, 619 } 620 oldc := &v2alpha1api.CiliumBGPVirtualRouter{ 621 LocalASN: 64125, 622 Neighbors: []v2alpha1api.CiliumBGPNeighbor{}, 623 ServiceSelector: tt.oldServiceSelector, 624 } 625 testSC, err := instance.NewServerWithConfig(context.Background(), log, srvParams) 626 if err != nil { 627 t.Fatalf("failed to create test bgp server: %v", err) 628 } 629 testSC.Config = oldc 630 631 diffstore := store.NewFakeDiffStore[*slim_corev1.Service]() 632 epDiffStore := store.NewFakeDiffStore[*k8s.Endpoints]() 633 634 reconciler := NewServiceReconciler(diffstore, epDiffStore).Reconciler.(*ServiceReconciler) 635 reconciler.Init(testSC) 636 defer reconciler.Cleanup(testSC) 637 638 for _, obj := range tt.upsertedServices { 639 diffstore.Upsert(obj) 640 } 641 for _, key := range tt.deletedServices { 642 diffstore.Delete(key) 643 } 644 for _, obj := range tt.upsertedEndpoints { 645 epDiffStore.Upsert(obj) 646 } 647 648 serviceAnnouncements := reconciler.getMetadata(testSC) 649 650 for svcKey, cidrs := range tt.advertised { 651 for _, cidr := range cidrs { 652 prefix := netip.MustParsePrefix(cidr) 653 advrtResp, err := testSC.Server.AdvertisePath(context.Background(), types.PathRequest{ 654 Path: types.NewPathForPrefix(prefix), 655 }) 656 if err != nil { 657 t.Fatalf("failed to advertise initial svc lb cidr routes: %v", err) 658 } 659 660 serviceAnnouncements[svcKey] = append(serviceAnnouncements[svcKey], advrtResp.Path) 661 } 662 } 663 664 newc := &v2alpha1api.CiliumBGPVirtualRouter{ 665 LocalASN: 64125, 666 Neighbors: []v2alpha1api.CiliumBGPNeighbor{}, 667 ServiceSelector: tt.newServiceSelector, 668 ServiceAdvertisements: []v2alpha1api.BGPServiceAddressType{v2alpha1api.BGPLoadBalancerIPAddr}, 669 } 670 671 // Run the reconciler twice to ensure idempotency. This 672 // simulates the retrying behavior of the controller. 673 for i := 0; i < 2; i++ { 674 t.Run(tt.name, func(t *testing.T) { 675 err = reconciler.Reconcile(context.Background(), ReconcileParams{ 676 CurrentServer: testSC, 677 DesiredConfig: newc, 678 CiliumNode: &v2api.CiliumNode{ 679 ObjectMeta: meta_v1.ObjectMeta{ 680 Name: "node1", 681 }, 682 }, 683 }) 684 if err != nil { 685 t.Fatalf("failed to reconcile new lb svc advertisements: %v", err) 686 } 687 }) 688 } 689 690 // if we disable exports of pod cidr ensure no advertisements are 691 // still present. 692 if tt.newServiceSelector == nil && !containsLbClass(tt.upsertedServices) { 693 if len(serviceAnnouncements) > 0 { 694 t.Fatal("disabled export but advertisements still present") 695 } 696 } 697 698 log.Printf("%+v %+v", serviceAnnouncements, tt.updated) 699 700 // ensure we see tt.updated in testSC.ServiceAnnouncements 701 for svcKey, cidrs := range tt.updated { 702 for _, cidr := range cidrs { 703 prefix := netip.MustParsePrefix(cidr) 704 var seen bool 705 for _, advrt := range serviceAnnouncements[svcKey] { 706 if advrt.NLRI.String() == prefix.String() { 707 seen = true 708 } 709 } 710 if !seen { 711 t.Fatalf("failed to advertise %v", cidr) 712 } 713 } 714 } 715 716 // ensure testSC.PodCIDRAnnouncements does not contain advertisements 717 // not in tt.updated 718 for svcKey, advrts := range serviceAnnouncements { 719 for _, advrt := range advrts { 720 var seen bool 721 for _, cidr := range tt.updated[svcKey] { 722 if advrt.NLRI.String() == cidr { 723 seen = true 724 } 725 } 726 if !seen { 727 t.Fatalf("unwanted advert %+v", advrt) 728 } 729 } 730 } 731 732 }) 733 } 734 } 735 736 func TestServiceReconcilerWithClusterIP(t *testing.T) { 737 blueSelector := slim_metav1.LabelSelector{MatchLabels: map[string]string{"color": "blue"}} 738 redSelector := slim_metav1.LabelSelector{MatchLabels: map[string]string{"color": "red"}} 739 svc1Name := resource.Key{Name: "svc-1", Namespace: "default"} 740 svc1NonDefaultName := resource.Key{Name: "svc-1", Namespace: "non-default"} 741 svc2NonDefaultName := resource.Key{Name: "svc-2", Namespace: "non-default"} 742 clusterIPV4 := "192.168.0.1" 743 clusterIPV4Prefix := clusterIPV4 + "/32" 744 clusterIPV6 := "fd00:192:168::1" 745 clusterIPV6Prefix := clusterIPV6 + "/128" 746 747 svc1 := &slim_corev1.Service{ 748 ObjectMeta: slim_metav1.ObjectMeta{ 749 Name: svc1Name.Name, 750 Namespace: svc1Name.Namespace, 751 Labels: blueSelector.MatchLabels, 752 }, 753 Spec: slim_corev1.ServiceSpec{ 754 Type: slim_corev1.ServiceTypeClusterIP, 755 ClusterIP: clusterIPV4, 756 ClusterIPs: []string{ 757 clusterIPV4, 758 }, 759 }, 760 Status: slim_corev1.ServiceStatus{ 761 LoadBalancer: slim_corev1.LoadBalancerStatus{}, 762 }, 763 } 764 765 svc1TwoIngress := svc1.DeepCopy() 766 svc1TwoIngress.Spec.ClusterIPs = append(svc1TwoIngress.Spec.ClusterIPs, clusterIPV6) 767 svc1RedLabel := svc1.DeepCopy() 768 svc1RedLabel.ObjectMeta.Labels = redSelector.MatchLabels 769 770 svc1NonDefault := svc1.DeepCopy() 771 svc1NonDefault.Namespace = svc1NonDefaultName.Namespace 772 773 svc1NonClusterIP := svc1.DeepCopy() 774 svc1NonClusterIP.Spec.ClusterIP = "None" 775 svc1NonClusterIP.Spec.ClusterIPs = append(svc1NonClusterIP.Spec.ClusterIPs, "None") 776 svc1ITPLocal := svc1.DeepCopy() 777 internalTrafficPolicyLocal := slim_corev1.ServiceInternalTrafficPolicyLocal 778 svc1ITPLocal.Spec.InternalTrafficPolicy = &internalTrafficPolicyLocal 779 780 svc1ITPLocalTwoIngress := svc1TwoIngress.DeepCopy() 781 svc1ITPLocalTwoIngress.Spec.InternalTrafficPolicy = &internalTrafficPolicyLocal 782 783 svc1IPv6ITPLocal := svc1.DeepCopy() 784 svc1IPv6ITPLocal.Spec.ClusterIPs = append(svc1IPv6ITPLocal.Spec.ClusterIPs, clusterIPV6) 785 svc1IPv6ITPLocal.Spec.InternalTrafficPolicy = &internalTrafficPolicyLocal 786 787 svc1LbClass := svc1.DeepCopy() 788 svc1LbClass.Spec.LoadBalancerClass = ptr.To[string](v2alpha1api.BGPLoadBalancerClass) 789 790 svc1UnsupportedClass := svc1LbClass.DeepCopy() 791 svc1UnsupportedClass.Spec.LoadBalancerClass = ptr.To[string]("io.vendor/unsupported-class") 792 793 svc2NonDefault := &slim_corev1.Service{ 794 ObjectMeta: slim_metav1.ObjectMeta{ 795 Name: svc2NonDefaultName.Name, 796 Namespace: svc2NonDefaultName.Namespace, 797 Labels: blueSelector.MatchLabels, 798 }, 799 Spec: slim_corev1.ServiceSpec{ 800 Type: slim_corev1.ServiceTypeClusterIP, 801 ClusterIP: clusterIPV4, 802 ClusterIPs: []string{ 803 clusterIPV4, 804 }, 805 }, 806 Status: slim_corev1.ServiceStatus{ 807 LoadBalancer: slim_corev1.LoadBalancerStatus{}, 808 }, 809 } 810 811 eps1IPv4Local := &k8s.Endpoints{ 812 ObjectMeta: slim_metav1.ObjectMeta{ 813 Name: "svc-1-ipv4", 814 Namespace: "default", 815 }, 816 EndpointSliceID: k8s.EndpointSliceID{ 817 ServiceID: k8s.ServiceID{ 818 Name: "svc-1", 819 Namespace: "default", 820 }, 821 EndpointSliceName: "svc-1-ipv4", 822 }, 823 Backends: map[cmtypes.AddrCluster]*k8s.Backend{ 824 cmtypes.MustParseAddrCluster("10.0.0.1"): { 825 NodeName: "node1", 826 }, 827 }, 828 } 829 830 eps1IPv4Remote := &k8s.Endpoints{ 831 ObjectMeta: slim_metav1.ObjectMeta{ 832 Name: "svc-1-ipv4", 833 Namespace: "default", 834 }, 835 EndpointSliceID: k8s.EndpointSliceID{ 836 ServiceID: k8s.ServiceID{ 837 Name: "svc-1", 838 Namespace: "default", 839 }, 840 EndpointSliceName: "svc-1-ipv4", 841 }, 842 Backends: map[cmtypes.AddrCluster]*k8s.Backend{ 843 cmtypes.MustParseAddrCluster("10.0.0.2"): { 844 NodeName: "node2", 845 }, 846 }, 847 } 848 849 eps1IPv4Mixed := &k8s.Endpoints{ 850 ObjectMeta: slim_metav1.ObjectMeta{ 851 Name: "svc-1-ipv4", 852 Namespace: "default", 853 }, 854 EndpointSliceID: k8s.EndpointSliceID{ 855 ServiceID: k8s.ServiceID{ 856 Name: "svc-1", 857 Namespace: "default", 858 }, 859 EndpointSliceName: "svc-1-ipv4", 860 }, 861 Backends: map[cmtypes.AddrCluster]*k8s.Backend{ 862 cmtypes.MustParseAddrCluster("10.0.0.1"): { 863 NodeName: "node1", 864 }, 865 cmtypes.MustParseAddrCluster("10.0.0.2"): { 866 NodeName: "node2", 867 }, 868 }, 869 } 870 871 eps1IPv6Local := &k8s.Endpoints{ 872 ObjectMeta: slim_metav1.ObjectMeta{ 873 Name: "svc-1-ipv6", 874 Namespace: "default", 875 }, 876 EndpointSliceID: k8s.EndpointSliceID{ 877 ServiceID: k8s.ServiceID{ 878 Name: "svc-1", 879 Namespace: "default", 880 }, 881 EndpointSliceName: "svc-1-ipv6", 882 }, 883 Backends: map[cmtypes.AddrCluster]*k8s.Backend{ 884 cmtypes.MustParseAddrCluster("fd00:10::1"): { 885 NodeName: "node1", 886 }, 887 }, 888 } 889 890 eps1IPv6Remote := &k8s.Endpoints{ 891 ObjectMeta: slim_metav1.ObjectMeta{ 892 Name: "svc-1-ipv6", 893 Namespace: "default", 894 }, 895 EndpointSliceID: k8s.EndpointSliceID{ 896 ServiceID: k8s.ServiceID{ 897 Name: "svc-1", 898 Namespace: "default", 899 }, 900 EndpointSliceName: "svc-1-ipv6", 901 }, 902 Backends: map[cmtypes.AddrCluster]*k8s.Backend{ 903 cmtypes.MustParseAddrCluster("fd00:10::2"): { 904 NodeName: "node2", 905 }, 906 }, 907 } 908 909 eps1IPv6Mixed := &k8s.Endpoints{ 910 ObjectMeta: slim_metav1.ObjectMeta{ 911 Name: "svc-1-ipv4", 912 Namespace: "default", 913 }, 914 EndpointSliceID: k8s.EndpointSliceID{ 915 ServiceID: k8s.ServiceID{ 916 Name: "svc-1", 917 Namespace: "default", 918 }, 919 EndpointSliceName: "svc-1-ipv4", 920 }, 921 Backends: map[cmtypes.AddrCluster]*k8s.Backend{ 922 cmtypes.MustParseAddrCluster("fd00:10::1"): { 923 NodeName: "node1", 924 }, 925 cmtypes.MustParseAddrCluster("fd00:10::2"): { 926 NodeName: "node2", 927 }, 928 }, 929 } 930 931 var table = []struct { 932 // name of the test case 933 name string 934 // The service selector of the vRouter 935 oldServiceSelector *slim_metav1.LabelSelector 936 // The service selector of the vRouter 937 newServiceSelector *slim_metav1.LabelSelector 938 // the advertised PodCIDR blocks the test begins with 939 advertised map[resource.Key][]string 940 // the services which will be "upserted" in the diffstore 941 upsertedServices []*slim_corev1.Service 942 // the services which will be "deleted" in the diffstore 943 deletedServices []resource.Key 944 // the endpoints which will be "upserted" in the diffstore 945 upsertedEndpoints []*k8s.Endpoints 946 // the updated PodCIDR blocks to reconcile, these are string encoded 947 // for the convenience of attaching directly to the NodeSpec.PodCIDRs 948 // field. 949 updated map[resource.Key][]string 950 // error nil or not 951 err error 952 }{ 953 // Add 1 clusterIP 954 { 955 name: "svc-1-clusterIP", 956 oldServiceSelector: &blueSelector, 957 newServiceSelector: &blueSelector, 958 advertised: make(map[resource.Key][]string), 959 upsertedServices: []*slim_corev1.Service{svc1}, 960 updated: map[resource.Key][]string{ 961 svc1Name: { 962 clusterIPV4Prefix, 963 }, 964 }, 965 }, 966 // Add 2 clusterIP 967 { 968 name: "svc-2-clusterIP", 969 oldServiceSelector: &blueSelector, 970 newServiceSelector: &blueSelector, 971 advertised: make(map[resource.Key][]string), 972 upsertedServices: []*slim_corev1.Service{svc1TwoIngress}, 973 updated: map[resource.Key][]string{ 974 svc1Name: { 975 clusterIPV4Prefix, 976 clusterIPV6Prefix, 977 }, 978 }, 979 }, 980 // Delete service 981 { 982 name: "delete-svc", 983 oldServiceSelector: &blueSelector, 984 newServiceSelector: &blueSelector, 985 advertised: map[resource.Key][]string{ 986 svc1Name: { 987 clusterIPV4Prefix, 988 }, 989 }, 990 deletedServices: []resource.Key{ 991 svc1Name, 992 }, 993 updated: map[resource.Key][]string{}, 994 }, 995 // Update service to no longer match 996 { 997 name: "update-service-no-match", 998 oldServiceSelector: &blueSelector, 999 newServiceSelector: &blueSelector, 1000 advertised: map[resource.Key][]string{ 1001 svc1Name: { 1002 clusterIPV4Prefix, 1003 }, 1004 }, 1005 upsertedServices: []*slim_corev1.Service{svc1RedLabel}, 1006 updated: map[resource.Key][]string{}, 1007 }, 1008 // Update vRouter to no longer match 1009 { 1010 name: "update-vrouter-selector", 1011 oldServiceSelector: &blueSelector, 1012 newServiceSelector: &redSelector, 1013 advertised: map[resource.Key][]string{ 1014 svc1Name: { 1015 clusterIPV4Prefix, 1016 }, 1017 }, 1018 upsertedServices: []*slim_corev1.Service{svc1}, 1019 updated: map[resource.Key][]string{}, 1020 }, 1021 // 1 -> 2 clusterIP 1022 { 1023 name: "update-1-to-2-clusterIP", 1024 oldServiceSelector: &blueSelector, 1025 newServiceSelector: &blueSelector, 1026 advertised: map[resource.Key][]string{ 1027 svc1Name: { 1028 clusterIPV4Prefix, 1029 }, 1030 }, 1031 upsertedServices: []*slim_corev1.Service{svc1TwoIngress}, 1032 updated: map[resource.Key][]string{ 1033 svc1Name: { 1034 clusterIPV4Prefix, 1035 clusterIPV6Prefix, 1036 }, 1037 }, 1038 }, 1039 // No selector 1040 { 1041 name: "no-selector", 1042 oldServiceSelector: nil, 1043 newServiceSelector: nil, 1044 advertised: map[resource.Key][]string{}, 1045 upsertedServices: []*slim_corev1.Service{svc1}, 1046 updated: map[resource.Key][]string{}, 1047 }, 1048 // Namespace selector 1049 { 1050 name: "svc-namespace-selector", 1051 oldServiceSelector: &slim_metav1.LabelSelector{MatchLabels: map[string]string{"io.kubernetes.service.namespace": "default"}}, 1052 newServiceSelector: &slim_metav1.LabelSelector{MatchLabels: map[string]string{"io.kubernetes.service.namespace": "default"}}, 1053 advertised: map[resource.Key][]string{}, 1054 upsertedServices: []*slim_corev1.Service{ 1055 svc1, 1056 svc2NonDefault, 1057 }, 1058 updated: map[resource.Key][]string{ 1059 svc1Name: { 1060 clusterIPV4Prefix, 1061 }, 1062 }, 1063 }, 1064 // Service name selector 1065 { 1066 name: "svc-name-selector", 1067 oldServiceSelector: &slim_metav1.LabelSelector{MatchLabels: map[string]string{"io.kubernetes.service.name": "svc-1"}}, 1068 newServiceSelector: &slim_metav1.LabelSelector{MatchLabels: map[string]string{"io.kubernetes.service.name": "svc-1"}}, 1069 advertised: map[resource.Key][]string{}, 1070 upsertedServices: []*slim_corev1.Service{ 1071 svc1, 1072 }, 1073 updated: map[resource.Key][]string{ 1074 svc1Name: { 1075 clusterIPV4Prefix, 1076 }, 1077 }, 1078 }, 1079 // BGP load balancer class with matching selectors for service. 1080 { 1081 name: "lb-class-and-selectors", 1082 oldServiceSelector: &blueSelector, 1083 newServiceSelector: &blueSelector, 1084 advertised: map[resource.Key][]string{}, 1085 upsertedServices: []*slim_corev1.Service{svc1LbClass}, 1086 updated: map[resource.Key][]string{ 1087 svc1Name: { 1088 clusterIPV4Prefix, 1089 }, 1090 }, 1091 }, 1092 // BGP load balancer class with no selectors for service. 1093 { 1094 name: "lb-class-no-selectors", 1095 oldServiceSelector: nil, 1096 newServiceSelector: nil, 1097 advertised: map[resource.Key][]string{}, 1098 upsertedServices: []*slim_corev1.Service{svc1LbClass}, 1099 updated: map[resource.Key][]string{}, 1100 }, 1101 // BGP load balancer class with selectors for a different service. 1102 { 1103 name: "lb-class-with-diff-selectors", 1104 oldServiceSelector: &slim_metav1.LabelSelector{MatchLabels: map[string]string{"io.kubernetes.service.name": "svc-2"}}, 1105 newServiceSelector: &slim_metav1.LabelSelector{MatchLabels: map[string]string{"io.kubernetes.service.name": "svc-2"}}, 1106 advertised: map[resource.Key][]string{}, 1107 upsertedServices: []*slim_corev1.Service{svc1LbClass}, 1108 updated: map[resource.Key][]string{}, 1109 }, 1110 // Unsupported load balancer class with no matching selectors for service. 1111 { 1112 name: "unsupported-lb-class-with-no-selectors", 1113 oldServiceSelector: nil, 1114 newServiceSelector: nil, 1115 advertised: map[resource.Key][]string{}, 1116 upsertedServices: []*slim_corev1.Service{svc1UnsupportedClass}, 1117 updated: map[resource.Key][]string{}, 1118 }, 1119 // No-clusterIP service 1120 { 1121 name: "non-clusterIP svc", 1122 oldServiceSelector: &blueSelector, 1123 newServiceSelector: &blueSelector, 1124 advertised: map[resource.Key][]string{}, 1125 upsertedServices: []*slim_corev1.Service{svc1NonClusterIP}, 1126 updated: map[resource.Key][]string{}, 1127 }, 1128 // Service without endpoints 1129 { 1130 name: "etp-local-no-endpoints", 1131 oldServiceSelector: &blueSelector, 1132 newServiceSelector: &blueSelector, 1133 advertised: map[resource.Key][]string{}, 1134 upsertedServices: []*slim_corev1.Service{svc1ITPLocal}, 1135 upsertedEndpoints: []*k8s.Endpoints{}, 1136 updated: map[resource.Key][]string{}, 1137 }, 1138 // internalTrafficPolicyLocal=Local && IPv4 && single slice && local endpoint 1139 { 1140 name: "itp-local-ipv4-single-slice-local", 1141 oldServiceSelector: &blueSelector, 1142 newServiceSelector: &blueSelector, 1143 advertised: map[resource.Key][]string{}, 1144 upsertedServices: []*slim_corev1.Service{svc1ITPLocal}, 1145 upsertedEndpoints: []*k8s.Endpoints{eps1IPv4Local}, 1146 updated: map[resource.Key][]string{ 1147 svc1Name: { 1148 clusterIPV4Prefix, 1149 }, 1150 }, 1151 }, 1152 // internalTrafficPolicyLocal=Local && IPv4 && single slice && remote endpoint 1153 { 1154 name: "itp-local-ipv4-single-slice-remote", 1155 oldServiceSelector: &blueSelector, 1156 newServiceSelector: &blueSelector, 1157 advertised: map[resource.Key][]string{}, 1158 upsertedServices: []*slim_corev1.Service{svc1ITPLocal}, 1159 upsertedEndpoints: []*k8s.Endpoints{eps1IPv4Remote}, 1160 updated: map[resource.Key][]string{}, 1161 }, 1162 // internalTrafficPolicyLocal=Local && IPv4 && single slice && mixed endpoint 1163 { 1164 name: "itp-local-ipv4-single-slice-mixed", 1165 oldServiceSelector: &blueSelector, 1166 newServiceSelector: &blueSelector, 1167 advertised: map[resource.Key][]string{}, 1168 upsertedServices: []*slim_corev1.Service{svc1ITPLocal}, 1169 upsertedEndpoints: []*k8s.Endpoints{eps1IPv4Mixed}, 1170 updated: map[resource.Key][]string{ 1171 svc1Name: { 1172 clusterIPV4Prefix, 1173 }, 1174 }, 1175 }, 1176 // internalTrafficPolicyLocal=Local && IPv6 && single slice && local endpoint 1177 { 1178 name: "itp-local-ipv6-single-slice-local", 1179 oldServiceSelector: &blueSelector, 1180 newServiceSelector: &blueSelector, 1181 advertised: map[resource.Key][]string{}, 1182 upsertedServices: []*slim_corev1.Service{svc1IPv6ITPLocal}, 1183 upsertedEndpoints: []*k8s.Endpoints{eps1IPv6Local}, 1184 updated: map[resource.Key][]string{ 1185 svc1Name: { 1186 clusterIPV4Prefix, 1187 clusterIPV6Prefix, 1188 }, 1189 }, 1190 }, 1191 // internalTrafficPolicyLocal=Local && IPv6 && single slice && remote endpoint 1192 { 1193 name: "itp-local-ipv6-single-slice-remote", 1194 oldServiceSelector: &blueSelector, 1195 newServiceSelector: &blueSelector, 1196 advertised: map[resource.Key][]string{}, 1197 upsertedServices: []*slim_corev1.Service{svc1IPv6ITPLocal}, 1198 upsertedEndpoints: []*k8s.Endpoints{eps1IPv6Remote}, 1199 updated: map[resource.Key][]string{}, 1200 }, 1201 // internalTrafficPolicyLocal=Local && IPv6 && single slice && mixed endpoint 1202 { 1203 name: "itp-local-ipv6-single-slice-mixed", 1204 oldServiceSelector: &blueSelector, 1205 newServiceSelector: &blueSelector, 1206 advertised: map[resource.Key][]string{}, 1207 upsertedServices: []*slim_corev1.Service{svc1IPv6ITPLocal}, 1208 upsertedEndpoints: []*k8s.Endpoints{eps1IPv6Mixed}, 1209 updated: map[resource.Key][]string{ 1210 svc1Name: { 1211 clusterIPV4Prefix, 1212 clusterIPV6Prefix, 1213 }, 1214 }, 1215 }, 1216 // internalTrafficPolicyLocal=Local && Dual && two slices && local endpoint 1217 { 1218 name: "itp-local-dual-two-slices-local", 1219 oldServiceSelector: &blueSelector, 1220 newServiceSelector: &blueSelector, 1221 advertised: map[resource.Key][]string{}, 1222 upsertedServices: []*slim_corev1.Service{svc1ITPLocalTwoIngress}, 1223 upsertedEndpoints: []*k8s.Endpoints{ 1224 eps1IPv4Local, 1225 eps1IPv6Local, 1226 }, 1227 updated: map[resource.Key][]string{ 1228 svc1Name: { 1229 clusterIPV4Prefix, 1230 clusterIPV6Prefix, 1231 }, 1232 }, 1233 }, 1234 // internalTrafficPolicyLocal=Local && Dual && two slices && remote endpoint 1235 { 1236 name: "itp-local-dual-two-slices-remote", 1237 oldServiceSelector: &blueSelector, 1238 newServiceSelector: &blueSelector, 1239 advertised: map[resource.Key][]string{}, 1240 upsertedServices: []*slim_corev1.Service{svc1ITPLocalTwoIngress}, 1241 upsertedEndpoints: []*k8s.Endpoints{ 1242 eps1IPv4Remote, 1243 eps1IPv6Remote, 1244 }, 1245 updated: map[resource.Key][]string{ 1246 svc1Name: {}, 1247 }, 1248 }, 1249 // internalTrafficPolicyLocal=Local && Dual && two slices && mixed endpoint 1250 { 1251 name: "itp-local-dual-two-slices-mixed", 1252 oldServiceSelector: &blueSelector, 1253 newServiceSelector: &blueSelector, 1254 advertised: map[resource.Key][]string{}, 1255 upsertedServices: []*slim_corev1.Service{svc1ITPLocalTwoIngress}, 1256 upsertedEndpoints: []*k8s.Endpoints{ 1257 eps1IPv4Mixed, 1258 eps1IPv6Mixed, 1259 }, 1260 updated: map[resource.Key][]string{ 1261 svc1Name: { 1262 clusterIPV4Prefix, 1263 clusterIPV6Prefix, 1264 }, 1265 }, 1266 }, 1267 } 1268 for _, tt := range table { 1269 t.Run(tt.name, func(t *testing.T) { 1270 // setup our test server, create a BgpServer, advertise the tt.advertised 1271 // networks, and store each returned Advertisement in testSC.PodCIDRAnnouncements 1272 srvParams := types.ServerParameters{ 1273 Global: types.BGPGlobal{ 1274 ASN: 64125, 1275 RouterID: "127.0.0.1", 1276 ListenPort: -1, 1277 }, 1278 } 1279 oldc := &v2alpha1api.CiliumBGPVirtualRouter{ 1280 LocalASN: 64125, 1281 Neighbors: []v2alpha1api.CiliumBGPNeighbor{}, 1282 ServiceSelector: tt.oldServiceSelector, 1283 } 1284 testSC, err := instance.NewServerWithConfig(context.Background(), log, srvParams) 1285 if err != nil { 1286 t.Fatalf("failed to create test bgp server: %v", err) 1287 } 1288 testSC.Config = oldc 1289 1290 diffstore := store.NewFakeDiffStore[*slim_corev1.Service]() 1291 epDiffStore := store.NewFakeDiffStore[*k8s.Endpoints]() 1292 1293 reconciler := NewServiceReconciler(diffstore, epDiffStore).Reconciler.(*ServiceReconciler) 1294 reconciler.Init(testSC) 1295 defer reconciler.Cleanup(testSC) 1296 1297 for _, obj := range tt.upsertedServices { 1298 diffstore.Upsert(obj) 1299 } 1300 for _, key := range tt.deletedServices { 1301 diffstore.Delete(key) 1302 } 1303 for _, obj := range tt.upsertedEndpoints { 1304 epDiffStore.Upsert(obj) 1305 } 1306 1307 serviceAnnouncements := reconciler.getMetadata(testSC) 1308 1309 for svcKey, cidrs := range tt.advertised { 1310 for _, cidr := range cidrs { 1311 prefix := netip.MustParsePrefix(cidr) 1312 advrtResp, err := testSC.Server.AdvertisePath(context.Background(), types.PathRequest{ 1313 Path: types.NewPathForPrefix(prefix), 1314 }) 1315 if err != nil { 1316 t.Fatalf("failed to advertise initial svc lb cidr routes: %v", err) 1317 } 1318 1319 serviceAnnouncements[svcKey] = append(serviceAnnouncements[svcKey], advrtResp.Path) 1320 } 1321 } 1322 1323 newc := &v2alpha1api.CiliumBGPVirtualRouter{ 1324 LocalASN: 64125, 1325 Neighbors: []v2alpha1api.CiliumBGPNeighbor{}, 1326 ServiceSelector: tt.newServiceSelector, 1327 ServiceAdvertisements: []v2alpha1api.BGPServiceAddressType{v2alpha1api.BGPClusterIPAddr}, 1328 } 1329 1330 // Run the reconciler twice to ensure idempotency. This 1331 // simulates the retrying behavior of the controller. 1332 for i := 0; i < 2; i++ { 1333 t.Run(tt.name, func(t *testing.T) { 1334 err = reconciler.Reconcile(context.Background(), ReconcileParams{ 1335 CurrentServer: testSC, 1336 DesiredConfig: newc, 1337 CiliumNode: &v2api.CiliumNode{ 1338 ObjectMeta: meta_v1.ObjectMeta{ 1339 Name: "node1", 1340 }, 1341 }, 1342 }) 1343 if err != nil { 1344 t.Fatalf("failed to reconcile new lb svc advertisements: %v", err) 1345 } 1346 }) 1347 } 1348 1349 // if we disable exports of pod cidr ensure no advertisements are 1350 // still present. 1351 if tt.newServiceSelector == nil && !containsLbClass(tt.upsertedServices) { 1352 if len(serviceAnnouncements) > 0 { 1353 t.Fatal("disabled export but advertisements still present") 1354 } 1355 } 1356 1357 log.Printf("%+v %+v", serviceAnnouncements, tt.updated) 1358 1359 // ensure we see tt.updated in testSC.ServiceAnnouncements 1360 for svcKey, cidrs := range tt.updated { 1361 for _, cidr := range cidrs { 1362 prefix := netip.MustParsePrefix(cidr) 1363 var seen bool 1364 for _, advrt := range serviceAnnouncements[svcKey] { 1365 if advrt.NLRI.String() == prefix.String() { 1366 seen = true 1367 } 1368 } 1369 if !seen { 1370 t.Fatalf("failed to advertise %v", cidr) 1371 } 1372 } 1373 } 1374 1375 // ensure testSC.PodCIDRAnnouncements does not contain advertisements 1376 // not in tt.updated 1377 for svcKey, advrts := range serviceAnnouncements { 1378 for _, advrt := range advrts { 1379 var seen bool 1380 for _, cidr := range tt.updated[svcKey] { 1381 if advrt.NLRI.String() == cidr { 1382 seen = true 1383 } 1384 } 1385 if !seen { 1386 t.Fatalf("unwanted advert %+v", advrt) 1387 } 1388 } 1389 } 1390 1391 }) 1392 } 1393 } 1394 1395 func TestServiceReconcilerWithExternalIP(t *testing.T) { 1396 blueSelector := slim_metav1.LabelSelector{MatchLabels: map[string]string{"color": "blue"}} 1397 redSelector := slim_metav1.LabelSelector{MatchLabels: map[string]string{"color": "red"}} 1398 svc1Name := resource.Key{Name: "svc-1", Namespace: "default"} 1399 svc1NonDefaultName := resource.Key{Name: "svc-1", Namespace: "non-default"} 1400 svc2NonDefaultName := resource.Key{Name: "svc-2", Namespace: "non-default"} 1401 externalIPV4 := "192.168.0.1" 1402 externalIPV4Prefix := externalIPV4 + "/32" 1403 externalIPV6 := "fd00:192:168::1" 1404 externalIPV6Prefix := externalIPV6 + "/128" 1405 1406 svc1 := &slim_corev1.Service{ 1407 ObjectMeta: slim_metav1.ObjectMeta{ 1408 Name: svc1Name.Name, 1409 Namespace: svc1Name.Namespace, 1410 Labels: blueSelector.MatchLabels, 1411 }, 1412 Spec: slim_corev1.ServiceSpec{ 1413 Type: slim_corev1.ServiceTypeClusterIP, 1414 ExternalIPs: []string{ 1415 externalIPV4, 1416 }, 1417 }, 1418 Status: slim_corev1.ServiceStatus{ 1419 LoadBalancer: slim_corev1.LoadBalancerStatus{}, 1420 }, 1421 } 1422 1423 svc1TwoIngress := svc1.DeepCopy() 1424 svc1TwoIngress.Spec.ExternalIPs = append(svc1TwoIngress.Spec.ExternalIPs, externalIPV6) 1425 svc1RedLabel := svc1.DeepCopy() 1426 svc1RedLabel.ObjectMeta.Labels = redSelector.MatchLabels 1427 1428 svc1NonDefault := svc1.DeepCopy() 1429 svc1NonDefault.Namespace = svc1NonDefaultName.Namespace 1430 1431 svc1NonExternalIP := svc1.DeepCopy() 1432 svc1NonExternalIP.Spec.ClusterIP = externalIPV4 1433 svc1NonExternalIP.Spec.ExternalIPs = []string{} 1434 1435 svc1ETPLocal := svc1.DeepCopy() 1436 svc1ETPLocal.Spec.ExternalTrafficPolicy = slim_corev1.ServiceExternalTrafficPolicyLocal 1437 1438 svc1ETPLocalTwoIngress := svc1TwoIngress.DeepCopy() 1439 svc1ETPLocalTwoIngress.Spec.ExternalTrafficPolicy = slim_corev1.ServiceExternalTrafficPolicyLocal 1440 1441 svc1IPv6ETPLocal := svc1.DeepCopy() 1442 svc1IPv6ETPLocal.Spec.ExternalIPs = append(svc1IPv6ETPLocal.Spec.ExternalIPs, externalIPV6) 1443 svc1IPv6ETPLocal.Spec.ExternalTrafficPolicy = slim_corev1.ServiceExternalTrafficPolicyLocal 1444 1445 svc1LbClass := svc1.DeepCopy() 1446 svc1LbClass.Spec.LoadBalancerClass = ptr.To[string](v2alpha1api.BGPLoadBalancerClass) 1447 1448 svc1UnsupportedClass := svc1LbClass.DeepCopy() 1449 svc1UnsupportedClass.Spec.LoadBalancerClass = ptr.To[string]("io.vendor/unsupported-class") 1450 1451 svc2NonDefault := &slim_corev1.Service{ 1452 ObjectMeta: slim_metav1.ObjectMeta{ 1453 Name: svc2NonDefaultName.Name, 1454 Namespace: svc2NonDefaultName.Namespace, 1455 Labels: blueSelector.MatchLabels, 1456 }, 1457 Spec: slim_corev1.ServiceSpec{ 1458 Type: slim_corev1.ServiceTypeClusterIP, 1459 ExternalIPs: []string{ 1460 externalIPV4, 1461 }, 1462 }, 1463 Status: slim_corev1.ServiceStatus{ 1464 LoadBalancer: slim_corev1.LoadBalancerStatus{}, 1465 }, 1466 } 1467 1468 eps1IPv4Local := &k8s.Endpoints{ 1469 ObjectMeta: slim_metav1.ObjectMeta{ 1470 Name: "svc-1-ipv4", 1471 Namespace: "default", 1472 }, 1473 EndpointSliceID: k8s.EndpointSliceID{ 1474 ServiceID: k8s.ServiceID{ 1475 Name: "svc-1", 1476 Namespace: "default", 1477 }, 1478 EndpointSliceName: "svc-1-ipv4", 1479 }, 1480 Backends: map[cmtypes.AddrCluster]*k8s.Backend{ 1481 cmtypes.MustParseAddrCluster("10.0.0.1"): { 1482 NodeName: "node1", 1483 }, 1484 }, 1485 } 1486 1487 eps1IPv4Remote := &k8s.Endpoints{ 1488 ObjectMeta: slim_metav1.ObjectMeta{ 1489 Name: "svc-1-ipv4", 1490 Namespace: "default", 1491 }, 1492 EndpointSliceID: k8s.EndpointSliceID{ 1493 ServiceID: k8s.ServiceID{ 1494 Name: "svc-1", 1495 Namespace: "default", 1496 }, 1497 EndpointSliceName: "svc-1-ipv4", 1498 }, 1499 Backends: map[cmtypes.AddrCluster]*k8s.Backend{ 1500 cmtypes.MustParseAddrCluster("10.0.0.2"): { 1501 NodeName: "node2", 1502 }, 1503 }, 1504 } 1505 1506 eps1IPv4Mixed := &k8s.Endpoints{ 1507 ObjectMeta: slim_metav1.ObjectMeta{ 1508 Name: "svc-1-ipv4", 1509 Namespace: "default", 1510 }, 1511 EndpointSliceID: k8s.EndpointSliceID{ 1512 ServiceID: k8s.ServiceID{ 1513 Name: "svc-1", 1514 Namespace: "default", 1515 }, 1516 EndpointSliceName: "svc-1-ipv4", 1517 }, 1518 Backends: map[cmtypes.AddrCluster]*k8s.Backend{ 1519 cmtypes.MustParseAddrCluster("10.0.0.1"): { 1520 NodeName: "node1", 1521 }, 1522 cmtypes.MustParseAddrCluster("10.0.0.2"): { 1523 NodeName: "node2", 1524 }, 1525 }, 1526 } 1527 1528 eps1IPv6Local := &k8s.Endpoints{ 1529 ObjectMeta: slim_metav1.ObjectMeta{ 1530 Name: "svc-1-ipv6", 1531 Namespace: "default", 1532 }, 1533 EndpointSliceID: k8s.EndpointSliceID{ 1534 ServiceID: k8s.ServiceID{ 1535 Name: "svc-1", 1536 Namespace: "default", 1537 }, 1538 EndpointSliceName: "svc-1-ipv6", 1539 }, 1540 Backends: map[cmtypes.AddrCluster]*k8s.Backend{ 1541 cmtypes.MustParseAddrCluster("fd00:10::1"): { 1542 NodeName: "node1", 1543 }, 1544 }, 1545 } 1546 1547 eps1IPv6Remote := &k8s.Endpoints{ 1548 ObjectMeta: slim_metav1.ObjectMeta{ 1549 Name: "svc-1-ipv6", 1550 Namespace: "default", 1551 }, 1552 EndpointSliceID: k8s.EndpointSliceID{ 1553 ServiceID: k8s.ServiceID{ 1554 Name: "svc-1", 1555 Namespace: "default", 1556 }, 1557 EndpointSliceName: "svc-1-ipv6", 1558 }, 1559 Backends: map[cmtypes.AddrCluster]*k8s.Backend{ 1560 cmtypes.MustParseAddrCluster("fd00:10::2"): { 1561 NodeName: "node2", 1562 }, 1563 }, 1564 } 1565 1566 eps1IPv6Mixed := &k8s.Endpoints{ 1567 ObjectMeta: slim_metav1.ObjectMeta{ 1568 Name: "svc-1-ipv4", 1569 Namespace: "default", 1570 }, 1571 EndpointSliceID: k8s.EndpointSliceID{ 1572 ServiceID: k8s.ServiceID{ 1573 Name: "svc-1", 1574 Namespace: "default", 1575 }, 1576 EndpointSliceName: "svc-1-ipv4", 1577 }, 1578 Backends: map[cmtypes.AddrCluster]*k8s.Backend{ 1579 cmtypes.MustParseAddrCluster("fd00:10::1"): { 1580 NodeName: "node1", 1581 }, 1582 cmtypes.MustParseAddrCluster("fd00:10::2"): { 1583 NodeName: "node2", 1584 }, 1585 }, 1586 } 1587 1588 var table = []struct { 1589 // name of the test case 1590 name string 1591 // The service selector of the vRouter 1592 oldServiceSelector *slim_metav1.LabelSelector 1593 // The service selector of the vRouter 1594 newServiceSelector *slim_metav1.LabelSelector 1595 // the advertised PodCIDR blocks the test begins with 1596 advertised map[resource.Key][]string 1597 // the services which will be "upserted" in the diffstore 1598 upsertedServices []*slim_corev1.Service 1599 // the services which will be "deleted" in the diffstore 1600 deletedServices []resource.Key 1601 // the endpoints which will be "upserted" in the diffstore 1602 upsertedEndpoints []*k8s.Endpoints 1603 // the updated PodCIDR blocks to reconcile, these are string encoded 1604 // for the convenience of attaching directly to the NodeSpec.PodCIDRs 1605 // field. 1606 updated map[resource.Key][]string 1607 // error nil or not 1608 err error 1609 }{ 1610 // Add 1 externalIP 1611 { 1612 name: "svc-1-externalIP", 1613 oldServiceSelector: &blueSelector, 1614 newServiceSelector: &blueSelector, 1615 advertised: make(map[resource.Key][]string), 1616 upsertedServices: []*slim_corev1.Service{svc1}, 1617 updated: map[resource.Key][]string{ 1618 svc1Name: { 1619 externalIPV4Prefix, 1620 }, 1621 }, 1622 }, 1623 // Add 2 externalIP 1624 { 1625 name: "svc-2-externalIP", 1626 oldServiceSelector: &blueSelector, 1627 newServiceSelector: &blueSelector, 1628 advertised: make(map[resource.Key][]string), 1629 upsertedServices: []*slim_corev1.Service{svc1TwoIngress}, 1630 updated: map[resource.Key][]string{ 1631 svc1Name: { 1632 externalIPV4Prefix, 1633 externalIPV6Prefix, 1634 }, 1635 }, 1636 }, 1637 // Delete service 1638 { 1639 name: "delete-svc", 1640 oldServiceSelector: &blueSelector, 1641 newServiceSelector: &blueSelector, 1642 advertised: map[resource.Key][]string{ 1643 svc1Name: { 1644 externalIPV4Prefix, 1645 }, 1646 }, 1647 deletedServices: []resource.Key{ 1648 svc1Name, 1649 }, 1650 updated: map[resource.Key][]string{}, 1651 }, 1652 // Update service to no longer match 1653 { 1654 name: "update-service-no-match", 1655 oldServiceSelector: &blueSelector, 1656 newServiceSelector: &blueSelector, 1657 advertised: map[resource.Key][]string{ 1658 svc1Name: { 1659 externalIPV4Prefix, 1660 }, 1661 }, 1662 upsertedServices: []*slim_corev1.Service{svc1RedLabel}, 1663 updated: map[resource.Key][]string{}, 1664 }, 1665 // Update vRouter to no longer match 1666 { 1667 name: "update-vrouter-selector", 1668 oldServiceSelector: &blueSelector, 1669 newServiceSelector: &redSelector, 1670 advertised: map[resource.Key][]string{ 1671 svc1Name: { 1672 externalIPV4Prefix, 1673 }, 1674 }, 1675 upsertedServices: []*slim_corev1.Service{svc1}, 1676 updated: map[resource.Key][]string{}, 1677 }, 1678 // 1 -> 2 externalIP 1679 { 1680 name: "update-1-to-2-externalIP", 1681 oldServiceSelector: &blueSelector, 1682 newServiceSelector: &blueSelector, 1683 advertised: map[resource.Key][]string{ 1684 svc1Name: { 1685 externalIPV4Prefix, 1686 }, 1687 }, 1688 upsertedServices: []*slim_corev1.Service{svc1TwoIngress}, 1689 updated: map[resource.Key][]string{ 1690 svc1Name: { 1691 externalIPV4Prefix, 1692 externalIPV6Prefix, 1693 }, 1694 }, 1695 }, 1696 // No selector 1697 { 1698 name: "no-selector", 1699 oldServiceSelector: nil, 1700 newServiceSelector: nil, 1701 advertised: map[resource.Key][]string{}, 1702 upsertedServices: []*slim_corev1.Service{svc1}, 1703 updated: map[resource.Key][]string{}, 1704 }, 1705 // Namespace selector 1706 { 1707 name: "svc-namespace-selector", 1708 oldServiceSelector: &slim_metav1.LabelSelector{MatchLabels: map[string]string{"io.kubernetes.service.namespace": "default"}}, 1709 newServiceSelector: &slim_metav1.LabelSelector{MatchLabels: map[string]string{"io.kubernetes.service.namespace": "default"}}, 1710 advertised: map[resource.Key][]string{}, 1711 upsertedServices: []*slim_corev1.Service{ 1712 svc1, 1713 svc2NonDefault, 1714 }, 1715 updated: map[resource.Key][]string{ 1716 svc1Name: { 1717 externalIPV4Prefix, 1718 }, 1719 }, 1720 }, 1721 // Service name selector 1722 { 1723 name: "svc-name-selector", 1724 oldServiceSelector: &slim_metav1.LabelSelector{MatchLabels: map[string]string{"io.kubernetes.service.name": "svc-1"}}, 1725 newServiceSelector: &slim_metav1.LabelSelector{MatchLabels: map[string]string{"io.kubernetes.service.name": "svc-1"}}, 1726 advertised: map[resource.Key][]string{}, 1727 upsertedServices: []*slim_corev1.Service{ 1728 svc1, 1729 }, 1730 updated: map[resource.Key][]string{ 1731 svc1Name: { 1732 externalIPV4Prefix, 1733 }, 1734 }, 1735 }, 1736 // BGP load balancer class with matching selectors for service. 1737 { 1738 name: "lb-class-and-selectors", 1739 oldServiceSelector: &blueSelector, 1740 newServiceSelector: &blueSelector, 1741 advertised: map[resource.Key][]string{}, 1742 upsertedServices: []*slim_corev1.Service{svc1LbClass}, 1743 updated: map[resource.Key][]string{ 1744 svc1Name: { 1745 externalIPV4Prefix, 1746 }, 1747 }, 1748 }, 1749 // BGP load balancer class with no selectors for service. 1750 { 1751 name: "lb-class-no-selectors", 1752 oldServiceSelector: nil, 1753 newServiceSelector: nil, 1754 advertised: map[resource.Key][]string{}, 1755 upsertedServices: []*slim_corev1.Service{svc1LbClass}, 1756 updated: map[resource.Key][]string{}, 1757 }, 1758 // BGP load balancer class with selectors for a different service. 1759 { 1760 name: "lb-class-with-diff-selectors", 1761 oldServiceSelector: &slim_metav1.LabelSelector{MatchLabels: map[string]string{"io.kubernetes.service.name": "svc-2"}}, 1762 newServiceSelector: &slim_metav1.LabelSelector{MatchLabels: map[string]string{"io.kubernetes.service.name": "svc-2"}}, 1763 advertised: map[resource.Key][]string{}, 1764 upsertedServices: []*slim_corev1.Service{svc1LbClass}, 1765 updated: map[resource.Key][]string{}, 1766 }, 1767 // Unsupported load balancer class with no matching selectors for service. 1768 { 1769 name: "unsupported-lb-class-with-no-selectors", 1770 oldServiceSelector: nil, 1771 newServiceSelector: nil, 1772 advertised: map[resource.Key][]string{}, 1773 upsertedServices: []*slim_corev1.Service{svc1UnsupportedClass}, 1774 updated: map[resource.Key][]string{}, 1775 }, 1776 // No-externalIP service 1777 { 1778 name: "non-externalIP svc", 1779 oldServiceSelector: &blueSelector, 1780 newServiceSelector: &blueSelector, 1781 advertised: map[resource.Key][]string{}, 1782 upsertedServices: []*slim_corev1.Service{svc1NonExternalIP}, 1783 updated: map[resource.Key][]string{}, 1784 }, 1785 // Service without endpoints 1786 { 1787 name: "etp-local-no-endpoints", 1788 oldServiceSelector: &blueSelector, 1789 newServiceSelector: &blueSelector, 1790 advertised: map[resource.Key][]string{}, 1791 upsertedServices: []*slim_corev1.Service{svc1ETPLocal}, 1792 upsertedEndpoints: []*k8s.Endpoints{}, 1793 updated: map[resource.Key][]string{}, 1794 }, 1795 // externalTrafficPolicy=Local && IPv4 && single slice && local endpoint 1796 { 1797 name: "etp-local-ipv4-single-slice-local", 1798 oldServiceSelector: &blueSelector, 1799 newServiceSelector: &blueSelector, 1800 advertised: map[resource.Key][]string{}, 1801 upsertedServices: []*slim_corev1.Service{svc1ETPLocal}, 1802 upsertedEndpoints: []*k8s.Endpoints{eps1IPv4Local}, 1803 updated: map[resource.Key][]string{ 1804 svc1Name: { 1805 externalIPV4Prefix, 1806 }, 1807 }, 1808 }, 1809 // externalTrafficPolicy=Local && IPv4 && single slice && remote endpoint 1810 { 1811 name: "etp-local-ipv4-single-slice-remote", 1812 oldServiceSelector: &blueSelector, 1813 newServiceSelector: &blueSelector, 1814 advertised: map[resource.Key][]string{}, 1815 upsertedServices: []*slim_corev1.Service{svc1ETPLocal}, 1816 upsertedEndpoints: []*k8s.Endpoints{eps1IPv4Remote}, 1817 updated: map[resource.Key][]string{}, 1818 }, 1819 // externalTrafficPolicy=Local && IPv4 && single slice && mixed endpoint 1820 { 1821 name: "etp-local-ipv4-single-slice-mixed", 1822 oldServiceSelector: &blueSelector, 1823 newServiceSelector: &blueSelector, 1824 advertised: map[resource.Key][]string{}, 1825 upsertedServices: []*slim_corev1.Service{svc1ETPLocal}, 1826 upsertedEndpoints: []*k8s.Endpoints{eps1IPv4Mixed}, 1827 updated: map[resource.Key][]string{ 1828 svc1Name: { 1829 externalIPV4Prefix, 1830 }, 1831 }, 1832 }, 1833 // externalTrafficPolicy=Local && IPv6 && single slice && local endpoint 1834 { 1835 name: "etp-local-ipv6-single-slice-local", 1836 oldServiceSelector: &blueSelector, 1837 newServiceSelector: &blueSelector, 1838 advertised: map[resource.Key][]string{}, 1839 upsertedServices: []*slim_corev1.Service{svc1IPv6ETPLocal}, 1840 upsertedEndpoints: []*k8s.Endpoints{eps1IPv6Local}, 1841 updated: map[resource.Key][]string{ 1842 svc1Name: { 1843 externalIPV4Prefix, 1844 externalIPV6Prefix, 1845 }, 1846 }, 1847 }, 1848 // externalTrafficPolicy=Local && IPv6 && single slice && remote endpoint 1849 { 1850 name: "etp-local-ipv6-single-slice-remote", 1851 oldServiceSelector: &blueSelector, 1852 newServiceSelector: &blueSelector, 1853 advertised: map[resource.Key][]string{}, 1854 upsertedServices: []*slim_corev1.Service{svc1IPv6ETPLocal}, 1855 upsertedEndpoints: []*k8s.Endpoints{eps1IPv6Remote}, 1856 updated: map[resource.Key][]string{}, 1857 }, 1858 // externalTrafficPolicy=Local && IPv6 && single slice && mixed endpoint 1859 { 1860 name: "etp-local-ipv6-single-slice-mixed", 1861 oldServiceSelector: &blueSelector, 1862 newServiceSelector: &blueSelector, 1863 advertised: map[resource.Key][]string{}, 1864 upsertedServices: []*slim_corev1.Service{svc1IPv6ETPLocal}, 1865 upsertedEndpoints: []*k8s.Endpoints{eps1IPv6Mixed}, 1866 updated: map[resource.Key][]string{ 1867 svc1Name: { 1868 externalIPV4Prefix, 1869 externalIPV6Prefix, 1870 }, 1871 }, 1872 }, 1873 // externalTrafficPolicy=Local && Dual && two slices && local endpoint 1874 { 1875 name: "etp-local-dual-two-slices-local", 1876 oldServiceSelector: &blueSelector, 1877 newServiceSelector: &blueSelector, 1878 advertised: map[resource.Key][]string{}, 1879 upsertedServices: []*slim_corev1.Service{svc1ETPLocalTwoIngress}, 1880 upsertedEndpoints: []*k8s.Endpoints{ 1881 eps1IPv4Local, 1882 eps1IPv6Local, 1883 }, 1884 updated: map[resource.Key][]string{ 1885 svc1Name: { 1886 externalIPV4Prefix, 1887 externalIPV6Prefix, 1888 }, 1889 }, 1890 }, 1891 // externalTrafficPolicy=Local && Dual && two slices && remote endpoint 1892 { 1893 name: "etp-local-dual-two-slices-remote", 1894 oldServiceSelector: &blueSelector, 1895 newServiceSelector: &blueSelector, 1896 advertised: map[resource.Key][]string{}, 1897 upsertedServices: []*slim_corev1.Service{svc1ETPLocalTwoIngress}, 1898 upsertedEndpoints: []*k8s.Endpoints{ 1899 eps1IPv4Remote, 1900 eps1IPv6Remote, 1901 }, 1902 updated: map[resource.Key][]string{ 1903 svc1Name: {}, 1904 }, 1905 }, 1906 // externalTrafficPolicy=Local && Dual && two slices && mixed endpoint 1907 { 1908 name: "etp-local-dual-two-slices-mixed", 1909 oldServiceSelector: &blueSelector, 1910 newServiceSelector: &blueSelector, 1911 advertised: map[resource.Key][]string{}, 1912 upsertedServices: []*slim_corev1.Service{svc1ETPLocalTwoIngress}, 1913 upsertedEndpoints: []*k8s.Endpoints{ 1914 eps1IPv4Mixed, 1915 eps1IPv6Mixed, 1916 }, 1917 updated: map[resource.Key][]string{ 1918 svc1Name: { 1919 externalIPV4Prefix, 1920 externalIPV6Prefix, 1921 }, 1922 }, 1923 }, 1924 } 1925 for _, tt := range table { 1926 t.Run(tt.name, func(t *testing.T) { 1927 // setup our test server, create a BgpServer, advertise the tt.advertised 1928 // networks, and store each returned Advertisement in testSC.PodCIDRAnnouncements 1929 srvParams := types.ServerParameters{ 1930 Global: types.BGPGlobal{ 1931 ASN: 64125, 1932 RouterID: "127.0.0.1", 1933 ListenPort: -1, 1934 }, 1935 } 1936 oldc := &v2alpha1api.CiliumBGPVirtualRouter{ 1937 LocalASN: 64125, 1938 Neighbors: []v2alpha1api.CiliumBGPNeighbor{}, 1939 ServiceSelector: tt.oldServiceSelector, 1940 } 1941 testSC, err := instance.NewServerWithConfig(context.Background(), log, srvParams) 1942 if err != nil { 1943 t.Fatalf("failed to create test bgp server: %v", err) 1944 } 1945 testSC.Config = oldc 1946 1947 diffstore := store.NewFakeDiffStore[*slim_corev1.Service]() 1948 epDiffStore := store.NewFakeDiffStore[*k8s.Endpoints]() 1949 1950 reconciler := NewServiceReconciler(diffstore, epDiffStore).Reconciler.(*ServiceReconciler) 1951 reconciler.Init(testSC) 1952 defer reconciler.Cleanup(testSC) 1953 1954 for _, obj := range tt.upsertedServices { 1955 diffstore.Upsert(obj) 1956 } 1957 for _, key := range tt.deletedServices { 1958 diffstore.Delete(key) 1959 } 1960 for _, obj := range tt.upsertedEndpoints { 1961 epDiffStore.Upsert(obj) 1962 } 1963 1964 serviceAnnouncements := reconciler.getMetadata(testSC) 1965 1966 for svcKey, cidrs := range tt.advertised { 1967 for _, cidr := range cidrs { 1968 prefix := netip.MustParsePrefix(cidr) 1969 advrtResp, err := testSC.Server.AdvertisePath(context.Background(), types.PathRequest{ 1970 Path: types.NewPathForPrefix(prefix), 1971 }) 1972 if err != nil { 1973 t.Fatalf("failed to advertise initial svc externalIP cidr routes: %v", err) 1974 } 1975 1976 serviceAnnouncements[svcKey] = append(serviceAnnouncements[svcKey], advrtResp.Path) 1977 } 1978 } 1979 1980 newc := &v2alpha1api.CiliumBGPVirtualRouter{ 1981 LocalASN: 64125, 1982 Neighbors: []v2alpha1api.CiliumBGPNeighbor{}, 1983 ServiceSelector: tt.newServiceSelector, 1984 ServiceAdvertisements: []v2alpha1api.BGPServiceAddressType{v2alpha1api.BGPExternalIPAddr}, 1985 } 1986 1987 // Run the reconciler twice to ensure idempotency. This 1988 // simulates the retrying behavior of the controller. 1989 for i := 0; i < 2; i++ { 1990 t.Run(tt.name, func(t *testing.T) { 1991 err = reconciler.Reconcile(context.Background(), ReconcileParams{ 1992 CurrentServer: testSC, 1993 DesiredConfig: newc, 1994 CiliumNode: &v2api.CiliumNode{ 1995 ObjectMeta: meta_v1.ObjectMeta{ 1996 Name: "node1", 1997 }, 1998 }, 1999 }) 2000 if err != nil { 2001 t.Fatalf("failed to reconcile new externalIP svc advertisements: %v", err) 2002 } 2003 }) 2004 } 2005 2006 // if we disable exports of pod cidr ensure no advertisements are 2007 // still present. 2008 if tt.newServiceSelector == nil && !containsLbClass(tt.upsertedServices) { 2009 if len(serviceAnnouncements) > 0 { 2010 t.Fatal("disabled export but advertisements still present") 2011 } 2012 } 2013 2014 log.Printf("%+v %+v", serviceAnnouncements, tt.updated) 2015 2016 // ensure we see tt.updated in testSC.ServiceAnnouncements 2017 for svcKey, cidrs := range tt.updated { 2018 for _, cidr := range cidrs { 2019 prefix := netip.MustParsePrefix(cidr) 2020 var seen bool 2021 for _, advrt := range serviceAnnouncements[svcKey] { 2022 if advrt.NLRI.String() == prefix.String() { 2023 seen = true 2024 } 2025 } 2026 if !seen { 2027 t.Fatalf("failed to advertise %v", cidr) 2028 } 2029 } 2030 } 2031 2032 // ensure testSC.PodCIDRAnnouncements does not contain advertisements 2033 // not in tt.updated 2034 for svcKey, advrts := range serviceAnnouncements { 2035 for _, advrt := range advrts { 2036 var seen bool 2037 for _, cidr := range tt.updated[svcKey] { 2038 if advrt.NLRI.String() == cidr { 2039 seen = true 2040 } 2041 } 2042 if !seen { 2043 t.Fatalf("unwanted advert %+v", advrt) 2044 } 2045 } 2046 } 2047 2048 }) 2049 } 2050 } 2051 2052 func TestEPUpdateOnly(t *testing.T) { 2053 blueSelector := slim_metav1.LabelSelector{MatchLabels: map[string]string{"color": "blue"}} 2054 svc1Name := resource.Key{Name: "svc-1", Namespace: "default"} 2055 clusterIPV4 := "192.168.0.1" 2056 clusterIPV4Prefix := clusterIPV4 + "/32" 2057 2058 svc1 := &slim_corev1.Service{ 2059 ObjectMeta: slim_metav1.ObjectMeta{ 2060 Name: svc1Name.Name, 2061 Namespace: svc1Name.Namespace, 2062 Labels: blueSelector.MatchLabels, 2063 }, 2064 Spec: slim_corev1.ServiceSpec{ 2065 Type: slim_corev1.ServiceTypeClusterIP, 2066 ClusterIP: clusterIPV4, 2067 ClusterIPs: []string{ 2068 clusterIPV4, 2069 }, 2070 }, 2071 } 2072 2073 svc1WithITP := svc1.DeepCopy() 2074 internalTrafficPolicyLocal := slim_corev1.ServiceInternalTrafficPolicyLocal 2075 svc1WithITP.Spec.InternalTrafficPolicy = &internalTrafficPolicyLocal 2076 2077 eps1IPv4Local := &k8s.Endpoints{ 2078 ObjectMeta: slim_metav1.ObjectMeta{ 2079 Name: "svc-1-ipv4", 2080 Namespace: "default", 2081 }, 2082 EndpointSliceID: k8s.EndpointSliceID{ 2083 ServiceID: k8s.ServiceID{ 2084 Name: "svc-1", 2085 Namespace: "default", 2086 }, 2087 EndpointSliceName: "svc-1-ipv4", 2088 }, 2089 Backends: map[cmtypes.AddrCluster]*k8s.Backend{ 2090 cmtypes.MustParseAddrCluster("10.0.0.1"): { 2091 NodeName: "node1", 2092 }, 2093 }, 2094 } 2095 eps1IPv4LocalUpdated := eps1IPv4Local.DeepCopy() 2096 eps1IPv4LocalUpdated.Backends = map[cmtypes.AddrCluster]*k8s.Backend{ 2097 cmtypes.MustParseAddrCluster("10.0.0.1"): { 2098 NodeName: "node2", 2099 }, 2100 } 2101 2102 steps := []struct { 2103 name string 2104 vr *v2alpha1api.CiliumBGPVirtualRouter 2105 upsertServices []*slim_corev1.Service 2106 upsertEPs []*k8s.Endpoints 2107 expectedMetadata map[resource.Key][]string 2108 }{ 2109 { 2110 name: "initial setup, cluster wide service", 2111 upsertServices: []*slim_corev1.Service{svc1}, 2112 expectedMetadata: map[resource.Key][]string{ 2113 svc1Name: {clusterIPV4Prefix}, 2114 }, 2115 }, 2116 { 2117 name: "set service to internalTrafficPolicy=Local", 2118 upsertServices: []*slim_corev1.Service{svc1WithITP}, 2119 expectedMetadata: map[resource.Key][]string{}, // since there is no local endpoint, no metadata should be added 2120 }, 2121 { 2122 name: "add local endpoint", 2123 upsertServices: []*slim_corev1.Service{}, // no update to service 2124 upsertEPs: []*k8s.Endpoints{eps1IPv4Local}, // update endpoints 2125 expectedMetadata: map[resource.Key][]string{ // metadata should be added 2126 svc1Name: {clusterIPV4Prefix}, 2127 }, 2128 }, 2129 { 2130 name: "remove local endpoint", 2131 upsertServices: []*slim_corev1.Service{}, // no update to service 2132 upsertEPs: []*k8s.Endpoints{eps1IPv4LocalUpdated}, // update endpoint to have backend as node2 2133 expectedMetadata: map[resource.Key][]string{ // metadata should be removed 2134 }, 2135 }, 2136 } 2137 2138 srvParams := types.ServerParameters{ 2139 Global: types.BGPGlobal{ 2140 ASN: 64125, 2141 RouterID: "127.0.0.1", 2142 ListenPort: -1, 2143 }, 2144 } 2145 2146 req := require.New(t) 2147 2148 vr := &v2alpha1api.CiliumBGPVirtualRouter{ 2149 LocalASN: 64125, 2150 Neighbors: []v2alpha1api.CiliumBGPNeighbor{}, 2151 ServiceSelector: &blueSelector, 2152 ServiceAdvertisements: []v2alpha1api.BGPServiceAddressType{v2alpha1api.BGPClusterIPAddr}, 2153 } 2154 2155 testSC, err := instance.NewServerWithConfig(context.Background(), log, srvParams) 2156 req.NoError(err) 2157 2158 testSC.Config = vr 2159 2160 diffstore := store.NewFakeDiffStore[*slim_corev1.Service]() 2161 epDiffStore := store.NewFakeDiffStore[*k8s.Endpoints]() 2162 reconciler := NewServiceReconciler(diffstore, epDiffStore).Reconciler.(*ServiceReconciler) 2163 reconciler.Init(testSC) 2164 defer reconciler.Cleanup(testSC) 2165 2166 for _, step := range steps { 2167 t.Logf("running step: %s", step.name) 2168 2169 for _, svc := range step.upsertServices { 2170 diffstore.Upsert(svc) 2171 } 2172 2173 for _, ep := range step.upsertEPs { 2174 epDiffStore.Upsert(ep) 2175 } 2176 2177 err := reconciler.Reconcile(context.Background(), ReconcileParams{ 2178 CurrentServer: testSC, 2179 DesiredConfig: vr, 2180 CiliumNode: &v2api.CiliumNode{ 2181 ObjectMeta: meta_v1.ObjectMeta{ 2182 Name: "node1", 2183 }, 2184 }, 2185 }) 2186 req.NoError(err) 2187 2188 // running paths 2189 running := make(map[resource.Key][]string) 2190 for key, paths := range reconciler.getMetadata(testSC) { 2191 for _, path := range paths { 2192 running[key] = append(running[key], path.NLRI.String()) 2193 } 2194 } 2195 2196 req.Equal(step.expectedMetadata, running) 2197 } 2198 } 2199 2200 func TestServiceReconcilerWithExternalIPAndClusterIP(t *testing.T) { 2201 blueSelector := slim_metav1.LabelSelector{MatchLabels: map[string]string{"color": "blue"}} 2202 svc1Name := resource.Key{Name: "svc-1", Namespace: "default"} 2203 svc1NonDefaultName := resource.Key{Name: "svc-1", Namespace: "non-default"} 2204 externalIPV4 := "192.168.0.1" 2205 externalIPV4Prefix := externalIPV4 + "/32" 2206 clusterIPV4 := "10.0.100.1" 2207 clusterIPV4Prefix := clusterIPV4 + "/32" 2208 svc1 := &slim_corev1.Service{ 2209 ObjectMeta: slim_metav1.ObjectMeta{ 2210 Name: svc1Name.Name, 2211 Namespace: svc1Name.Namespace, 2212 Labels: blueSelector.MatchLabels, 2213 }, 2214 Spec: slim_corev1.ServiceSpec{ 2215 Type: slim_corev1.ServiceTypeClusterIP, 2216 ClusterIP: clusterIPV4, 2217 ClusterIPs: []string{ 2218 clusterIPV4, 2219 }, 2220 ExternalIPs: []string{ 2221 externalIPV4, 2222 }, 2223 }, 2224 Status: slim_corev1.ServiceStatus{ 2225 LoadBalancer: slim_corev1.LoadBalancerStatus{}, 2226 }, 2227 } 2228 2229 svc1NonDefault := svc1.DeepCopy() 2230 svc1NonDefault.Namespace = svc1NonDefaultName.Namespace 2231 2232 svc1ExternalIPAndClusterIP := svc1.DeepCopy() 2233 svc1ExternalIPAndClusterIP.Spec.ClusterIP = clusterIPV4 2234 svc1ExternalIPAndClusterIP.Spec.ExternalIPs = []string{externalIPV4} 2235 2236 var table = []struct { 2237 // name of the test case 2238 name string 2239 // The service selector of the vRouter 2240 oldServiceSelector *slim_metav1.LabelSelector 2241 // The service selector of the vRouter 2242 newServiceSelector *slim_metav1.LabelSelector 2243 // the advertised PodCIDR blocks the test begins with 2244 advertised map[resource.Key][]string 2245 // the services which will be "upserted" in the diffstore 2246 upsertedServices []*slim_corev1.Service 2247 // the services which will be "deleted" in the diffstore 2248 deletedServices []resource.Key 2249 // the endpoints which will be "upserted" in the diffstore 2250 upsertedEndpoints []*k8s.Endpoints 2251 // the updated PodCIDR blocks to reconcile, these are string encoded 2252 // for the convenience of attaching directly to the NodeSpec.PodCIDRs 2253 // field. 2254 updated map[resource.Key][]string 2255 // error nil or not 2256 err error 2257 }{ 2258 // externalIP and clusterIP service 2259 { 2260 name: "externalIP and clusterIP svc", 2261 oldServiceSelector: &blueSelector, 2262 newServiceSelector: &blueSelector, 2263 advertised: map[resource.Key][]string{}, 2264 upsertedServices: []*slim_corev1.Service{svc1ExternalIPAndClusterIP}, 2265 updated: map[resource.Key][]string{ 2266 svc1Name: { 2267 clusterIPV4Prefix, 2268 externalIPV4Prefix, 2269 }, 2270 }, 2271 }, 2272 } 2273 for _, tt := range table { 2274 t.Run(tt.name, func(t *testing.T) { 2275 // setup our test server, create a BgpServer, advertise the tt.advertised 2276 // networks, and store each returned Advertisement in testSC.PodCIDRAnnouncements 2277 srvParams := types.ServerParameters{ 2278 Global: types.BGPGlobal{ 2279 ASN: 64125, 2280 RouterID: "127.0.0.1", 2281 ListenPort: -1, 2282 }, 2283 } 2284 oldc := &v2alpha1api.CiliumBGPVirtualRouter{ 2285 LocalASN: 64125, 2286 Neighbors: []v2alpha1api.CiliumBGPNeighbor{}, 2287 ServiceSelector: tt.oldServiceSelector, 2288 } 2289 testSC, err := instance.NewServerWithConfig(context.Background(), log, srvParams) 2290 if err != nil { 2291 t.Fatalf("failed to create test bgp server: %v", err) 2292 } 2293 testSC.Config = oldc 2294 2295 diffstore := store.NewFakeDiffStore[*slim_corev1.Service]() 2296 epDiffStore := store.NewFakeDiffStore[*k8s.Endpoints]() 2297 2298 reconciler := NewServiceReconciler(diffstore, epDiffStore).Reconciler.(*ServiceReconciler) 2299 reconciler.Init(testSC) 2300 defer reconciler.Cleanup(testSC) 2301 2302 for _, obj := range tt.upsertedServices { 2303 diffstore.Upsert(obj) 2304 } 2305 for _, key := range tt.deletedServices { 2306 diffstore.Delete(key) 2307 } 2308 for _, obj := range tt.upsertedEndpoints { 2309 epDiffStore.Upsert(obj) 2310 } 2311 2312 serviceAnnouncements := reconciler.getMetadata(testSC) 2313 2314 for svcKey, cidrs := range tt.advertised { 2315 for _, cidr := range cidrs { 2316 prefix := netip.MustParsePrefix(cidr) 2317 advrtResp, err := testSC.Server.AdvertisePath(context.Background(), types.PathRequest{ 2318 Path: types.NewPathForPrefix(prefix), 2319 }) 2320 if err != nil { 2321 t.Fatalf("failed to advertise initial svc externalIP cidr routes: %v", err) 2322 } 2323 2324 serviceAnnouncements[svcKey] = append(serviceAnnouncements[svcKey], advrtResp.Path) 2325 } 2326 } 2327 2328 newc := &v2alpha1api.CiliumBGPVirtualRouter{ 2329 LocalASN: 64125, 2330 Neighbors: []v2alpha1api.CiliumBGPNeighbor{}, 2331 ServiceSelector: tt.newServiceSelector, 2332 ServiceAdvertisements: []v2alpha1api.BGPServiceAddressType{v2alpha1api.BGPExternalIPAddr, v2alpha1api.BGPClusterIPAddr}, 2333 } 2334 2335 // Run the reconciler twice to ensure idempotency. This 2336 // simulates the retrying behavior of the controller. 2337 for i := 0; i < 2; i++ { 2338 t.Run(tt.name, func(t *testing.T) { 2339 err = reconciler.Reconcile(context.Background(), ReconcileParams{ 2340 CurrentServer: testSC, 2341 DesiredConfig: newc, 2342 CiliumNode: &v2api.CiliumNode{ 2343 ObjectMeta: meta_v1.ObjectMeta{ 2344 Name: "node1", 2345 }, 2346 }, 2347 }) 2348 if err != nil { 2349 t.Fatalf("failed to reconcile new externalIP svc advertisements: %v", err) 2350 } 2351 }) 2352 } 2353 2354 // if we disable exports of pod cidr ensure no advertisements are 2355 // still present. 2356 if tt.newServiceSelector == nil && !containsLbClass(tt.upsertedServices) { 2357 if len(serviceAnnouncements) > 0 { 2358 t.Fatal("disabled export but advertisements still present") 2359 } 2360 } 2361 2362 log.Printf("%+v %+v", serviceAnnouncements, tt.updated) 2363 2364 // ensure we see tt.updated in testSC.ServiceAnnouncements 2365 for svcKey, cidrs := range tt.updated { 2366 for _, cidr := range cidrs { 2367 prefix := netip.MustParsePrefix(cidr) 2368 var seen bool 2369 for _, advrt := range serviceAnnouncements[svcKey] { 2370 if advrt.NLRI.String() == prefix.String() { 2371 seen = true 2372 } 2373 } 2374 if !seen { 2375 t.Fatalf("failed to advertise %v", cidr) 2376 } 2377 } 2378 } 2379 2380 // ensure testSC.PodCIDRAnnouncements does not contain advertisements 2381 // not in tt.updated 2382 for svcKey, advrts := range serviceAnnouncements { 2383 for _, advrt := range advrts { 2384 var seen bool 2385 for _, cidr := range tt.updated[svcKey] { 2386 if advrt.NLRI.String() == cidr { 2387 seen = true 2388 } 2389 } 2390 if !seen { 2391 t.Fatalf("unwanted advert %+v", advrt) 2392 } 2393 } 2394 } 2395 2396 }) 2397 } 2398 } 2399 func containsLbClass(svcs []*slim_corev1.Service) bool { 2400 for _, svc := range svcs { 2401 if svc.Spec.LoadBalancerClass != nil && *svc.Spec.LoadBalancerClass == v2alpha1api.BGPLoadBalancerClass { 2402 return true 2403 } 2404 } 2405 return false 2406 }