k8s.io/kubernetes@v1.29.3/test/integration/endpointslice/endpointslicemirroring_test.go (about) 1 /* 2 Copyright 2020 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package endpointslice 18 19 import ( 20 "context" 21 "fmt" 22 "sort" 23 "testing" 24 "time" 25 26 corev1 "k8s.io/api/core/v1" 27 discovery "k8s.io/api/discovery/v1" 28 apiequality "k8s.io/apimachinery/pkg/api/equality" 29 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 30 "k8s.io/apimachinery/pkg/util/wait" 31 "k8s.io/client-go/informers" 32 clientset "k8s.io/client-go/kubernetes" 33 "k8s.io/klog/v2/ktesting" 34 kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" 35 "k8s.io/kubernetes/pkg/controller/endpoint" 36 "k8s.io/kubernetes/pkg/controller/endpointslice" 37 "k8s.io/kubernetes/pkg/controller/endpointslicemirroring" 38 "k8s.io/kubernetes/test/integration/framework" 39 ) 40 41 func TestEndpointSliceMirroring(t *testing.T) { 42 // Disable ServiceAccount admission plugin as we don't have serviceaccount controller running. 43 server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins=ServiceAccount"}, framework.SharedEtcd()) 44 defer server.TearDownFn() 45 46 client, err := clientset.NewForConfig(server.ClientConfig) 47 if err != nil { 48 t.Fatalf("Error creating clientset: %v", err) 49 } 50 51 ctx, cancel := context.WithCancel(context.Background()) 52 resyncPeriod := 12 * time.Hour 53 informers := informers.NewSharedInformerFactory(client, resyncPeriod) 54 55 epController := endpoint.NewEndpointController( 56 informers.Core().V1().Pods(), 57 informers.Core().V1().Services(), 58 informers.Core().V1().Endpoints(), 59 client, 60 1*time.Second) 61 62 epsController := endpointslice.NewController( 63 ctx, 64 informers.Core().V1().Pods(), 65 informers.Core().V1().Services(), 66 informers.Core().V1().Nodes(), 67 informers.Discovery().V1().EndpointSlices(), 68 int32(100), 69 client, 70 1*time.Second) 71 72 epsmController := endpointslicemirroring.NewController( 73 ctx, 74 informers.Core().V1().Endpoints(), 75 informers.Discovery().V1().EndpointSlices(), 76 informers.Core().V1().Services(), 77 int32(100), 78 client, 79 1*time.Second) 80 81 // Start informer and controllers 82 defer cancel() 83 informers.Start(ctx.Done()) 84 go epController.Run(ctx, 5) 85 go epsController.Run(ctx, 5) 86 go epsmController.Run(ctx, 5) 87 88 testCases := []struct { 89 testName string 90 service *corev1.Service 91 customEndpoints *corev1.Endpoints 92 expectEndpointSlice int 93 expectEndpointSliceManagedBy string 94 }{{ 95 testName: "Service with selector", 96 service: &corev1.Service{ 97 ObjectMeta: metav1.ObjectMeta{ 98 Name: "test-123", 99 }, 100 Spec: corev1.ServiceSpec{ 101 Ports: []corev1.ServicePort{{ 102 Port: int32(80), 103 }}, 104 Selector: map[string]string{ 105 "foo": "bar", 106 }, 107 }, 108 }, 109 expectEndpointSlice: 1, 110 expectEndpointSliceManagedBy: "endpointslice-controller.k8s.io", 111 }, { 112 testName: "Service without selector", 113 service: &corev1.Service{ 114 ObjectMeta: metav1.ObjectMeta{ 115 Name: "test-123", 116 }, 117 Spec: corev1.ServiceSpec{ 118 Ports: []corev1.ServicePort{{ 119 Port: int32(80), 120 }}, 121 }, 122 }, 123 customEndpoints: &corev1.Endpoints{ 124 ObjectMeta: metav1.ObjectMeta{ 125 Name: "test-123", 126 }, 127 Subsets: []corev1.EndpointSubset{{ 128 Ports: []corev1.EndpointPort{{ 129 Port: 80, 130 }}, 131 Addresses: []corev1.EndpointAddress{{ 132 IP: "10.0.0.1", 133 }}, 134 }}, 135 }, 136 expectEndpointSlice: 1, 137 expectEndpointSliceManagedBy: "endpointslicemirroring-controller.k8s.io", 138 }, { 139 testName: "Service without selector Endpoint multiple subsets and same address", 140 service: &corev1.Service{ 141 ObjectMeta: metav1.ObjectMeta{ 142 Name: "test-123", 143 }, 144 Spec: corev1.ServiceSpec{ 145 Ports: []corev1.ServicePort{{ 146 Port: int32(80), 147 }}, 148 }, 149 }, 150 customEndpoints: &corev1.Endpoints{ 151 ObjectMeta: metav1.ObjectMeta{ 152 Name: "test-123", 153 }, 154 Subsets: []corev1.EndpointSubset{ 155 { 156 Ports: []corev1.EndpointPort{{ 157 Name: "port1", 158 Port: 80, 159 }}, 160 Addresses: []corev1.EndpointAddress{{ 161 IP: "10.0.0.1", 162 }}, 163 }, 164 { 165 Ports: []corev1.EndpointPort{{ 166 Name: "port2", 167 Port: 90, 168 }}, 169 Addresses: []corev1.EndpointAddress{{ 170 IP: "10.0.0.1", 171 }}, 172 }, 173 }, 174 }, 175 expectEndpointSlice: 1, 176 expectEndpointSliceManagedBy: "endpointslicemirroring-controller.k8s.io", 177 }, { 178 testName: "Service without selector Endpoint multiple subsets", 179 service: &corev1.Service{ 180 ObjectMeta: metav1.ObjectMeta{ 181 Name: "test-123", 182 }, 183 Spec: corev1.ServiceSpec{ 184 Ports: []corev1.ServicePort{{ 185 Port: int32(80), 186 }}, 187 }, 188 }, 189 customEndpoints: &corev1.Endpoints{ 190 ObjectMeta: metav1.ObjectMeta{ 191 Name: "test-123", 192 }, 193 Subsets: []corev1.EndpointSubset{ 194 { 195 Ports: []corev1.EndpointPort{{ 196 Name: "port1", 197 Port: 80, 198 }}, 199 Addresses: []corev1.EndpointAddress{{ 200 IP: "10.0.0.1", 201 }}, 202 }, 203 { 204 Ports: []corev1.EndpointPort{{ 205 Name: "port2", 206 Port: 90, 207 }}, 208 Addresses: []corev1.EndpointAddress{{ 209 IP: "10.0.0.2", 210 }}, 211 }, 212 }, 213 }, 214 expectEndpointSlice: 2, 215 expectEndpointSliceManagedBy: "endpointslicemirroring-controller.k8s.io", 216 }, { 217 testName: "Service without Endpoints", 218 service: &corev1.Service{ 219 ObjectMeta: metav1.ObjectMeta{ 220 Name: "test-123", 221 }, 222 Spec: corev1.ServiceSpec{ 223 Ports: []corev1.ServicePort{{ 224 Port: int32(80), 225 }}, 226 Selector: map[string]string{ 227 "foo": "bar", 228 }, 229 }, 230 }, 231 customEndpoints: nil, 232 expectEndpointSlice: 1, 233 expectEndpointSliceManagedBy: "endpointslice-controller.k8s.io", 234 }, { 235 testName: "Endpoints without Service", 236 service: nil, 237 customEndpoints: &corev1.Endpoints{ 238 ObjectMeta: metav1.ObjectMeta{ 239 Name: "test-123", 240 }, 241 Subsets: []corev1.EndpointSubset{{ 242 Ports: []corev1.EndpointPort{{ 243 Port: 80, 244 }}, 245 Addresses: []corev1.EndpointAddress{{ 246 IP: "10.0.0.1", 247 }}, 248 }}, 249 }, 250 expectEndpointSlice: 0, 251 }} 252 253 for i, tc := range testCases { 254 t.Run(tc.testName, func(t *testing.T) { 255 ns := framework.CreateNamespaceOrDie(client, fmt.Sprintf("test-endpointslice-mirroring-%d", i), t) 256 defer framework.DeleteNamespaceOrDie(client, ns, t) 257 258 resourceName := "" 259 if tc.service != nil { 260 resourceName = tc.service.Name 261 tc.service.Namespace = ns.Name 262 _, err = client.CoreV1().Services(ns.Name).Create(ctx, tc.service, metav1.CreateOptions{}) 263 if err != nil { 264 t.Fatalf("Error creating service: %v", err) 265 } 266 } 267 268 if tc.customEndpoints != nil { 269 resourceName = tc.customEndpoints.Name 270 tc.customEndpoints.Namespace = ns.Name 271 _, err = client.CoreV1().Endpoints(ns.Name).Create(ctx, tc.customEndpoints, metav1.CreateOptions{}) 272 if err != nil { 273 t.Fatalf("Error creating endpoints: %v", err) 274 } 275 } 276 277 err = wait.PollImmediate(1*time.Second, wait.ForeverTestTimeout, func() (bool, error) { 278 lSelector := discovery.LabelServiceName + "=" + resourceName 279 esList, err := client.DiscoveryV1().EndpointSlices(ns.Name).List(ctx, metav1.ListOptions{LabelSelector: lSelector}) 280 if err != nil { 281 t.Logf("Error listing EndpointSlices: %v", err) 282 return false, err 283 } 284 285 if tc.expectEndpointSlice > 0 { 286 if len(esList.Items) < tc.expectEndpointSlice { 287 t.Logf("Waiting for EndpointSlice to be created") 288 return false, nil 289 } 290 if len(esList.Items) != tc.expectEndpointSlice { 291 return false, fmt.Errorf("Only expected %d EndpointSlice, got %d", tc.expectEndpointSlice, len(esList.Items)) 292 } 293 endpointSlice := esList.Items[0] 294 if tc.expectEndpointSliceManagedBy != "" { 295 if endpointSlice.Labels[discovery.LabelManagedBy] != tc.expectEndpointSliceManagedBy { 296 return false, fmt.Errorf("Expected EndpointSlice to be managed by %s, got %s", tc.expectEndpointSliceManagedBy, endpointSlice.Labels[discovery.LabelManagedBy]) 297 } 298 } 299 } else if len(esList.Items) > 0 { 300 t.Logf("Waiting for EndpointSlices to be removed, still %d", len(esList.Items)) 301 return false, nil 302 } 303 304 return true, nil 305 }) 306 if err != nil { 307 t.Fatalf("Timed out waiting for conditions: %v", err) 308 } 309 }) 310 } 311 312 } 313 314 func TestEndpointSliceMirroringUpdates(t *testing.T) { 315 _, ctx := ktesting.NewTestContext(t) 316 // Disable ServiceAccount admission plugin as we don't have serviceaccount controller running. 317 server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins=ServiceAccount"}, framework.SharedEtcd()) 318 defer server.TearDownFn() 319 320 client, err := clientset.NewForConfig(server.ClientConfig) 321 if err != nil { 322 t.Fatalf("Error creating clientset: %v", err) 323 } 324 325 resyncPeriod := 12 * time.Hour 326 informers := informers.NewSharedInformerFactory(client, resyncPeriod) 327 328 epsmController := endpointslicemirroring.NewController( 329 ctx, 330 informers.Core().V1().Endpoints(), 331 informers.Discovery().V1().EndpointSlices(), 332 informers.Core().V1().Services(), 333 int32(100), 334 client, 335 1*time.Second) 336 337 // Start informer and controllers 338 ctx, cancel := context.WithCancel(ctx) 339 defer cancel() 340 informers.Start(ctx.Done()) 341 go epsmController.Run(ctx, 1) 342 343 testCases := []struct { 344 testName string 345 tweakEndpoint func(ep *corev1.Endpoints) 346 }{ 347 { 348 testName: "Update labels", 349 tweakEndpoint: func(ep *corev1.Endpoints) { 350 ep.Labels["foo"] = "bar" 351 }, 352 }, 353 { 354 testName: "Update annotations", 355 tweakEndpoint: func(ep *corev1.Endpoints) { 356 ep.Annotations["foo2"] = "bar2" 357 }, 358 }, 359 { 360 testName: "Update annotations but triggertime", 361 tweakEndpoint: func(ep *corev1.Endpoints) { 362 ep.Annotations["foo2"] = "bar2" 363 ep.Annotations[corev1.EndpointsLastChangeTriggerTime] = "date" 364 }, 365 }, 366 { 367 testName: "Update addresses", 368 tweakEndpoint: func(ep *corev1.Endpoints) { 369 ep.Subsets[0].Addresses = []corev1.EndpointAddress{{IP: "1.2.3.4"}, {IP: "1.2.3.6"}} 370 }, 371 }, 372 } 373 374 for i, tc := range testCases { 375 t.Run(tc.testName, func(t *testing.T) { 376 ns := framework.CreateNamespaceOrDie(client, fmt.Sprintf("test-endpointslice-mirroring-%d", i), t) 377 defer framework.DeleteNamespaceOrDie(client, ns, t) 378 379 service := &corev1.Service{ 380 ObjectMeta: metav1.ObjectMeta{ 381 Name: "test-123", 382 Namespace: ns.Name, 383 }, 384 Spec: corev1.ServiceSpec{ 385 Ports: []corev1.ServicePort{{ 386 Port: int32(80), 387 }}, 388 }, 389 } 390 391 customEndpoints := &corev1.Endpoints{ 392 ObjectMeta: metav1.ObjectMeta{ 393 Name: "test-123", 394 Namespace: ns.Name, 395 Labels: map[string]string{}, 396 Annotations: map[string]string{}, 397 }, 398 Subsets: []corev1.EndpointSubset{{ 399 Ports: []corev1.EndpointPort{{ 400 Port: 80, 401 }}, 402 Addresses: []corev1.EndpointAddress{{ 403 IP: "10.0.0.1", 404 }}, 405 }}, 406 } 407 408 _, err = client.CoreV1().Services(ns.Name).Create(ctx, service, metav1.CreateOptions{}) 409 if err != nil { 410 t.Fatalf("Error creating service: %v", err) 411 } 412 413 _, err = client.CoreV1().Endpoints(ns.Name).Create(ctx, customEndpoints, metav1.CreateOptions{}) 414 if err != nil { 415 t.Fatalf("Error creating endpoints: %v", err) 416 } 417 418 // update endpoint 419 tc.tweakEndpoint(customEndpoints) 420 _, err = client.CoreV1().Endpoints(ns.Name).Update(ctx, customEndpoints, metav1.UpdateOptions{}) 421 if err != nil { 422 t.Fatalf("Error updating endpoints: %v", err) 423 } 424 425 // verify the endpoint updates were mirrored 426 err = wait.PollImmediate(1*time.Second, wait.ForeverTestTimeout, func() (bool, error) { 427 lSelector := discovery.LabelServiceName + "=" + service.Name 428 esList, err := client.DiscoveryV1().EndpointSlices(ns.Name).List(ctx, metav1.ListOptions{LabelSelector: lSelector}) 429 if err != nil { 430 t.Logf("Error listing EndpointSlices: %v", err) 431 return false, err 432 } 433 434 if len(esList.Items) == 0 { 435 t.Logf("Waiting for EndpointSlice to be created") 436 return false, nil 437 } 438 439 for _, endpointSlice := range esList.Items { 440 if endpointSlice.Labels[discovery.LabelManagedBy] != "endpointslicemirroring-controller.k8s.io" { 441 return false, fmt.Errorf("Expected EndpointSlice to be managed by endpointslicemirroring-controller.k8s.io, got %s", endpointSlice.Labels[discovery.LabelManagedBy]) 442 } 443 444 // compare addresses 445 epAddresses := []string{} 446 for _, address := range customEndpoints.Subsets[0].Addresses { 447 epAddresses = append(epAddresses, address.IP) 448 } 449 450 sliceAddresses := []string{} 451 for _, sliceEndpoint := range endpointSlice.Endpoints { 452 sliceAddresses = append(sliceAddresses, sliceEndpoint.Addresses...) 453 } 454 455 sort.Strings(epAddresses) 456 sort.Strings(sliceAddresses) 457 458 if !apiequality.Semantic.DeepEqual(epAddresses, sliceAddresses) { 459 t.Logf("Expected EndpointSlice to have the same IP addresses, expected %v got %v", epAddresses, sliceAddresses) 460 return false, nil 461 } 462 463 // check labels were mirrored 464 if !isSubset(customEndpoints.Labels, endpointSlice.Labels) { 465 t.Logf("Expected EndpointSlice to mirror labels, expected %v to be in received %v", customEndpoints.Labels, endpointSlice.Labels) 466 return false, nil 467 } 468 469 // check annotations but endpoints.kubernetes.io/last-change-trigger-time were mirrored 470 annotations := map[string]string{} 471 for k, v := range customEndpoints.Annotations { 472 if k == corev1.EndpointsLastChangeTriggerTime { 473 continue 474 } 475 annotations[k] = v 476 } 477 if !apiequality.Semantic.DeepEqual(annotations, endpointSlice.Annotations) { 478 t.Logf("Expected EndpointSlice to mirror annotations, expected %v received %v", customEndpoints.Annotations, endpointSlice.Annotations) 479 return false, nil 480 } 481 } 482 return true, nil 483 }) 484 if err != nil { 485 t.Fatalf("Timed out waiting for conditions: %v", err) 486 } 487 }) 488 } 489 } 490 491 func TestEndpointSliceMirroringSelectorTransition(t *testing.T) { 492 _, ctx := ktesting.NewTestContext(t) 493 // Disable ServiceAccount admission plugin as we don't have serviceaccount controller running. 494 server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins=ServiceAccount"}, framework.SharedEtcd()) 495 defer server.TearDownFn() 496 497 client, err := clientset.NewForConfig(server.ClientConfig) 498 if err != nil { 499 t.Fatalf("Error creating clientset: %v", err) 500 } 501 502 resyncPeriod := 12 * time.Hour 503 informers := informers.NewSharedInformerFactory(client, resyncPeriod) 504 505 epsmController := endpointslicemirroring.NewController( 506 ctx, 507 informers.Core().V1().Endpoints(), 508 informers.Discovery().V1().EndpointSlices(), 509 informers.Core().V1().Services(), 510 int32(100), 511 client, 512 1*time.Second) 513 514 // Start informer and controllers 515 ctx, cancel := context.WithCancel(ctx) 516 defer cancel() 517 informers.Start(ctx.Done()) 518 go epsmController.Run(ctx, 1) 519 520 testCases := []struct { 521 testName string 522 startingSelector map[string]string 523 startingMirroredSlices int 524 endingSelector map[string]string 525 endingMirroredSlices int 526 }{ 527 { 528 testName: "nil -> {foo: bar} selector", 529 startingSelector: nil, 530 startingMirroredSlices: 1, 531 endingSelector: map[string]string{"foo": "bar"}, 532 endingMirroredSlices: 0, 533 }, 534 { 535 testName: "{foo: bar} -> nil selector", 536 startingSelector: map[string]string{"foo": "bar"}, 537 startingMirroredSlices: 0, 538 endingSelector: nil, 539 endingMirroredSlices: 1, 540 }, 541 { 542 testName: "{} -> {foo: bar} selector", 543 startingSelector: map[string]string{}, 544 startingMirroredSlices: 1, 545 endingSelector: map[string]string{"foo": "bar"}, 546 endingMirroredSlices: 0, 547 }, 548 { 549 testName: "{foo: bar} -> {} selector", 550 startingSelector: map[string]string{"foo": "bar"}, 551 startingMirroredSlices: 0, 552 endingSelector: map[string]string{}, 553 endingMirroredSlices: 1, 554 }, 555 } 556 557 for i, tc := range testCases { 558 t.Run(tc.testName, func(t *testing.T) { 559 ns := framework.CreateNamespaceOrDie(client, fmt.Sprintf("test-endpointslice-mirroring-%d", i), t) 560 defer framework.DeleteNamespaceOrDie(client, ns, t) 561 meta := metav1.ObjectMeta{Name: "test-123", Namespace: ns.Name} 562 563 service := &corev1.Service{ 564 ObjectMeta: meta, 565 Spec: corev1.ServiceSpec{ 566 Ports: []corev1.ServicePort{{ 567 Port: int32(80), 568 }}, 569 Selector: tc.startingSelector, 570 }, 571 } 572 573 customEndpoints := &corev1.Endpoints{ 574 ObjectMeta: meta, 575 Subsets: []corev1.EndpointSubset{{ 576 Ports: []corev1.EndpointPort{{ 577 Port: 80, 578 }}, 579 Addresses: []corev1.EndpointAddress{{ 580 IP: "10.0.0.1", 581 }}, 582 }}, 583 } 584 585 _, err = client.CoreV1().Services(ns.Name).Create(context.TODO(), service, metav1.CreateOptions{}) 586 if err != nil { 587 t.Fatalf("Error creating service: %v", err) 588 } 589 590 _, err = client.CoreV1().Endpoints(ns.Name).Create(context.TODO(), customEndpoints, metav1.CreateOptions{}) 591 if err != nil { 592 t.Fatalf("Error creating endpoints: %v", err) 593 } 594 595 // verify the expected number of mirrored slices exist 596 err = waitForMirroredSlices(t, client, ns.Name, service.Name, tc.startingMirroredSlices) 597 if err != nil { 598 t.Fatalf("Timed out waiting for initial mirrored slices to match expectations: %v", err) 599 } 600 601 service.Spec.Selector = tc.endingSelector 602 _, err = client.CoreV1().Services(ns.Name).Update(context.TODO(), service, metav1.UpdateOptions{}) 603 if err != nil { 604 t.Fatalf("Error updating service: %v", err) 605 } 606 607 // verify the expected number of mirrored slices exist 608 err = waitForMirroredSlices(t, client, ns.Name, service.Name, tc.endingMirroredSlices) 609 if err != nil { 610 t.Fatalf("Timed out waiting for final mirrored slices to match expectations: %v", err) 611 } 612 }) 613 } 614 } 615 616 func waitForMirroredSlices(t *testing.T, client *clientset.Clientset, nsName, svcName string, num int) error { 617 t.Helper() 618 return wait.PollImmediate(1*time.Second, wait.ForeverTestTimeout, func() (bool, error) { 619 lSelector := discovery.LabelServiceName + "=" + svcName 620 lSelector += "," + discovery.LabelManagedBy + "=endpointslicemirroring-controller.k8s.io" 621 esList, err := client.DiscoveryV1().EndpointSlices(nsName).List(context.TODO(), metav1.ListOptions{LabelSelector: lSelector}) 622 if err != nil { 623 t.Logf("Error listing EndpointSlices: %v", err) 624 return false, err 625 } 626 627 if len(esList.Items) != num { 628 t.Logf("Expected %d slices to be mirrored, got %d", num, len(esList.Items)) 629 return false, nil 630 } 631 632 return true, nil 633 }) 634 } 635 636 // isSubset check if all the elements in a exist in b 637 func isSubset(a, b map[string]string) bool { 638 if len(a) > len(b) { 639 return false 640 } 641 for k, v1 := range a { 642 if v2, ok := b[k]; !ok || v1 != v2 { 643 return false 644 } 645 } 646 return true 647 }