k8s.io/kubernetes@v1.29.3/pkg/proxy/ipvs/proxier_test.go (about) 1 //go:build !windows 2 // +build !windows 3 4 /* 5 Copyright 2017 The Kubernetes Authors. 6 7 Licensed under the Apache License, Version 2.0 (the "License"); 8 you may not use this file except in compliance with the License. 9 You may obtain a copy of the License at 10 11 http://www.apache.org/licenses/LICENSE-2.0 12 13 Unless required by applicable law or agreed to in writing, software 14 distributed under the License is distributed on an "AS IS" BASIS, 15 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 See the License for the specific language governing permissions and 17 limitations under the License. 18 */ 19 20 package ipvs 21 22 import ( 23 "bytes" 24 "fmt" 25 "net" 26 "reflect" 27 "sort" 28 "strings" 29 "testing" 30 "time" 31 32 "github.com/stretchr/testify/assert" 33 v1 "k8s.io/api/core/v1" 34 discovery "k8s.io/api/discovery/v1" 35 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 36 "k8s.io/apimachinery/pkg/types" 37 "k8s.io/apimachinery/pkg/util/intstr" 38 "k8s.io/apimachinery/pkg/util/sets" 39 utilfeature "k8s.io/apiserver/pkg/util/feature" 40 featuregatetesting "k8s.io/component-base/featuregate/testing" 41 "k8s.io/component-base/metrics/testutil" 42 "k8s.io/kubernetes/pkg/features" 43 "k8s.io/kubernetes/pkg/proxy" 44 "k8s.io/kubernetes/pkg/proxy/healthcheck" 45 utilipset "k8s.io/kubernetes/pkg/proxy/ipvs/ipset" 46 ipsettest "k8s.io/kubernetes/pkg/proxy/ipvs/ipset/testing" 47 netlinktest "k8s.io/kubernetes/pkg/proxy/ipvs/testing" 48 utilipvs "k8s.io/kubernetes/pkg/proxy/ipvs/util" 49 ipvstest "k8s.io/kubernetes/pkg/proxy/ipvs/util/testing" 50 "k8s.io/kubernetes/pkg/proxy/metrics" 51 proxyutil "k8s.io/kubernetes/pkg/proxy/util" 52 proxyutiliptables "k8s.io/kubernetes/pkg/proxy/util/iptables" 53 proxyutiltest "k8s.io/kubernetes/pkg/proxy/util/testing" 54 "k8s.io/kubernetes/pkg/util/async" 55 utiliptables "k8s.io/kubernetes/pkg/util/iptables" 56 iptablestest "k8s.io/kubernetes/pkg/util/iptables/testing" 57 "k8s.io/utils/exec" 58 fakeexec "k8s.io/utils/exec/testing" 59 netutils "k8s.io/utils/net" 60 "k8s.io/utils/ptr" 61 ) 62 63 const testHostname = "test-hostname" 64 65 // fakeIpvs implements utilipvs.Interface 66 type fakeIpvs struct { 67 ipvsErr string 68 vsCreated bool 69 } 70 71 func (f *fakeIpvs) Flush() error { 72 return nil 73 } 74 func (f *fakeIpvs) AddVirtualServer(*utilipvs.VirtualServer) error { 75 if f.ipvsErr == "AddVirtualServer" { 76 return fmt.Errorf("oops") 77 } 78 f.vsCreated = true 79 return nil 80 } 81 func (f *fakeIpvs) UpdateVirtualServer(*utilipvs.VirtualServer) error { 82 return nil 83 } 84 func (f *fakeIpvs) DeleteVirtualServer(*utilipvs.VirtualServer) error { 85 if f.ipvsErr == "DeleteVirtualServer" { 86 return fmt.Errorf("oops") 87 } 88 return nil 89 } 90 func (f *fakeIpvs) GetVirtualServer(*utilipvs.VirtualServer) (*utilipvs.VirtualServer, error) { 91 return nil, nil 92 } 93 func (f *fakeIpvs) GetVirtualServers() ([]*utilipvs.VirtualServer, error) { 94 if f.ipvsErr == "GetVirtualServers" { 95 return nil, fmt.Errorf("oops") 96 } 97 if f.vsCreated { 98 vs := []*utilipvs.VirtualServer{{}} 99 return vs, nil 100 } 101 return nil, nil 102 } 103 func (f *fakeIpvs) AddRealServer(*utilipvs.VirtualServer, *utilipvs.RealServer) error { 104 return nil 105 } 106 func (f *fakeIpvs) GetRealServers(*utilipvs.VirtualServer) ([]*utilipvs.RealServer, error) { 107 return nil, nil 108 } 109 func (f *fakeIpvs) DeleteRealServer(*utilipvs.VirtualServer, *utilipvs.RealServer) error { 110 return nil 111 } 112 func (f *fakeIpvs) UpdateRealServer(*utilipvs.VirtualServer, *utilipvs.RealServer) error { 113 return nil 114 } 115 func (f *fakeIpvs) ConfigureTimeouts(time.Duration, time.Duration, time.Duration) error { 116 return nil 117 } 118 119 // fakeIPSetVersioner implements IPSetVersioner. 120 type fakeIPSetVersioner struct { 121 version string 122 err error 123 } 124 125 func (fake *fakeIPSetVersioner) GetVersion() (string, error) { 126 return fake.version, fake.err 127 } 128 129 func NewFakeProxier(ipt utiliptables.Interface, ipvs utilipvs.Interface, ipset utilipset.Interface, nodeIPs []string, excludeCIDRs []*net.IPNet, ipFamily v1.IPFamily) *Proxier { 130 131 netlinkHandle := netlinktest.NewFakeNetlinkHandle(ipFamily == v1.IPv6Protocol) 132 netlinkHandle.SetLocalAddresses("eth0", nodeIPs...) 133 134 fcmd := fakeexec.FakeCmd{ 135 CombinedOutputScript: []fakeexec.FakeAction{ 136 func() ([]byte, []byte, error) { return []byte("dummy device have been created"), nil, nil }, 137 func() ([]byte, []byte, error) { return []byte(""), nil, nil }, 138 }, 139 } 140 fexec := &fakeexec.FakeExec{ 141 CommandScript: []fakeexec.FakeCommandAction{ 142 func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, 143 func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, 144 }, 145 LookPathFunc: func(cmd string) (string, error) { return cmd, nil }, 146 } 147 // initialize ipsetList with all sets we needed 148 ipsetList := make(map[string]*IPSet) 149 for _, is := range ipsetInfo { 150 ipsetList[is.name] = NewIPSet(ipset, is.name, is.setType, false, is.comment) 151 } 152 p := &Proxier{ 153 exec: fexec, 154 svcPortMap: make(proxy.ServicePortMap), 155 serviceChanges: proxy.NewServiceChangeTracker(newServiceInfo, ipFamily, nil, nil), 156 endpointsMap: make(proxy.EndpointsMap), 157 endpointsChanges: proxy.NewEndpointsChangeTracker(testHostname, nil, ipFamily, nil, nil), 158 excludeCIDRs: excludeCIDRs, 159 iptables: ipt, 160 ipvs: ipvs, 161 ipset: ipset, 162 strictARP: false, 163 localDetector: proxyutiliptables.NewNoOpLocalDetector(), 164 hostname: testHostname, 165 serviceHealthServer: healthcheck.NewFakeServiceHealthServer(), 166 ipvsScheduler: defaultScheduler, 167 iptablesData: bytes.NewBuffer(nil), 168 filterChainsData: bytes.NewBuffer(nil), 169 natChains: proxyutil.NewLineBuffer(), 170 natRules: proxyutil.NewLineBuffer(), 171 filterChains: proxyutil.NewLineBuffer(), 172 filterRules: proxyutil.NewLineBuffer(), 173 netlinkHandle: netlinkHandle, 174 ipsetList: ipsetList, 175 nodePortAddresses: proxyutil.NewNodePortAddresses(ipFamily, nil), 176 networkInterfacer: proxyutiltest.NewFakeNetwork(), 177 gracefuldeleteManager: NewGracefulTerminationManager(ipvs), 178 ipFamily: ipFamily, 179 } 180 p.setInitialized(true) 181 p.syncRunner = async.NewBoundedFrequencyRunner("test-sync-runner", p.syncProxyRules, 0, time.Minute, 1) 182 return p 183 } 184 185 func makeNSN(namespace, name string) types.NamespacedName { 186 return types.NamespacedName{Namespace: namespace, Name: name} 187 } 188 189 func makeServiceMap(proxier *Proxier, allServices ...*v1.Service) { 190 for i := range allServices { 191 proxier.OnServiceAdd(allServices[i]) 192 } 193 194 proxier.mu.Lock() 195 defer proxier.mu.Unlock() 196 proxier.servicesSynced = true 197 } 198 199 func makeEndpointSliceMap(proxier *Proxier, allEpSlices ...*discovery.EndpointSlice) { 200 for i := range allEpSlices { 201 proxier.OnEndpointSliceAdd(allEpSlices[i]) 202 } 203 proxier.mu.Lock() 204 defer proxier.mu.Unlock() 205 proxier.endpointSlicesSynced = true 206 } 207 208 func makeTestService(namespace, name string, svcFunc func(*v1.Service)) *v1.Service { 209 svc := &v1.Service{ 210 ObjectMeta: metav1.ObjectMeta{ 211 Name: name, 212 Namespace: namespace, 213 Annotations: map[string]string{}, 214 }, 215 Spec: v1.ServiceSpec{}, 216 Status: v1.ServiceStatus{}, 217 } 218 svcFunc(svc) 219 return svc 220 } 221 222 func populateEndpointSlices(proxier *Proxier, allEndpointSlices ...*discovery.EndpointSlice) { 223 for i := range allEndpointSlices { 224 proxier.OnEndpointSliceAdd(allEndpointSlices[i]) 225 } 226 } 227 228 func makeTestEndpointSlice(namespace, name string, sliceNum int, epsFunc func(*discovery.EndpointSlice)) *discovery.EndpointSlice { 229 eps := &discovery.EndpointSlice{ 230 ObjectMeta: metav1.ObjectMeta{ 231 Name: fmt.Sprintf("%s-%d", name, sliceNum), 232 Namespace: namespace, 233 Labels: map[string]string{discovery.LabelServiceName: name}, 234 }, 235 } 236 epsFunc(eps) 237 return eps 238 } 239 240 func TestCleanupLeftovers(t *testing.T) { 241 ipt := iptablestest.NewFake() 242 ipvs := ipvstest.NewFake() 243 ipset := ipsettest.NewFake(testIPSetVersion) 244 fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol) 245 svcIP := "10.20.30.41" 246 svcPort := 80 247 svcNodePort := 3001 248 svcPortName := proxy.ServicePortName{ 249 NamespacedName: makeNSN("ns1", "svc1"), 250 Port: "p80", 251 } 252 253 makeServiceMap(fp, 254 makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { 255 svc.Spec.Type = "NodePort" 256 svc.Spec.ClusterIP = svcIP 257 svc.Spec.Ports = []v1.ServicePort{{ 258 Name: svcPortName.Port, 259 Port: int32(svcPort), 260 Protocol: v1.ProtocolTCP, 261 NodePort: int32(svcNodePort), 262 }} 263 }), 264 ) 265 epIP := "10.180.0.1" 266 populateEndpointSlices(fp, 267 makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) { 268 eps.AddressType = discovery.AddressTypeIPv4 269 eps.Endpoints = []discovery.Endpoint{{ 270 Addresses: []string{epIP}, 271 }} 272 eps.Ports = []discovery.EndpointPort{{ 273 Name: ptr.To(svcPortName.Port), 274 Port: ptr.To(int32(svcPort)), 275 Protocol: ptr.To(v1.ProtocolTCP), 276 }} 277 }), 278 ) 279 280 fp.syncProxyRules() 281 282 // test cleanup left over 283 if CleanupLeftovers(ipvs, ipt, ipset) { 284 t.Errorf("Cleanup leftovers failed") 285 } 286 } 287 288 func TestCanUseIPVSProxier(t *testing.T) { 289 testCases := []struct { 290 name string 291 scheduler string 292 ipsetVersion string 293 ipsetErr error 294 ipvsErr string 295 ok bool 296 }{ 297 { 298 name: "happy days", 299 ipsetVersion: MinIPSetCheckVersion, 300 ok: true, 301 }, 302 { 303 name: "ipset error", 304 scheduler: "", 305 ipsetVersion: MinIPSetCheckVersion, 306 ipsetErr: fmt.Errorf("oops"), 307 ok: false, 308 }, 309 { 310 name: "ipset version too low", 311 scheduler: "rr", 312 ipsetVersion: "4.3.0", 313 ok: false, 314 }, 315 { 316 name: "GetVirtualServers fail", 317 ipsetVersion: MinIPSetCheckVersion, 318 ipvsErr: "GetVirtualServers", 319 ok: false, 320 }, 321 { 322 name: "AddVirtualServer fail", 323 ipsetVersion: MinIPSetCheckVersion, 324 ipvsErr: "AddVirtualServer", 325 ok: false, 326 }, 327 { 328 name: "DeleteVirtualServer fail", 329 ipsetVersion: MinIPSetCheckVersion, 330 ipvsErr: "DeleteVirtualServer", 331 ok: false, 332 }, 333 } 334 335 for _, tc := range testCases { 336 ipvs := &fakeIpvs{tc.ipvsErr, false} 337 versioner := &fakeIPSetVersioner{version: tc.ipsetVersion, err: tc.ipsetErr} 338 err := CanUseIPVSProxier(ipvs, versioner, tc.scheduler) 339 if (err == nil) != tc.ok { 340 t.Errorf("Case [%s], expect %v, got err: %v", tc.name, tc.ok, err) 341 } 342 } 343 } 344 345 func TestGetNodeIPs(t *testing.T) { 346 testCases := []struct { 347 isIPv6 bool 348 devAddresses map[string][]string 349 expectIPs []string 350 }{ 351 // case 0 352 { 353 devAddresses: map[string][]string{"eth0": {"1.2.3.4"}, "lo": {"127.0.0.1"}}, 354 expectIPs: []string{"1.2.3.4"}, 355 }, 356 // case 1 357 { 358 devAddresses: map[string][]string{"lo": {"127.0.0.1"}}, 359 expectIPs: []string{}, 360 }, 361 // case 2 362 { 363 devAddresses: map[string][]string{}, 364 expectIPs: []string{}, 365 }, 366 // case 3 367 { 368 devAddresses: map[string][]string{"encap0": {"10.20.30.40", "fe80::200:ff:fe01:1"}, "lo": {"127.0.0.1", "::1"}, "docker0": {"172.17.0.1"}}, 369 expectIPs: []string{"10.20.30.40", "172.17.0.1"}, 370 }, 371 // case 4 372 { 373 devAddresses: map[string][]string{"encaps9": {"10.20.30.40"}, "lo": {"127.0.0.1", "::1"}, "encap7": {"1000::", "10.20.30.31"}}, 374 expectIPs: []string{"10.20.30.40", "10.20.30.31"}, 375 }, 376 // case 5 377 { 378 devAddresses: map[string][]string{"kube-ipvs0": {"2000::", "1.2.3.4"}, "lo": {"127.0.0.1", "::1"}, "encap7": {"1000::", "10.20.30.31"}}, 379 expectIPs: []string{"10.20.30.31"}, 380 }, 381 // case 6 382 { 383 devAddresses: map[string][]string{"kube-ipvs0": {"1.2.3.4", "2.3.4.5"}, "lo": {"127.0.0.1", "::1"}}, 384 expectIPs: []string{}, 385 }, 386 // case 7 387 { 388 devAddresses: map[string][]string{"kube-ipvs0": {"1.2.3.4", "2.3.4.5"}}, 389 expectIPs: []string{}, 390 }, 391 // case 8 392 { 393 devAddresses: map[string][]string{"kube-ipvs0": {"1.2.3.4", "2.3.4.5"}, "eth5": {"3.4.5.6"}, "lo": {"127.0.0.1", "::1"}}, 394 expectIPs: []string{"3.4.5.6"}, 395 }, 396 // case 9 397 { 398 devAddresses: map[string][]string{"ipvs0": {"1.2.3.4"}, "lo": {"127.0.0.1", "::1"}, "encap7": {"10.20.30.31"}}, 399 expectIPs: []string{"10.20.30.31", "1.2.3.4"}, 400 }, 401 // case 10 402 { 403 isIPv6: true, 404 devAddresses: map[string][]string{"ipvs0": {"1.2.3.4", "1000::"}, "lo": {"127.0.0.1", "::1"}, "encap7": {"10.20.30.31", "2000::", "fe80::200:ff:fe01:1"}}, 405 expectIPs: []string{"1000::", "2000::"}, 406 }, 407 // case 11 408 { 409 isIPv6: true, 410 devAddresses: map[string][]string{"ipvs0": {"1.2.3.4", "1000::"}, "lo": {"127.0.0.1", "::1"}, "encap7": {"10.20.30.31", "2000::", "fe80::200:ff:fe01:1"}, "kube-ipvs0": {"1.2.3.4", "2.3.4.5", "2000::"}}, 411 expectIPs: []string{"1000::"}, 412 }, 413 } 414 415 for i, tc := range testCases { 416 fake := netlinktest.NewFakeNetlinkHandle(tc.isIPv6) 417 for dev, addresses := range testCases[i].devAddresses { 418 fake.SetLocalAddresses(dev, addresses...) 419 } 420 ips, err := fake.GetAllLocalAddresses() 421 if err != nil { 422 t.Errorf("Unexpected error: %v", err) 423 } 424 devIps, err := fake.GetLocalAddresses("kube-ipvs0") 425 if err != nil { 426 t.Errorf("Unexpected error: %v", err) 427 } 428 ips = ips.Difference(devIps) 429 if !ips.Equal(sets.New(tc.expectIPs...)) { 430 t.Errorf("case[%d], unexpected mismatch, expected: %v, got: %v", i, tc.expectIPs, ips) 431 } 432 } 433 } 434 435 func TestNodePortIPv4(t *testing.T) { 436 tests := []struct { 437 name string 438 services []*v1.Service 439 endpoints []*discovery.EndpointSlice 440 nodeIPs []string 441 nodePortAddresses []string 442 expectedIPVS *ipvstest.FakeIPVS 443 expectedIPSets netlinktest.ExpectedIPSet 444 expectedIptablesChains netlinktest.ExpectedIptablesChain 445 }{ 446 { 447 name: "1 service with node port, has 2 endpoints", 448 services: []*v1.Service{ 449 makeTestService("ns1", "svc1", func(svc *v1.Service) { 450 svc.Spec.Type = "NodePort" 451 svc.Spec.ClusterIP = "10.20.30.41" 452 svc.Spec.Ports = []v1.ServicePort{{ 453 Name: "p80", 454 Port: int32(80), 455 Protocol: v1.ProtocolTCP, 456 NodePort: int32(3001), 457 }} 458 }), 459 }, 460 endpoints: []*discovery.EndpointSlice{ 461 makeTestEndpointSlice("ns1", "svc1", 1, func(eps *discovery.EndpointSlice) { 462 eps.AddressType = discovery.AddressTypeIPv4 463 eps.Endpoints = []discovery.Endpoint{{ 464 Addresses: []string{"10.180.0.1"}, 465 }} 466 eps.Ports = []discovery.EndpointPort{{ 467 Name: ptr.To("p80"), 468 Port: ptr.To[int32](80), 469 Protocol: ptr.To(v1.ProtocolTCP), 470 }} 471 }), 472 makeTestEndpointSlice("ns1", "svc1", 2, func(eps *discovery.EndpointSlice) { 473 eps.AddressType = discovery.AddressTypeIPv6 474 eps.Endpoints = []discovery.Endpoint{{ 475 Addresses: []string{"1002:ab8::2:10"}, 476 }} 477 eps.Ports = []discovery.EndpointPort{{ 478 Name: ptr.To("p80"), 479 Port: ptr.To[int32](80), 480 Protocol: ptr.To(v1.ProtocolTCP), 481 }} 482 }), 483 }, 484 nodeIPs: []string{"100.101.102.103", "2001:db8::1:1"}, 485 nodePortAddresses: []string{}, 486 expectedIPVS: &ipvstest.FakeIPVS{ 487 Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{ 488 { 489 IP: "10.20.30.41", 490 Port: 80, 491 Protocol: "TCP", 492 }: { 493 Address: netutils.ParseIPSloppy("10.20.30.41"), 494 Protocol: "TCP", 495 Port: uint16(80), 496 Scheduler: "rr", 497 }, 498 { 499 IP: "100.101.102.103", 500 Port: 3001, 501 Protocol: "TCP", 502 }: { 503 Address: netutils.ParseIPSloppy("100.101.102.103"), 504 Protocol: "TCP", 505 Port: uint16(3001), 506 Scheduler: "rr", 507 }, 508 }, 509 Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{ 510 { 511 IP: "10.20.30.41", 512 Port: 80, 513 Protocol: "TCP", 514 }: { 515 { 516 Address: netutils.ParseIPSloppy("10.180.0.1"), 517 Port: uint16(80), 518 Weight: 1, 519 }, 520 }, 521 { 522 IP: "100.101.102.103", 523 Port: 3001, 524 Protocol: "TCP", 525 }: { 526 { 527 Address: netutils.ParseIPSloppy("10.180.0.1"), 528 Port: uint16(80), 529 Weight: 1, 530 }, 531 }, 532 }, 533 }, 534 }, 535 { 536 name: "1 UDP service with node port, has endpoints", 537 services: []*v1.Service{ 538 makeTestService("ns1", "svc1", func(svc *v1.Service) { 539 svc.Spec.Type = "NodePort" 540 svc.Spec.ClusterIP = "10.20.30.41" 541 svc.Spec.Ports = []v1.ServicePort{{ 542 Name: "p80", 543 Port: int32(80), 544 Protocol: v1.ProtocolUDP, 545 NodePort: int32(3001), 546 }} 547 }), 548 }, 549 endpoints: []*discovery.EndpointSlice{ 550 makeTestEndpointSlice("ns1", "svc1", 1, func(eps *discovery.EndpointSlice) { 551 eps.AddressType = discovery.AddressTypeIPv4 552 eps.Endpoints = []discovery.Endpoint{{ 553 Addresses: []string{"10.180.0.1"}, 554 }} 555 eps.Ports = []discovery.EndpointPort{{ 556 Name: ptr.To("p80"), 557 Port: ptr.To[int32](80), 558 Protocol: ptr.To(v1.ProtocolUDP), 559 }} 560 }), 561 }, 562 nodeIPs: []string{"100.101.102.103"}, 563 nodePortAddresses: []string{"0.0.0.0/0"}, 564 expectedIPVS: &ipvstest.FakeIPVS{ 565 Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{ 566 { 567 IP: "10.20.30.41", 568 Port: 80, 569 Protocol: "UDP", 570 }: { 571 Address: netutils.ParseIPSloppy("10.20.30.41"), 572 Protocol: "UDP", 573 Port: uint16(80), 574 Scheduler: "rr", 575 }, 576 { 577 IP: "100.101.102.103", 578 Port: 3001, 579 Protocol: "UDP", 580 }: { 581 Address: netutils.ParseIPSloppy("100.101.102.103"), 582 Protocol: "UDP", 583 Port: uint16(3001), 584 Scheduler: "rr", 585 }, 586 }, 587 Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{ 588 { 589 IP: "10.20.30.41", 590 Port: 80, 591 Protocol: "UDP", 592 }: { 593 { 594 Address: netutils.ParseIPSloppy("10.180.0.1"), 595 Port: uint16(80), 596 Weight: 1, 597 }, 598 }, 599 { 600 IP: "100.101.102.103", 601 Port: 3001, 602 Protocol: "UDP", 603 }: { 604 { 605 Address: netutils.ParseIPSloppy("10.180.0.1"), 606 Port: uint16(80), 607 Weight: 1, 608 }, 609 }, 610 }, 611 }, 612 expectedIPSets: netlinktest.ExpectedIPSet{ 613 kubeNodePortSetUDP: {{ 614 Port: 3001, 615 Protocol: strings.ToLower(string(v1.ProtocolUDP)), 616 SetType: utilipset.BitmapPort, 617 }}, 618 }, 619 expectedIptablesChains: netlinktest.ExpectedIptablesChain{ 620 string(kubeNodePortChain): {{ 621 JumpChain: string(kubeMarkMasqChain), MatchSet: kubeNodePortSetUDP, 622 }, { 623 JumpChain: "ACCEPT", MatchSet: kubeHealthCheckNodePortSet, 624 }}, 625 string(kubeServicesChain): {{ 626 JumpChain: "RETURN", SourceAddress: "127.0.0.0/8", 627 }, { 628 JumpChain: string(kubeMarkMasqChain), MatchSet: kubeClusterIPSet, 629 }, { 630 JumpChain: string(kubeNodePortChain), MatchSet: "", 631 }, { 632 JumpChain: "ACCEPT", MatchSet: kubeClusterIPSet, 633 }}, 634 }, 635 }, 636 { 637 name: "service has node port but no endpoints", 638 services: []*v1.Service{ 639 makeTestService("ns1", "svc1", func(svc *v1.Service) { 640 svc.Spec.Type = "NodePort" 641 svc.Spec.ClusterIP = "10.20.30.41" 642 svc.Spec.Ports = []v1.ServicePort{{ 643 Name: "p80", 644 Port: int32(80), 645 Protocol: v1.ProtocolTCP, 646 NodePort: int32(3001), 647 }} 648 }), 649 }, 650 endpoints: []*discovery.EndpointSlice{}, 651 nodeIPs: []string{"100.101.102.103"}, 652 nodePortAddresses: []string{}, 653 expectedIPVS: &ipvstest.FakeIPVS{ 654 Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{ 655 { 656 IP: "10.20.30.41", 657 Port: 80, 658 Protocol: "TCP", 659 }: { 660 Address: netutils.ParseIPSloppy("10.20.30.41"), 661 Protocol: "TCP", 662 Port: uint16(80), 663 Scheduler: "rr", 664 }, 665 { 666 IP: "100.101.102.103", 667 Port: 3001, 668 Protocol: "TCP", 669 }: { 670 Address: netutils.ParseIPSloppy("100.101.102.103"), 671 Protocol: "TCP", 672 Port: uint16(3001), 673 Scheduler: "rr", 674 }, 675 }, 676 Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{ 677 { 678 IP: "10.20.30.41", 679 Port: 80, 680 Protocol: "TCP", 681 }: {}, // no real servers corresponding to no endpoints 682 { 683 IP: "100.101.102.103", 684 Port: 3001, 685 Protocol: "TCP", 686 }: {}, // no real servers corresponding to no endpoints 687 }, 688 }, 689 }, 690 { 691 name: "node port service with protocol sctp on a node with multiple nodeIPs", 692 services: []*v1.Service{ 693 makeTestService("ns1", "svc1", func(svc *v1.Service) { 694 svc.Spec.Type = "NodePort" 695 svc.Spec.ClusterIP = "10.20.30.41" 696 svc.Spec.Ports = []v1.ServicePort{{ 697 Name: "p80", 698 Port: int32(80), 699 Protocol: v1.ProtocolSCTP, 700 NodePort: int32(3001), 701 }} 702 }), 703 }, 704 endpoints: []*discovery.EndpointSlice{ 705 makeTestEndpointSlice("ns1", "svc1", 1, func(eps *discovery.EndpointSlice) { 706 eps.AddressType = discovery.AddressTypeIPv4 707 eps.Endpoints = []discovery.Endpoint{{ 708 Addresses: []string{"10.180.0.1"}, 709 }} 710 eps.Ports = []discovery.EndpointPort{{ 711 Name: ptr.To("p80"), 712 Port: ptr.To[int32](80), 713 Protocol: ptr.To(v1.ProtocolSCTP), 714 }} 715 }), 716 }, 717 nodeIPs: []string{ 718 "100.101.102.103", 719 "100.101.102.104", 720 "100.101.102.105", 721 "2001:db8::1:1", 722 "2001:db8::1:2", 723 "2001:db8::1:3", 724 }, 725 nodePortAddresses: []string{}, 726 expectedIPVS: &ipvstest.FakeIPVS{ 727 Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{ 728 { 729 IP: "10.20.30.41", 730 Port: 80, 731 Protocol: "SCTP", 732 }: { 733 Address: netutils.ParseIPSloppy("10.20.30.41"), 734 Protocol: "SCTP", 735 Port: uint16(80), 736 Scheduler: "rr", 737 }, 738 { 739 IP: "100.101.102.103", 740 Port: 3001, 741 Protocol: "SCTP", 742 }: { 743 Address: netutils.ParseIPSloppy("100.101.102.103"), 744 Protocol: "SCTP", 745 Port: uint16(3001), 746 Scheduler: "rr", 747 }, 748 { 749 IP: "100.101.102.104", 750 Port: 3001, 751 Protocol: "SCTP", 752 }: { 753 Address: netutils.ParseIPSloppy("100.101.102.104"), 754 Protocol: "SCTP", 755 Port: uint16(3001), 756 Scheduler: "rr", 757 }, 758 { 759 IP: "100.101.102.105", 760 Port: 3001, 761 Protocol: "SCTP", 762 }: { 763 Address: netutils.ParseIPSloppy("100.101.102.105"), 764 Protocol: "SCTP", 765 Port: uint16(3001), 766 Scheduler: "rr", 767 }, 768 }, 769 Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{ 770 { 771 IP: "10.20.30.41", 772 Port: 80, 773 Protocol: "SCTP", 774 }: { 775 { 776 Address: netutils.ParseIPSloppy("10.180.0.1"), 777 Port: uint16(80), 778 Weight: 1, 779 }, 780 }, 781 { 782 IP: "100.101.102.103", 783 Port: 3001, 784 Protocol: "SCTP", 785 }: { 786 { 787 Address: netutils.ParseIPSloppy("10.180.0.1"), 788 Port: uint16(80), 789 Weight: 1, 790 }, 791 }, 792 { 793 IP: "100.101.102.104", 794 Port: 3001, 795 Protocol: "SCTP", 796 }: { 797 { 798 Address: netutils.ParseIPSloppy("10.180.0.1"), 799 Port: uint16(80), 800 Weight: 1, 801 }, 802 }, 803 { 804 IP: "100.101.102.105", 805 Port: 3001, 806 Protocol: "SCTP", 807 }: { 808 { 809 Address: netutils.ParseIPSloppy("10.180.0.1"), 810 Port: uint16(80), 811 Weight: 1, 812 }, 813 }, 814 }, 815 }, 816 expectedIPSets: netlinktest.ExpectedIPSet{ 817 kubeNodePortSetSCTP: { 818 { 819 IP: "100.101.102.103", 820 Port: 3001, 821 Protocol: strings.ToLower(string(v1.ProtocolSCTP)), 822 SetType: utilipset.HashIPPort, 823 }, 824 { 825 IP: "100.101.102.104", 826 Port: 3001, 827 Protocol: strings.ToLower(string(v1.ProtocolSCTP)), 828 SetType: utilipset.HashIPPort, 829 }, 830 { 831 IP: "100.101.102.105", 832 Port: 3001, 833 Protocol: strings.ToLower(string(v1.ProtocolSCTP)), 834 SetType: utilipset.HashIPPort, 835 }, 836 }, 837 }, 838 }, 839 { 840 name: "node port service with protocol sctp and externalTrafficPolicy local", 841 services: []*v1.Service{ 842 makeTestService("ns1", "svc1", func(svc *v1.Service) { 843 svc.Spec.Type = "NodePort" 844 svc.Spec.ClusterIP = "10.20.30.41" 845 svc.Spec.Ports = []v1.ServicePort{{ 846 Name: "p80", 847 Port: int32(80), 848 Protocol: v1.ProtocolSCTP, 849 NodePort: int32(3001), 850 }} 851 svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyLocal 852 }), 853 }, 854 endpoints: []*discovery.EndpointSlice{ 855 makeTestEndpointSlice("ns1", "svc1", 1, func(eps *discovery.EndpointSlice) { 856 eps.AddressType = discovery.AddressTypeIPv4 857 eps.Endpoints = []discovery.Endpoint{{ 858 Addresses: []string{"10.180.0.1"}, 859 NodeName: ptr.To(testHostname), 860 }, { 861 Addresses: []string{"10.180.1.1"}, 862 NodeName: ptr.To("otherHost"), 863 }} 864 eps.Ports = []discovery.EndpointPort{{ 865 Name: ptr.To("p80"), 866 Port: ptr.To[int32](80), 867 Protocol: ptr.To(v1.ProtocolSCTP), 868 }} 869 }), 870 }, 871 nodeIPs: []string{"100.101.102.103"}, 872 nodePortAddresses: []string{}, 873 expectedIPVS: &ipvstest.FakeIPVS{ 874 Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{ 875 { 876 IP: "10.20.30.41", 877 Port: 80, 878 Protocol: "SCTP", 879 }: { 880 Address: netutils.ParseIPSloppy("10.20.30.41"), 881 Protocol: "SCTP", 882 Port: uint16(80), 883 Scheduler: "rr", 884 }, 885 { 886 IP: "100.101.102.103", 887 Port: 3001, 888 Protocol: "SCTP", 889 }: { 890 Address: netutils.ParseIPSloppy("100.101.102.103"), 891 Protocol: "SCTP", 892 Port: uint16(3001), 893 Scheduler: "rr", 894 }, 895 }, 896 Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{ 897 { 898 IP: "10.20.30.41", 899 Port: 80, 900 Protocol: "SCTP", 901 }: { 902 { 903 Address: netutils.ParseIPSloppy("10.180.0.1"), 904 Port: uint16(80), 905 Weight: 1, 906 }, 907 { 908 Address: netutils.ParseIPSloppy("10.180.1.1"), 909 Port: uint16(80), 910 Weight: 1, 911 }, 912 }, 913 { 914 IP: "100.101.102.103", 915 Port: 3001, 916 Protocol: "SCTP", 917 }: { 918 { 919 Address: netutils.ParseIPSloppy("10.180.0.1"), 920 Port: uint16(80), 921 Weight: 1, 922 }, 923 }, 924 }, 925 }, 926 expectedIPSets: netlinktest.ExpectedIPSet{ 927 kubeNodePortSetSCTP: { 928 { 929 IP: "100.101.102.103", 930 Port: 3001, 931 Protocol: strings.ToLower(string(v1.ProtocolSCTP)), 932 SetType: utilipset.HashIPPort, 933 }, 934 }, 935 kubeNodePortLocalSetSCTP: { 936 { 937 IP: "100.101.102.103", 938 Port: 3001, 939 Protocol: strings.ToLower(string(v1.ProtocolSCTP)), 940 SetType: utilipset.HashIPPort, 941 }, 942 }, 943 }, 944 expectedIptablesChains: netlinktest.ExpectedIptablesChain{ 945 string(kubeNodePortChain): {{ 946 JumpChain: "RETURN", MatchSet: kubeNodePortLocalSetSCTP, 947 }, { 948 JumpChain: string(kubeMarkMasqChain), MatchSet: kubeNodePortSetSCTP, 949 }, { 950 JumpChain: "ACCEPT", MatchSet: kubeHealthCheckNodePortSet, 951 }}, 952 }, 953 }, 954 } 955 956 for _, test := range tests { 957 t.Run(test.name, func(t *testing.T) { 958 ipt := iptablestest.NewFake() 959 ipvs := ipvstest.NewFake() 960 ipset := ipsettest.NewFake(testIPSetVersion) 961 fp := NewFakeProxier(ipt, ipvs, ipset, test.nodeIPs, nil, v1.IPv4Protocol) 962 fp.nodePortAddresses = proxyutil.NewNodePortAddresses(v1.IPv4Protocol, test.nodePortAddresses) 963 964 makeServiceMap(fp, test.services...) 965 populateEndpointSlices(fp, test.endpoints...) 966 967 fp.syncProxyRules() 968 969 if !reflect.DeepEqual(ipvs, test.expectedIPVS) { 970 t.Logf("actual ipvs state: %+v", ipvs) 971 t.Logf("expected ipvs state: %+v", test.expectedIPVS) 972 t.Errorf("unexpected IPVS state") 973 } 974 975 if test.expectedIPSets != nil { 976 checkIPSet(t, fp, test.expectedIPSets) 977 } 978 979 if test.expectedIptablesChains != nil { 980 checkIptables(t, ipt, test.expectedIptablesChains) 981 } 982 }) 983 } 984 } 985 986 func TestNodePortIPv6(t *testing.T) { 987 tests := []struct { 988 name string 989 services []*v1.Service 990 endpoints []*discovery.EndpointSlice 991 nodeIPs []string 992 nodePortAddresses []string 993 expectedIPVS *ipvstest.FakeIPVS 994 expectedIPSets netlinktest.ExpectedIPSet 995 expectedIptablesChains netlinktest.ExpectedIptablesChain 996 }{ 997 { 998 name: "1 service with node port, has 2 endpoints", 999 services: []*v1.Service{ 1000 makeTestService("ns1", "svc1", func(svc *v1.Service) { 1001 svc.Spec.Type = "NodePort" 1002 svc.Spec.ClusterIP = "2020::1" 1003 svc.Spec.Ports = []v1.ServicePort{{ 1004 Name: "p80", 1005 Port: int32(80), 1006 Protocol: v1.ProtocolTCP, 1007 NodePort: int32(3001), 1008 }} 1009 }), 1010 }, 1011 endpoints: []*discovery.EndpointSlice{ 1012 makeTestEndpointSlice("ns1", "svc1", 1, func(eps *discovery.EndpointSlice) { 1013 eps.AddressType = discovery.AddressTypeIPv4 1014 eps.Endpoints = []discovery.Endpoint{{ 1015 Addresses: []string{"10.180.0.1"}, 1016 }} 1017 eps.Ports = []discovery.EndpointPort{{ 1018 Name: ptr.To("p80"), 1019 Port: ptr.To[int32](80), 1020 Protocol: ptr.To(v1.ProtocolTCP), 1021 }} 1022 }), 1023 makeTestEndpointSlice("ns1", "svc1", 2, func(eps *discovery.EndpointSlice) { 1024 eps.AddressType = discovery.AddressTypeIPv6 1025 eps.Endpoints = []discovery.Endpoint{{ 1026 Addresses: []string{"1002:ab8::2:10"}, 1027 }} 1028 eps.Ports = []discovery.EndpointPort{{ 1029 Name: ptr.To("p80"), 1030 Port: ptr.To[int32](80), 1031 Protocol: ptr.To(v1.ProtocolTCP), 1032 }} 1033 }), 1034 }, 1035 nodeIPs: []string{"100.101.102.103", "2001:db8::1:1"}, 1036 nodePortAddresses: []string{}, 1037 expectedIPVS: &ipvstest.FakeIPVS{ 1038 Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{ 1039 { 1040 IP: "2001:db8::1:1", 1041 Port: 3001, 1042 Protocol: "TCP", 1043 }: { 1044 Address: netutils.ParseIPSloppy("2001:db8::1:1"), 1045 Protocol: "TCP", 1046 Port: uint16(3001), 1047 Scheduler: "rr", 1048 }, 1049 { 1050 IP: "2020::1", 1051 Port: 80, 1052 Protocol: "TCP", 1053 }: { 1054 Address: netutils.ParseIPSloppy("2020::1"), 1055 Protocol: "TCP", 1056 Port: uint16(80), 1057 Scheduler: "rr", 1058 }, 1059 }, 1060 Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{ 1061 { 1062 IP: "2001:db8::1:1", 1063 Port: 3001, 1064 Protocol: "TCP", 1065 }: { 1066 { 1067 Address: netutils.ParseIPSloppy("1002:ab8::2:10"), 1068 Port: uint16(80), 1069 Weight: 1, 1070 }, 1071 }, 1072 1073 { 1074 IP: "2020::1", 1075 Port: 80, 1076 Protocol: "TCP", 1077 }: { 1078 { 1079 Address: netutils.ParseIPSloppy("1002:ab8::2:10"), 1080 Port: uint16(80), 1081 Weight: 1, 1082 }, 1083 }, 1084 }, 1085 }, 1086 }, 1087 1088 { 1089 name: "1 UDP service with node port, has endpoints (no action on IPv6 Proxier)", 1090 services: []*v1.Service{ 1091 makeTestService("ns1", "svc1", func(svc *v1.Service) { 1092 svc.Spec.Type = "NodePort" 1093 svc.Spec.ClusterIP = "10.20.30.41" 1094 svc.Spec.Ports = []v1.ServicePort{{ 1095 Name: "p80", 1096 Port: int32(80), 1097 Protocol: v1.ProtocolUDP, 1098 NodePort: int32(3001), 1099 }} 1100 }), 1101 }, 1102 endpoints: []*discovery.EndpointSlice{ 1103 makeTestEndpointSlice("ns1", "svc1", 1, func(eps *discovery.EndpointSlice) { 1104 eps.AddressType = discovery.AddressTypeIPv6 1105 eps.Endpoints = []discovery.Endpoint{{ 1106 Addresses: []string{"10.180.0.1"}, 1107 }} 1108 eps.Ports = []discovery.EndpointPort{{ 1109 Name: ptr.To("p80"), 1110 Port: ptr.To[int32](80), 1111 Protocol: ptr.To(v1.ProtocolUDP), 1112 }} 1113 }), 1114 }, 1115 nodeIPs: []string{"100.101.102.103"}, 1116 nodePortAddresses: []string{"0.0.0.0/0"}, 1117 /*since this is a node with only IPv4, proxier should not do anything */ 1118 expectedIPVS: &ipvstest.FakeIPVS{ 1119 Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{}, 1120 Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{}, 1121 }, 1122 expectedIPSets: nil, 1123 expectedIptablesChains: nil, 1124 }, 1125 1126 { 1127 name: "service has node port but no endpoints", 1128 services: []*v1.Service{ 1129 makeTestService("ns1", "svc1", func(svc *v1.Service) { 1130 svc.Spec.Type = "NodePort" 1131 svc.Spec.ClusterIP = "2020::1" 1132 svc.Spec.Ports = []v1.ServicePort{{ 1133 Name: "p80", 1134 Port: int32(80), 1135 Protocol: v1.ProtocolTCP, 1136 NodePort: int32(3001), 1137 }} 1138 }), 1139 }, 1140 endpoints: []*discovery.EndpointSlice{}, 1141 nodeIPs: []string{"100.101.102.103", "2001:db8::1:1"}, 1142 nodePortAddresses: []string{}, 1143 expectedIPVS: &ipvstest.FakeIPVS{ 1144 Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{ 1145 { 1146 IP: "2001:db8::1:1", 1147 Port: 3001, 1148 Protocol: "TCP", 1149 }: { 1150 Address: netutils.ParseIPSloppy("2001:db8::1:1"), 1151 Protocol: "TCP", 1152 Port: uint16(3001), 1153 Scheduler: "rr", 1154 }, 1155 { 1156 IP: "2020::1", 1157 Port: 80, 1158 Protocol: "TCP", 1159 }: { 1160 Address: netutils.ParseIPSloppy("2020::1"), 1161 Protocol: "TCP", 1162 Port: uint16(80), 1163 Scheduler: "rr", 1164 }, 1165 }, 1166 Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{ 1167 { 1168 IP: "2020::1", 1169 Port: 80, 1170 Protocol: "TCP", 1171 }: {}, // no real servers corresponding to no endpoints 1172 { 1173 IP: "2001:db8::1:1", 1174 Port: 3001, 1175 Protocol: "TCP", 1176 }: {}, // no real servers corresponding to no endpoints 1177 }, 1178 }, 1179 }, 1180 1181 { 1182 name: "node port service with protocol sctp on a node with multiple nodeIPs", 1183 services: []*v1.Service{ 1184 makeTestService("ns1", "svc1", func(svc *v1.Service) { 1185 svc.Spec.Type = "NodePort" 1186 svc.Spec.ClusterIP = "2020::1" 1187 svc.Spec.Ports = []v1.ServicePort{{ 1188 Name: "p80", 1189 Port: int32(80), 1190 Protocol: v1.ProtocolSCTP, 1191 NodePort: int32(3001), 1192 }} 1193 }), 1194 }, 1195 endpoints: []*discovery.EndpointSlice{ 1196 makeTestEndpointSlice("ns1", "svc1", 1, func(eps *discovery.EndpointSlice) { 1197 eps.AddressType = discovery.AddressTypeIPv6 1198 eps.Endpoints = []discovery.Endpoint{{ 1199 Addresses: []string{"2001::1"}, 1200 }} 1201 eps.Ports = []discovery.EndpointPort{{ 1202 Name: ptr.To("p80"), 1203 Port: ptr.To[int32](80), 1204 Protocol: ptr.To(v1.ProtocolSCTP), 1205 }} 1206 }), 1207 }, 1208 nodeIPs: []string{"2001:db8::1:1", "2001:db8::1:2"}, 1209 nodePortAddresses: []string{}, 1210 expectedIPVS: &ipvstest.FakeIPVS{ 1211 Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{ 1212 { 1213 IP: "2001:db8::1:1", 1214 Port: 3001, 1215 Protocol: "SCTP", 1216 }: { 1217 Address: netutils.ParseIPSloppy("2001:db8::1:1"), 1218 Protocol: "SCTP", 1219 Port: uint16(3001), 1220 Scheduler: "rr", 1221 }, 1222 { 1223 IP: "2001:db8::1:2", 1224 Port: 3001, 1225 Protocol: "SCTP", 1226 }: { 1227 Address: netutils.ParseIPSloppy("2001:db8::1:2"), 1228 Protocol: "SCTP", 1229 Port: uint16(3001), 1230 Scheduler: "rr", 1231 }, 1232 { 1233 IP: "2020::1", 1234 Port: 80, 1235 Protocol: "SCTP", 1236 }: { 1237 Address: netutils.ParseIPSloppy("2020::1"), 1238 Protocol: "SCTP", 1239 Port: uint16(80), 1240 Scheduler: "rr", 1241 }, 1242 }, 1243 Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{ 1244 { 1245 IP: "2001:db8::1:1", 1246 Port: 3001, 1247 Protocol: "SCTP", 1248 }: { 1249 { 1250 Address: netutils.ParseIPSloppy("2001::1"), 1251 Port: uint16(80), 1252 Weight: 1, 1253 }, 1254 }, 1255 { 1256 IP: "2001:db8::1:2", 1257 Port: 3001, 1258 Protocol: "SCTP", 1259 }: { 1260 { 1261 Address: netutils.ParseIPSloppy("2001::1"), 1262 Port: uint16(80), 1263 Weight: 1, 1264 }, 1265 }, 1266 { 1267 IP: "2020::1", 1268 Port: 80, 1269 Protocol: "SCTP", 1270 }: { 1271 { 1272 Address: netutils.ParseIPSloppy("2001::1"), 1273 Port: uint16(80), 1274 Weight: 1, 1275 }, 1276 }, 1277 }, 1278 }, 1279 expectedIPSets: netlinktest.ExpectedIPSet{ 1280 kubeNodePortSetSCTP: { 1281 { 1282 IP: "2001:db8::1:1", 1283 Port: 3001, 1284 Protocol: strings.ToLower(string(v1.ProtocolSCTP)), 1285 SetType: utilipset.HashIPPort, 1286 }, 1287 { 1288 IP: "2001:db8::1:2", 1289 Port: 3001, 1290 Protocol: strings.ToLower(string(v1.ProtocolSCTP)), 1291 SetType: utilipset.HashIPPort, 1292 }, 1293 }, 1294 }, 1295 }, 1296 } 1297 1298 for _, test := range tests { 1299 t.Run(test.name, func(t *testing.T) { 1300 ipt := iptablestest.NewFake() 1301 ipvs := ipvstest.NewFake() 1302 ipset := ipsettest.NewFake(testIPSetVersion) 1303 fp := NewFakeProxier(ipt, ipvs, ipset, test.nodeIPs, nil, v1.IPv6Protocol) 1304 fp.nodePortAddresses = proxyutil.NewNodePortAddresses(v1.IPv6Protocol, test.nodePortAddresses) 1305 1306 makeServiceMap(fp, test.services...) 1307 populateEndpointSlices(fp, test.endpoints...) 1308 1309 fp.syncProxyRules() 1310 1311 if !reflect.DeepEqual(ipvs, test.expectedIPVS) { 1312 t.Logf("actual ipvs state: %+v", ipvs) 1313 t.Logf("expected ipvs state: %+v", test.expectedIPVS) 1314 t.Errorf("unexpected IPVS state") 1315 } 1316 1317 if test.expectedIPSets != nil { 1318 checkIPSet(t, fp, test.expectedIPSets) 1319 } 1320 1321 if test.expectedIptablesChains != nil { 1322 checkIptables(t, ipt, test.expectedIptablesChains) 1323 } 1324 }) 1325 } 1326 } 1327 1328 func Test_syncEndpoint_updateWeightsOnRestart(t *testing.T) { 1329 ipt := iptablestest.NewFake() 1330 ipvs := ipvstest.NewFake() 1331 ipset := ipsettest.NewFake(testIPSetVersion) 1332 fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol) 1333 1334 svc1 := makeTestService("ns1", "svc1", func(svc *v1.Service) { 1335 svc.Spec.ClusterIP = "10.20.30.41" 1336 svc.Spec.Ports = []v1.ServicePort{{ 1337 Name: "p80", 1338 Port: int32(80), 1339 Protocol: v1.ProtocolTCP, 1340 }} 1341 }) 1342 epSlice1 := makeTestEndpointSlice("ns1", "svc1", 1, func(eps *discovery.EndpointSlice) { 1343 eps.AddressType = discovery.AddressTypeIPv4 1344 eps.Endpoints = []discovery.Endpoint{{ 1345 Addresses: []string{"10.180.0.1"}, 1346 }} 1347 eps.Ports = []discovery.EndpointPort{{ 1348 Name: ptr.To("p80"), 1349 Port: ptr.To[int32](80), 1350 Protocol: ptr.To(v1.ProtocolTCP), 1351 }} 1352 }) 1353 1354 // sync proxy rules to get to the desired initial state 1355 makeServiceMap(fp, svc1) 1356 makeEndpointSliceMap(fp, epSlice1) 1357 fp.syncProxyRules() 1358 1359 serv := &utilipvs.VirtualServer{ 1360 Address: netutils.ParseIPSloppy("10.20.30.41"), 1361 Port: uint16(80), 1362 Protocol: string(v1.ProtocolTCP), 1363 Scheduler: fp.ipvsScheduler, 1364 } 1365 1366 vs, err := fp.ipvs.GetVirtualServer(serv) 1367 if err != nil { 1368 t.Errorf("failed to get virtual server, err: %v", err) 1369 } 1370 1371 rss, err := fp.ipvs.GetRealServers(vs) 1372 if err != nil { 1373 t.Errorf("failed to get real servers, err: %v", err) 1374 } 1375 for _, rs := range rss { 1376 rs.Weight = 0 1377 if err = fp.ipvs.UpdateRealServer(vs, rs); err != nil { 1378 t.Errorf("failed to update real server: %v, err: %v", rs, err) 1379 } 1380 } 1381 1382 // simulate a restart by enabling initial sync logic. 1383 fp.initialSync = true 1384 err = fp.syncEndpoint(proxy.ServicePortName{ 1385 NamespacedName: types.NamespacedName{ 1386 Name: "svc1", 1387 Namespace: "ns1", 1388 }, 1389 Port: "80", 1390 Protocol: v1.ProtocolTCP, 1391 }, true, vs) 1392 if err != nil { 1393 t.Errorf("failed to sync endpoint, err: %v", err) 1394 } 1395 1396 rss, err = fp.ipvs.GetRealServers(vs) 1397 if err != nil { 1398 t.Errorf("failed to get real server, err: %v", err) 1399 } 1400 for _, rs := range rss { 1401 if rs.Weight != 1 { 1402 t.Logf("unexpected realserver weight: %d, expected weight: 1", rs.Weight) 1403 t.Errorf("unexpected realserver state") 1404 } 1405 } 1406 } 1407 1408 func TestIPv4Proxier(t *testing.T) { 1409 tests := []struct { 1410 name string 1411 services []*v1.Service 1412 endpoints []*discovery.EndpointSlice 1413 expectedIPVS *ipvstest.FakeIPVS 1414 }{ 1415 { 1416 name: "2 services with Cluster IP, each with endpoints", 1417 services: []*v1.Service{ 1418 makeTestService("ns1", "svc1", func(svc *v1.Service) { 1419 svc.Spec.ClusterIP = "10.20.30.41" 1420 svc.Spec.Ports = []v1.ServicePort{{ 1421 Name: "p80", 1422 Port: int32(80), 1423 Protocol: v1.ProtocolTCP, 1424 }} 1425 }), 1426 makeTestService("ns2", "svc2", func(svc *v1.Service) { 1427 svc.Spec.ClusterIP = "1002:ab8::2:1" 1428 svc.Spec.Ports = []v1.ServicePort{{ 1429 Name: "p8080", 1430 Port: int32(8080), 1431 Protocol: v1.ProtocolTCP, 1432 }} 1433 }), 1434 }, 1435 endpoints: []*discovery.EndpointSlice{ 1436 makeTestEndpointSlice("ns1", "svc1", 1, func(eps *discovery.EndpointSlice) { 1437 eps.AddressType = discovery.AddressTypeIPv4 1438 eps.Endpoints = []discovery.Endpoint{{ 1439 Addresses: []string{"10.180.0.1"}, 1440 }} 1441 eps.Ports = []discovery.EndpointPort{{ 1442 Name: ptr.To("p80"), 1443 Port: ptr.To[int32](80), 1444 Protocol: ptr.To(v1.ProtocolTCP), 1445 }} 1446 }), 1447 makeTestEndpointSlice("ns2", "svc2", 1, func(eps *discovery.EndpointSlice) { 1448 eps.AddressType = discovery.AddressTypeIPv6 1449 eps.Endpoints = []discovery.Endpoint{{ 1450 Addresses: []string{"1009:ab8::5:6"}, 1451 }} 1452 eps.Ports = []discovery.EndpointPort{{ 1453 Name: ptr.To("p8080"), 1454 Port: ptr.To[int32](8080), 1455 Protocol: ptr.To(v1.ProtocolTCP), 1456 }} 1457 }), 1458 }, 1459 expectedIPVS: &ipvstest.FakeIPVS{ 1460 Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{ 1461 { 1462 IP: "10.20.30.41", 1463 Port: 80, 1464 Protocol: "TCP", 1465 }: { 1466 Address: netutils.ParseIPSloppy("10.20.30.41"), 1467 Protocol: "TCP", 1468 Port: uint16(80), 1469 Scheduler: "rr", 1470 }, 1471 }, 1472 Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{ 1473 { 1474 IP: "10.20.30.41", 1475 Port: 80, 1476 Protocol: "TCP", 1477 }: { 1478 { 1479 Address: netutils.ParseIPSloppy("10.180.0.1"), 1480 Port: uint16(80), 1481 Weight: 1, 1482 }, 1483 }, 1484 }, 1485 }, 1486 }, 1487 { 1488 name: "cluster IP service with no endpoints", 1489 services: []*v1.Service{ 1490 makeTestService("ns1", "svc1", func(svc *v1.Service) { 1491 svc.Spec.ClusterIP = "10.20.30.41" 1492 svc.Spec.Ports = []v1.ServicePort{{ 1493 Name: "p80", 1494 Port: int32(80), 1495 Protocol: v1.ProtocolTCP, 1496 }} 1497 }), 1498 }, 1499 endpoints: []*discovery.EndpointSlice{}, 1500 expectedIPVS: &ipvstest.FakeIPVS{ 1501 Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{ 1502 { 1503 IP: "10.20.30.41", 1504 Port: 80, 1505 Protocol: "TCP", 1506 }: { 1507 Address: netutils.ParseIPSloppy("10.20.30.41"), 1508 Protocol: "TCP", 1509 Port: uint16(80), 1510 Scheduler: "rr", 1511 }, 1512 }, 1513 Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{ 1514 { 1515 IP: "10.20.30.41", 1516 Port: 80, 1517 Protocol: "TCP", 1518 }: {}, 1519 }, 1520 }, 1521 }, 1522 } 1523 1524 for _, test := range tests { 1525 t.Run(test.name, func(t *testing.T) { 1526 ipt := iptablestest.NewFake() 1527 ipvs := ipvstest.NewFake() 1528 ipset := ipsettest.NewFake(testIPSetVersion) 1529 fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol) 1530 1531 makeServiceMap(fp, test.services...) 1532 populateEndpointSlices(fp, test.endpoints...) 1533 1534 fp.syncProxyRules() 1535 1536 if !reflect.DeepEqual(ipvs, test.expectedIPVS) { 1537 t.Logf("actual ipvs state: %v", ipvs) 1538 t.Logf("expected ipvs state: %v", test.expectedIPVS) 1539 t.Errorf("unexpected IPVS state") 1540 } 1541 }) 1542 } 1543 } 1544 1545 func TestIPv6Proxier(t *testing.T) { 1546 tests := []struct { 1547 name string 1548 services []*v1.Service 1549 endpoints []*discovery.EndpointSlice 1550 expectedIPVS *ipvstest.FakeIPVS 1551 }{ 1552 { 1553 name: "2 services with Cluster IP, each with endpoints", 1554 services: []*v1.Service{ 1555 makeTestService("ns1", "svc1", func(svc *v1.Service) { 1556 svc.Spec.ClusterIP = "10.20.30.41" 1557 svc.Spec.Ports = []v1.ServicePort{{ 1558 Name: "p80", 1559 Port: int32(80), 1560 Protocol: v1.ProtocolTCP, 1561 }} 1562 }), 1563 makeTestService("ns2", "svc2", func(svc *v1.Service) { 1564 svc.Spec.ClusterIP = "1002:ab8::2:1" 1565 svc.Spec.Ports = []v1.ServicePort{{ 1566 Name: "p8080", 1567 Port: int32(8080), 1568 Protocol: v1.ProtocolTCP, 1569 }} 1570 }), 1571 }, 1572 endpoints: []*discovery.EndpointSlice{ 1573 makeTestEndpointSlice("ns1", "svc1", 1, func(eps *discovery.EndpointSlice) { 1574 eps.AddressType = discovery.AddressTypeIPv4 1575 eps.Endpoints = []discovery.Endpoint{{ 1576 Addresses: []string{"10.180.0.1"}, 1577 }} 1578 eps.Ports = []discovery.EndpointPort{{ 1579 Name: ptr.To("p80"), 1580 Port: ptr.To[int32](80), 1581 Protocol: ptr.To(v1.ProtocolTCP), 1582 }} 1583 }), 1584 makeTestEndpointSlice("ns2", "svc2", 1, func(eps *discovery.EndpointSlice) { 1585 eps.AddressType = discovery.AddressTypeIPv6 1586 eps.Endpoints = []discovery.Endpoint{{ 1587 Addresses: []string{"1009:ab8::5:6"}, 1588 }} 1589 eps.Ports = []discovery.EndpointPort{{ 1590 Name: ptr.To("p8080"), 1591 Port: ptr.To[int32](8080), 1592 Protocol: ptr.To(v1.ProtocolTCP), 1593 }} 1594 }), 1595 }, 1596 expectedIPVS: &ipvstest.FakeIPVS{ 1597 Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{ 1598 { 1599 IP: "1002:ab8::2:1", 1600 Port: 8080, 1601 Protocol: "TCP", 1602 }: { 1603 Address: netutils.ParseIPSloppy("1002:ab8::2:1"), 1604 Protocol: "TCP", 1605 Port: uint16(8080), 1606 Scheduler: "rr", 1607 }, 1608 }, 1609 Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{ 1610 { 1611 IP: "1002:ab8::2:1", 1612 Port: 8080, 1613 Protocol: "TCP", 1614 }: { 1615 { 1616 Address: netutils.ParseIPSloppy("1009:ab8::5:6"), 1617 Port: uint16(8080), 1618 Weight: 1, 1619 }, 1620 }, 1621 }, 1622 }, 1623 }, 1624 { 1625 name: "cluster IP service with no endpoints", 1626 services: []*v1.Service{ 1627 makeTestService("ns1", "svc1", func(svc *v1.Service) { 1628 svc.Spec.ClusterIP = "2001::1" 1629 svc.Spec.Ports = []v1.ServicePort{{ 1630 Name: "p80", 1631 Port: int32(80), 1632 Protocol: v1.ProtocolTCP, 1633 }} 1634 }), 1635 }, 1636 endpoints: []*discovery.EndpointSlice{}, 1637 expectedIPVS: &ipvstest.FakeIPVS{ 1638 Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{ 1639 { 1640 IP: "2001::1", 1641 Port: 80, 1642 Protocol: "TCP", 1643 }: { 1644 Address: netutils.ParseIPSloppy("2001::1"), 1645 Protocol: "TCP", 1646 Port: uint16(80), 1647 Scheduler: "rr", 1648 }, 1649 }, 1650 Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{ 1651 { 1652 IP: "2001::1", 1653 Port: 80, 1654 Protocol: "TCP", 1655 }: {}, 1656 }, 1657 }, 1658 }, 1659 } 1660 1661 for _, test := range tests { 1662 t.Run(test.name, func(t *testing.T) { 1663 ipt := iptablestest.NewFake() 1664 ipvs := ipvstest.NewFake() 1665 ipset := ipsettest.NewFake(testIPSetVersion) 1666 fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv6Protocol) 1667 1668 makeServiceMap(fp, test.services...) 1669 populateEndpointSlices(fp, test.endpoints...) 1670 1671 fp.syncProxyRules() 1672 1673 if !reflect.DeepEqual(ipvs, test.expectedIPVS) { 1674 t.Logf("actual ipvs state: %v", ipvs) 1675 t.Logf("expected ipvs state: %v", test.expectedIPVS) 1676 t.Errorf("unexpected IPVS state") 1677 } 1678 }) 1679 } 1680 } 1681 1682 func TestMasqueradeRule(t *testing.T) { 1683 for _, testcase := range []bool{false, true} { 1684 ipt := iptablestest.NewFake().SetHasRandomFully(testcase) 1685 ipvs := ipvstest.NewFake() 1686 ipset := ipsettest.NewFake(testIPSetVersion) 1687 fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol) 1688 makeServiceMap(fp) 1689 fp.syncProxyRules() 1690 1691 buf := bytes.NewBuffer(nil) 1692 _ = ipt.SaveInto(utiliptables.TableNAT, buf) 1693 natRules := strings.Split(buf.String(), "\n") 1694 var hasMasqueradeJump, hasMasqRandomFully bool 1695 for _, line := range natRules { 1696 rule, _ := iptablestest.ParseRule(line, false) 1697 if rule != nil && rule.Chain == kubePostroutingChain && rule.Jump != nil && rule.Jump.Value == "MASQUERADE" { 1698 hasMasqueradeJump = true 1699 if rule.RandomFully != nil { 1700 hasMasqRandomFully = true 1701 } 1702 break 1703 } 1704 } 1705 1706 if !hasMasqueradeJump { 1707 t.Errorf("Failed to find -j MASQUERADE in %s chain", kubePostroutingChain) 1708 } 1709 if hasMasqRandomFully != testcase { 1710 probs := map[bool]string{false: "found", true: "did not find"} 1711 t.Errorf("%s --random-fully in -j MASQUERADE rule in %s chain for HasRandomFully()=%v", probs[testcase], kubePostroutingChain, testcase) 1712 } 1713 } 1714 } 1715 1716 func TestExternalIPsNoEndpoint(t *testing.T) { 1717 ipt := iptablestest.NewFake() 1718 ipvs := ipvstest.NewFake() 1719 ipset := ipsettest.NewFake(testIPSetVersion) 1720 fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol) 1721 svcIP := "10.20.30.41" 1722 svcPort := 80 1723 svcExternalIPs := "50.60.70.81" 1724 svcPortName := proxy.ServicePortName{ 1725 NamespacedName: makeNSN("ns1", "svc1"), 1726 Port: "p80", 1727 } 1728 1729 makeServiceMap(fp, 1730 makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { 1731 svc.Spec.Type = "ClusterIP" 1732 svc.Spec.ClusterIP = svcIP 1733 svc.Spec.ExternalIPs = []string{svcExternalIPs} 1734 svc.Spec.Ports = []v1.ServicePort{{ 1735 Name: svcPortName.Port, 1736 Port: int32(svcPort), 1737 Protocol: v1.ProtocolTCP, 1738 TargetPort: intstr.FromInt32(int32(svcPort)), 1739 }} 1740 }), 1741 ) 1742 fp.syncProxyRules() 1743 1744 // check ipvs service and destinations 1745 services, err := ipvs.GetVirtualServers() 1746 if err != nil { 1747 t.Errorf("Failed to get ipvs services, err: %v", err) 1748 } 1749 if len(services) != 2 { 1750 t.Errorf("Expect 2 ipvs services, got %d", len(services)) 1751 } 1752 found := false 1753 for _, svc := range services { 1754 if svc.Address.String() == svcExternalIPs && svc.Port == uint16(svcPort) && svc.Protocol == string(v1.ProtocolTCP) { 1755 found = true 1756 destinations, _ := ipvs.GetRealServers(svc) 1757 if len(destinations) != 0 { 1758 t.Errorf("Unexpected %d destinations, expect 0 destinations", len(destinations)) 1759 } 1760 break 1761 } 1762 } 1763 if !found { 1764 t.Errorf("Expect external ip type service, got none") 1765 } 1766 } 1767 1768 func TestExternalIPs(t *testing.T) { 1769 ipt := iptablestest.NewFake() 1770 ipvs := ipvstest.NewFake() 1771 ipset := ipsettest.NewFake(testIPSetVersion) 1772 fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol) 1773 svcIP := "10.20.30.41" 1774 svcPort := 80 1775 svcExternalIPs := sets.New[string]("50.60.70.81", "2012::51", "127.0.0.1") 1776 svcPortName := proxy.ServicePortName{ 1777 NamespacedName: makeNSN("ns1", "svc1"), 1778 Port: "p80", 1779 } 1780 1781 makeServiceMap(fp, 1782 makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { 1783 svc.Spec.Type = "ClusterIP" 1784 svc.Spec.ClusterIP = svcIP 1785 svc.Spec.ExternalIPs = svcExternalIPs.UnsortedList() 1786 svc.Spec.Ports = []v1.ServicePort{{ 1787 Name: svcPortName.Port, 1788 Port: int32(svcPort), 1789 Protocol: v1.ProtocolTCP, 1790 TargetPort: intstr.FromInt32(int32(svcPort)), 1791 }} 1792 }), 1793 ) 1794 1795 epIP := "10.180.0.1" 1796 populateEndpointSlices(fp, 1797 makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) { 1798 eps.AddressType = discovery.AddressTypeIPv4 1799 eps.Endpoints = []discovery.Endpoint{{ 1800 Addresses: []string{epIP}, 1801 }} 1802 eps.Ports = []discovery.EndpointPort{{ 1803 Name: ptr.To(svcPortName.Port), 1804 Port: ptr.To(int32(svcPort)), 1805 Protocol: ptr.To(v1.ProtocolUDP), 1806 }} 1807 }), 1808 ) 1809 1810 fp.syncProxyRules() 1811 1812 // check ipvs service and destinations 1813 services, err := ipvs.GetVirtualServers() 1814 if err != nil { 1815 t.Errorf("Failed to get ipvs services, err: %v", err) 1816 } 1817 if len(services) != 3 { // ipvs filters out by ipfamily 1818 t.Errorf("Expect 3 ipvs services, got %d", len(services)) 1819 } 1820 found := false 1821 for _, svc := range services { 1822 if svcExternalIPs.Has(svc.Address.String()) && svc.Port == uint16(svcPort) && svc.Protocol == string(v1.ProtocolTCP) { 1823 found = true 1824 destinations, _ := ipvs.GetRealServers(svc) 1825 for _, dest := range destinations { 1826 if dest.Address.String() != epIP || dest.Port != uint16(svcPort) { 1827 t.Errorf("service Endpoint mismatch ipvs service destination") 1828 } 1829 } 1830 break 1831 } 1832 } 1833 if !found { 1834 t.Errorf("Expect external ip type service, got none") 1835 } 1836 } 1837 1838 func TestOnlyLocalExternalIPs(t *testing.T) { 1839 ipt := iptablestest.NewFake() 1840 ipvs := ipvstest.NewFake() 1841 ipset := ipsettest.NewFake(testIPSetVersion) 1842 fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol) 1843 svcIP := "10.20.30.41" 1844 svcPort := 80 1845 svcExternalIPs := sets.New[string]("50.60.70.81", "2012::51", "127.0.0.1") 1846 svcPortName := proxy.ServicePortName{ 1847 NamespacedName: makeNSN("ns1", "svc1"), 1848 Port: "p80", 1849 } 1850 1851 makeServiceMap(fp, 1852 makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { 1853 svc.Spec.Type = "NodePort" 1854 svc.Spec.ClusterIP = svcIP 1855 svc.Spec.ExternalIPs = svcExternalIPs.UnsortedList() 1856 svc.Spec.Ports = []v1.ServicePort{{ 1857 Name: svcPortName.Port, 1858 Port: int32(svcPort), 1859 Protocol: v1.ProtocolTCP, 1860 TargetPort: intstr.FromInt32(int32(svcPort)), 1861 }} 1862 svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyLocal 1863 }), 1864 ) 1865 epIP := "10.180.0.1" 1866 epIP1 := "10.180.1.1" 1867 thisHostname := testHostname 1868 otherHostname := "other-hostname" 1869 populateEndpointSlices(fp, 1870 makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) { 1871 eps.AddressType = discovery.AddressTypeIPv4 1872 eps.Endpoints = []discovery.Endpoint{{ 1873 Addresses: []string{epIP}, 1874 NodeName: ptr.To(thisHostname), 1875 }, 1876 { 1877 Addresses: []string{epIP1}, 1878 NodeName: ptr.To(otherHostname), 1879 }} 1880 eps.Ports = []discovery.EndpointPort{{ 1881 Name: ptr.To(svcPortName.Port), 1882 Port: ptr.To(int32(svcPort)), 1883 Protocol: ptr.To(v1.ProtocolTCP), 1884 }} 1885 }), 1886 ) 1887 1888 fp.syncProxyRules() 1889 1890 // check ipvs service and destinations 1891 services, err := ipvs.GetVirtualServers() 1892 if err != nil { 1893 t.Errorf("Failed to get ipvs services, err: %v", err) 1894 } 1895 if len(services) != 3 { // ipvs filters out by IPFamily 1896 t.Errorf("Expect 3 ipvs services, got %d", len(services)) 1897 } 1898 found := false 1899 for _, svc := range services { 1900 if svcExternalIPs.Has(svc.Address.String()) && svc.Port == uint16(svcPort) && svc.Protocol == string(v1.ProtocolTCP) { 1901 found = true 1902 destinations, _ := ipvs.GetRealServers(svc) 1903 if len(destinations) != 1 { 1904 t.Errorf("Expect only 1 local endpoint. but got %v", len(destinations)) 1905 } 1906 for _, dest := range destinations { 1907 if dest.Address.String() != epIP || dest.Port != uint16(svcPort) { 1908 t.Errorf("service Endpoint mismatch ipvs service destination") 1909 } 1910 } 1911 break 1912 } 1913 } 1914 if !found { 1915 t.Errorf("Expect external ip type service, got none") 1916 } 1917 } 1918 1919 func TestLoadBalancer(t *testing.T) { 1920 ipt, fp := buildFakeProxier() 1921 svcIP := "10.20.30.41" 1922 svcPort := 80 1923 svcNodePort := 3001 1924 svcLBIP := "1.2.3.4" 1925 svcPortName := proxy.ServicePortName{ 1926 NamespacedName: makeNSN("ns1", "svc1"), 1927 Port: "p80", 1928 } 1929 1930 makeServiceMap(fp, 1931 makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { 1932 svc.Spec.Type = "LoadBalancer" 1933 svc.Spec.ClusterIP = svcIP 1934 svc.Spec.Ports = []v1.ServicePort{{ 1935 Name: svcPortName.Port, 1936 Port: int32(svcPort), 1937 Protocol: v1.ProtocolTCP, 1938 NodePort: int32(svcNodePort), 1939 }} 1940 svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{ 1941 IP: svcLBIP, 1942 }} 1943 }), 1944 ) 1945 1946 epIP := "10.180.0.1" 1947 populateEndpointSlices(fp, 1948 makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) { 1949 eps.AddressType = discovery.AddressTypeIPv4 1950 eps.Endpoints = []discovery.Endpoint{{ 1951 Addresses: []string{epIP}, 1952 }} 1953 eps.Ports = []discovery.EndpointPort{{ 1954 Name: ptr.To(svcPortName.Port), 1955 Port: ptr.To(int32(svcPort)), 1956 Protocol: ptr.To(v1.ProtocolUDP), 1957 }} 1958 }), 1959 ) 1960 1961 fp.syncProxyRules() 1962 1963 // Expect 2 services and 1 destination 1964 epVS := &netlinktest.ExpectedVirtualServer{ 1965 VSNum: 2, IP: svcLBIP, Port: uint16(svcNodePort), Protocol: string(v1.ProtocolTCP), 1966 RS: []netlinktest.ExpectedRealServer{{ 1967 IP: epIP, Port: uint16(svcPort), 1968 }}} 1969 checkIPVS(t, fp, epVS) 1970 1971 // check ipSet rules 1972 epIPSet := netlinktest.ExpectedIPSet{ 1973 kubeLoadBalancerSet: {{ 1974 IP: svcLBIP, 1975 Port: svcPort, 1976 Protocol: strings.ToLower(string(v1.ProtocolTCP)), 1977 SetType: utilipset.HashIPPort, 1978 }}, 1979 } 1980 checkIPSet(t, fp, epIPSet) 1981 1982 // Check iptables chain and rules 1983 epIpt := netlinktest.ExpectedIptablesChain{ 1984 string(kubeServicesChain): {{ 1985 JumpChain: "RETURN", SourceAddress: "127.0.0.0/8", 1986 }, { 1987 JumpChain: string(kubeLoadBalancerChain), MatchSet: kubeLoadBalancerSet, 1988 }, { 1989 JumpChain: string(kubeMarkMasqChain), MatchSet: kubeClusterIPSet, 1990 }, { 1991 JumpChain: string(kubeNodePortChain), MatchSet: "", 1992 }, { 1993 JumpChain: "ACCEPT", MatchSet: kubeClusterIPSet, 1994 }, { 1995 JumpChain: "ACCEPT", MatchSet: kubeLoadBalancerSet, 1996 }}, 1997 string(kubeLoadBalancerSet): {{ 1998 JumpChain: string(kubeMarkMasqChain), MatchSet: "", 1999 }}, 2000 } 2001 checkIptables(t, ipt, epIpt) 2002 } 2003 2004 func TestOnlyLocalNodePorts(t *testing.T) { 2005 nodeIP := netutils.ParseIPSloppy("100.101.102.103") 2006 ipt, fp := buildFakeProxier() 2007 2008 svcIP := "10.20.30.41" 2009 svcPort := 80 2010 svcNodePort := 3001 2011 svcPortName := proxy.ServicePortName{ 2012 NamespacedName: makeNSN("ns1", "svc1"), 2013 Port: "p80", 2014 } 2015 2016 makeServiceMap(fp, 2017 makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { 2018 svc.Spec.Type = "NodePort" 2019 svc.Spec.ClusterIP = svcIP 2020 svc.Spec.Ports = []v1.ServicePort{{ 2021 Name: svcPortName.Port, 2022 Port: int32(svcPort), 2023 Protocol: v1.ProtocolTCP, 2024 NodePort: int32(svcNodePort), 2025 }} 2026 svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyLocal 2027 }), 2028 ) 2029 2030 epIP := "10.180.0.1" 2031 epIP1 := "10.180.1.1" 2032 2033 populateEndpointSlices(fp, 2034 makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) { 2035 eps.AddressType = discovery.AddressTypeIPv4 2036 eps.Endpoints = []discovery.Endpoint{{ 2037 Addresses: []string{epIP}, 2038 NodeName: ptr.To(testHostname), 2039 }, { 2040 Addresses: []string{epIP1}, 2041 NodeName: ptr.To("other-hostname"), 2042 }} 2043 eps.Ports = []discovery.EndpointPort{{ 2044 Name: ptr.To(svcPortName.Port), 2045 Port: ptr.To(int32(svcPort)), 2046 Protocol: ptr.To(v1.ProtocolTCP), 2047 }} 2048 }), 2049 ) 2050 2051 itf := net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0} 2052 addrs := []net.Addr{&net.IPNet{IP: netutils.ParseIPSloppy("100.101.102.103"), Mask: net.CIDRMask(24, 32)}} 2053 itf1 := net.Interface{Index: 1, MTU: 0, Name: "eth1", HardwareAddr: nil, Flags: 0} 2054 addrs1 := []net.Addr{&net.IPNet{IP: netutils.ParseIPSloppy("2001:db8::"), Mask: net.CIDRMask(64, 128)}} 2055 fp.networkInterfacer.(*proxyutiltest.FakeNetwork).AddInterfaceAddr(&itf, addrs) 2056 fp.networkInterfacer.(*proxyutiltest.FakeNetwork).AddInterfaceAddr(&itf1, addrs1) 2057 fp.nodePortAddresses = proxyutil.NewNodePortAddresses(v1.IPv4Protocol, []string{"100.101.102.0/24"}) 2058 2059 fp.syncProxyRules() 2060 2061 // Expect 2 services and 1 destination 2062 epVS := &netlinktest.ExpectedVirtualServer{ 2063 VSNum: 2, IP: nodeIP.String(), Port: uint16(svcNodePort), Protocol: string(v1.ProtocolTCP), 2064 RS: []netlinktest.ExpectedRealServer{{ 2065 IP: epIP, Port: uint16(svcPort), 2066 }}} 2067 checkIPVS(t, fp, epVS) 2068 2069 // check ipSet rules 2070 epEntry := &utilipset.Entry{ 2071 Port: svcNodePort, 2072 Protocol: strings.ToLower(string(v1.ProtocolTCP)), 2073 SetType: utilipset.BitmapPort, 2074 } 2075 epIPSet := netlinktest.ExpectedIPSet{ 2076 kubeNodePortSetTCP: {epEntry}, 2077 kubeNodePortLocalSetTCP: {epEntry}, 2078 } 2079 checkIPSet(t, fp, epIPSet) 2080 2081 // Check iptables chain and rules 2082 epIpt := netlinktest.ExpectedIptablesChain{ 2083 string(kubeServicesChain): {{ 2084 JumpChain: "RETURN", SourceAddress: "127.0.0.0/8", 2085 }, { 2086 JumpChain: string(kubeMarkMasqChain), MatchSet: kubeClusterIPSet, 2087 }, { 2088 JumpChain: string(kubeNodePortChain), MatchSet: "", 2089 }, { 2090 JumpChain: "ACCEPT", MatchSet: kubeClusterIPSet, 2091 }}, 2092 string(kubeNodePortChain): {{ 2093 JumpChain: "RETURN", MatchSet: kubeNodePortLocalSetTCP, 2094 }, { 2095 JumpChain: string(kubeMarkMasqChain), MatchSet: kubeNodePortSetTCP, 2096 }, { 2097 JumpChain: "ACCEPT", MatchSet: kubeHealthCheckNodePortSet, 2098 }}, 2099 } 2100 checkIptables(t, ipt, epIpt) 2101 } 2102 2103 func TestHealthCheckNodePort(t *testing.T) { 2104 ipt, fp := buildFakeProxier() 2105 2106 svcIP := "10.20.30.41" 2107 svcPort := 80 2108 svcNodePort := 3000 2109 svcPortName := proxy.ServicePortName{ 2110 NamespacedName: makeNSN("ns1", "svc1"), 2111 Port: "p80", 2112 } 2113 2114 sampleSvc := makeTestService(svcPortName.Namespace, "", func(svc *v1.Service) { 2115 svc.Spec.Type = "LoadBalancer" 2116 svc.Spec.ClusterIP = svcIP 2117 svc.Spec.Ports = []v1.ServicePort{{ 2118 Name: svcPortName.Port, 2119 Port: int32(svcPort), 2120 Protocol: v1.ProtocolTCP, 2121 NodePort: int32(svcNodePort), 2122 }} 2123 svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyLocal 2124 }) 2125 2126 svc1, svc2, invalidSvc3 := *sampleSvc, *sampleSvc, *sampleSvc 2127 svc1.Name, svc1.Spec.HealthCheckNodePort = "valid-svc1", 30000 2128 svc2.Name, svc2.Spec.HealthCheckNodePort = "valid-svc2", 30001 2129 // make svc3 invalid by setting external traffic policy to cluster 2130 invalidSvc3.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyCluster 2131 invalidSvc3.Name, invalidSvc3.Spec.HealthCheckNodePort = "invalid-svc3", 30002 2132 2133 makeServiceMap(fp, 2134 &svc1, 2135 &svc2, 2136 &invalidSvc3, 2137 ) 2138 2139 itf := net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0} 2140 addrs := []net.Addr{&net.IPNet{IP: netutils.ParseIPSloppy("100.101.102.103"), Mask: net.CIDRMask(24, 32)}} 2141 itf1 := net.Interface{Index: 1, MTU: 0, Name: "eth1", HardwareAddr: nil, Flags: 0} 2142 addrs1 := []net.Addr{&net.IPNet{IP: netutils.ParseIPSloppy("2001:db8::"), Mask: net.CIDRMask(64, 128)}} 2143 fp.networkInterfacer.(*proxyutiltest.FakeNetwork).AddInterfaceAddr(&itf, addrs) 2144 fp.networkInterfacer.(*proxyutiltest.FakeNetwork).AddInterfaceAddr(&itf1, addrs1) 2145 fp.nodePortAddresses = proxyutil.NewNodePortAddresses(v1.IPv4Protocol, []string{"100.101.102.0/24"}) 2146 2147 fp.syncProxyRules() 2148 2149 // check ipSet rules 2150 makeTCPEntry := func(port int) *utilipset.Entry { 2151 return &utilipset.Entry{ 2152 Port: port, 2153 Protocol: strings.ToLower(string(v1.ProtocolTCP)), 2154 SetType: utilipset.BitmapPort, 2155 } 2156 } 2157 epIPSet := netlinktest.ExpectedIPSet{ 2158 // healthcheck node port set should only contain valid HC node ports 2159 kubeHealthCheckNodePortSet: {makeTCPEntry(30000), makeTCPEntry(30001)}, 2160 } 2161 checkIPSet(t, fp, epIPSet) 2162 2163 // Check iptables chain and rules 2164 epIpt := netlinktest.ExpectedIptablesChain{ 2165 string(kubeNodePortChain): {{ 2166 JumpChain: "RETURN", MatchSet: kubeNodePortLocalSetTCP, 2167 }, { 2168 JumpChain: string(kubeMarkMasqChain), MatchSet: kubeNodePortSetTCP, 2169 }, { 2170 JumpChain: "ACCEPT", MatchSet: kubeHealthCheckNodePortSet, 2171 }}, 2172 } 2173 checkIptables(t, ipt, epIpt) 2174 } 2175 2176 func TestLoadBalancerSourceRanges(t *testing.T) { 2177 ipt, fp := buildFakeProxier() 2178 2179 svcIP := "10.20.30.41" 2180 svcPort := 80 2181 svcLBIP := "1.2.3.4" 2182 svcLBSource := "10.0.0.0/8" 2183 svcPortName := proxy.ServicePortName{ 2184 NamespacedName: makeNSN("ns1", "svc1"), 2185 Port: "p80", 2186 } 2187 epIP := "10.180.0.1" 2188 2189 makeServiceMap(fp, 2190 makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { 2191 svc.Spec.Type = "LoadBalancer" 2192 svc.Spec.ClusterIP = svcIP 2193 svc.Spec.Ports = []v1.ServicePort{{ 2194 Name: svcPortName.Port, 2195 Port: int32(svcPort), 2196 Protocol: v1.ProtocolTCP, 2197 }} 2198 svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{ 2199 IP: svcLBIP, 2200 }} 2201 svc.Spec.LoadBalancerSourceRanges = []string{ 2202 svcLBSource, 2203 } 2204 }), 2205 ) 2206 populateEndpointSlices(fp, 2207 makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) { 2208 eps.AddressType = discovery.AddressTypeIPv4 2209 eps.Endpoints = []discovery.Endpoint{{ 2210 Addresses: []string{epIP}, 2211 }} 2212 eps.Ports = []discovery.EndpointPort{{ 2213 Name: ptr.To(svcPortName.Port), 2214 Port: ptr.To(int32(svcPort)), 2215 Protocol: ptr.To(v1.ProtocolTCP), 2216 }} 2217 }), 2218 ) 2219 2220 fp.syncProxyRules() 2221 2222 // Check ipvs service and destinations 2223 epVS := &netlinktest.ExpectedVirtualServer{ 2224 VSNum: 2, IP: svcLBIP, Port: uint16(svcPort), Protocol: string(v1.ProtocolTCP), 2225 RS: []netlinktest.ExpectedRealServer{{ 2226 IP: epIP, Port: uint16(svcPort), 2227 }}} 2228 checkIPVS(t, fp, epVS) 2229 2230 // Check ipset entry 2231 epIPSet := netlinktest.ExpectedIPSet{ 2232 kubeLoadBalancerSet: {{ 2233 IP: svcLBIP, 2234 Port: svcPort, 2235 Protocol: strings.ToLower(string(v1.ProtocolTCP)), 2236 SetType: utilipset.HashIPPort, 2237 }}, 2238 kubeLoadBalancerFWSet: {{ 2239 IP: svcLBIP, 2240 Port: svcPort, 2241 Protocol: strings.ToLower(string(v1.ProtocolTCP)), 2242 SetType: utilipset.HashIPPort, 2243 }}, 2244 kubeLoadBalancerSourceCIDRSet: {{ 2245 IP: svcLBIP, 2246 Port: svcPort, 2247 Protocol: strings.ToLower(string(v1.ProtocolTCP)), 2248 Net: svcLBSource, 2249 SetType: utilipset.HashIPPortNet, 2250 }}, 2251 } 2252 checkIPSet(t, fp, epIPSet) 2253 2254 // Check iptables chain and rules 2255 epIpt := netlinktest.ExpectedIptablesChain{ 2256 string(kubeServicesChain): {{ 2257 JumpChain: "RETURN", SourceAddress: "127.0.0.0/8", 2258 }, { 2259 JumpChain: string(kubeLoadBalancerChain), MatchSet: kubeLoadBalancerSet, 2260 }, { 2261 JumpChain: string(kubeMarkMasqChain), MatchSet: kubeClusterIPSet, 2262 }, { 2263 JumpChain: string(kubeNodePortChain), MatchSet: "", 2264 }, { 2265 JumpChain: "ACCEPT", MatchSet: kubeClusterIPSet, 2266 }, { 2267 JumpChain: "ACCEPT", MatchSet: kubeLoadBalancerSet, 2268 }}, 2269 string(kubeProxyFirewallChain): {{ 2270 JumpChain: string(kubeSourceRangesFirewallChain), MatchSet: kubeLoadBalancerFWSet, 2271 }}, 2272 string(kubeSourceRangesFirewallChain): {{ 2273 JumpChain: "RETURN", MatchSet: kubeLoadBalancerSourceCIDRSet, 2274 }, { 2275 JumpChain: "DROP", MatchSet: "", 2276 }}, 2277 } 2278 checkIptables(t, ipt, epIpt) 2279 } 2280 2281 func TestAcceptIPVSTraffic(t *testing.T) { 2282 ipt, fp := buildFakeProxier() 2283 2284 ingressIP := "1.2.3.4" 2285 externalIP := []string{"5.6.7.8"} 2286 svcInfos := []struct { 2287 svcType v1.ServiceType 2288 svcIP string 2289 svcName string 2290 epIP string 2291 }{ 2292 {v1.ServiceTypeClusterIP, "10.20.30.40", "svc1", "10.180.0.1"}, 2293 {v1.ServiceTypeLoadBalancer, "10.20.30.41", "svc2", "10.180.0.2"}, 2294 {v1.ServiceTypeNodePort, "10.20.30.42", "svc3", "10.180.0.3"}, 2295 } 2296 2297 for _, svcInfo := range svcInfos { 2298 makeServiceMap(fp, 2299 makeTestService("ns1", svcInfo.svcName, func(svc *v1.Service) { 2300 svc.Spec.Type = svcInfo.svcType 2301 svc.Spec.ClusterIP = svcInfo.svcIP 2302 svc.Spec.Ports = []v1.ServicePort{{ 2303 Name: "p80", 2304 Port: 80, 2305 Protocol: v1.ProtocolTCP, 2306 NodePort: 80, 2307 }} 2308 if svcInfo.svcType == v1.ServiceTypeLoadBalancer { 2309 svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{ 2310 IP: ingressIP, 2311 }} 2312 } 2313 if svcInfo.svcType == v1.ServiceTypeClusterIP { 2314 svc.Spec.ExternalIPs = externalIP 2315 } 2316 }), 2317 ) 2318 2319 populateEndpointSlices(fp, 2320 makeTestEndpointSlice("ns1", "p80", 1, func(eps *discovery.EndpointSlice) { 2321 eps.Endpoints = []discovery.Endpoint{{ 2322 Addresses: []string{svcInfo.epIP}, 2323 }} 2324 eps.Ports = []discovery.EndpointPort{{ 2325 Name: ptr.To("p80"), 2326 Port: ptr.To[int32](80), 2327 Protocol: ptr.To(v1.ProtocolUDP), 2328 }} 2329 }), 2330 ) 2331 } 2332 fp.syncProxyRules() 2333 2334 // Check iptables chain and rules 2335 epIpt := netlinktest.ExpectedIptablesChain{ 2336 string(kubeServicesChain): { 2337 {JumpChain: "RETURN", SourceAddress: "127.0.0.0/8"}, 2338 {JumpChain: string(kubeLoadBalancerChain), MatchSet: kubeLoadBalancerSet}, 2339 {JumpChain: string(kubeMarkMasqChain), MatchSet: kubeClusterIPSet}, 2340 {JumpChain: string(kubeMarkMasqChain), MatchSet: kubeExternalIPSet}, 2341 {JumpChain: "ACCEPT", MatchSet: kubeExternalIPSet}, // With externalTrafficOnlyArgs 2342 {JumpChain: "ACCEPT", MatchSet: kubeExternalIPSet}, // With dstLocalOnlyArgs 2343 {JumpChain: string(kubeNodePortChain), MatchSet: ""}, 2344 {JumpChain: "ACCEPT", MatchSet: kubeClusterIPSet}, 2345 {JumpChain: "ACCEPT", MatchSet: kubeLoadBalancerSet}, 2346 }, 2347 } 2348 checkIptables(t, ipt, epIpt) 2349 } 2350 2351 func TestOnlyLocalLoadBalancing(t *testing.T) { 2352 ipt, fp := buildFakeProxier() 2353 2354 svcIP := "10.20.30.41" 2355 svcPort := 80 2356 svcNodePort := 3001 2357 svcLBIP := "1.2.3.4" 2358 svcPortName := proxy.ServicePortName{ 2359 NamespacedName: makeNSN("ns1", "svc1"), 2360 Port: "p80", 2361 } 2362 2363 makeServiceMap(fp, 2364 makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { 2365 svc.Spec.Type = "LoadBalancer" 2366 svc.Spec.ClusterIP = svcIP 2367 svc.Spec.Ports = []v1.ServicePort{{ 2368 Name: svcPortName.Port, 2369 Port: int32(svcPort), 2370 Protocol: v1.ProtocolTCP, 2371 NodePort: int32(svcNodePort), 2372 }} 2373 svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{ 2374 IP: svcLBIP, 2375 }} 2376 svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyLocal 2377 }), 2378 ) 2379 2380 epIP := "10.180.0.1" 2381 epIP1 := "10.180.1.1" 2382 2383 populateEndpointSlices(fp, 2384 makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) { 2385 eps.AddressType = discovery.AddressTypeIPv4 2386 eps.Endpoints = []discovery.Endpoint{ 2387 { // **local** endpoint address, should be added as RS 2388 Addresses: []string{epIP}, 2389 NodeName: ptr.To(testHostname), 2390 }, 2391 { // **remote** endpoint address, should not be added as RS 2392 Addresses: []string{epIP1}, 2393 NodeName: ptr.To("other-hostname"), 2394 }} 2395 eps.Ports = []discovery.EndpointPort{{ 2396 Name: ptr.To(svcPortName.Port), 2397 Port: ptr.To(int32(svcPort)), 2398 Protocol: ptr.To(v1.ProtocolTCP), 2399 }} 2400 }), 2401 ) 2402 2403 fp.syncProxyRules() 2404 2405 // Expect 2 services and 1 destination 2406 epVS := &netlinktest.ExpectedVirtualServer{ 2407 VSNum: 2, IP: svcLBIP, Port: uint16(svcPort), Protocol: string(v1.ProtocolTCP), 2408 RS: []netlinktest.ExpectedRealServer{{ 2409 IP: epIP, Port: uint16(svcPort), 2410 }}} 2411 checkIPVS(t, fp, epVS) 2412 2413 // check ipSet rules 2414 epIPSet := netlinktest.ExpectedIPSet{ 2415 kubeLoadBalancerSet: {{ 2416 IP: svcLBIP, 2417 Port: svcPort, 2418 Protocol: strings.ToLower(string(v1.ProtocolTCP)), 2419 SetType: utilipset.HashIPPort, 2420 }}, 2421 kubeLoadBalancerLocalSet: {{ 2422 IP: svcLBIP, 2423 Port: svcPort, 2424 Protocol: strings.ToLower(string(v1.ProtocolTCP)), 2425 SetType: utilipset.HashIPPort, 2426 }}, 2427 } 2428 checkIPSet(t, fp, epIPSet) 2429 2430 // Check iptables chain and rules 2431 epIpt := netlinktest.ExpectedIptablesChain{ 2432 string(kubeServicesChain): {{ 2433 JumpChain: "RETURN", SourceAddress: "127.0.0.0/8", 2434 }, { 2435 JumpChain: string(kubeLoadBalancerChain), MatchSet: kubeLoadBalancerSet, 2436 }, { 2437 JumpChain: string(kubeMarkMasqChain), MatchSet: kubeClusterIPSet, 2438 }, { 2439 JumpChain: string(kubeNodePortChain), MatchSet: "", 2440 }, { 2441 JumpChain: "ACCEPT", MatchSet: kubeClusterIPSet, 2442 }, { 2443 JumpChain: "ACCEPT", MatchSet: kubeLoadBalancerSet, 2444 }}, 2445 string(kubeLoadBalancerChain): {{ 2446 JumpChain: "RETURN", MatchSet: kubeLoadBalancerLocalSet, 2447 }, { 2448 JumpChain: string(kubeMarkMasqChain), MatchSet: "", 2449 }}, 2450 } 2451 checkIptables(t, ipt, epIpt) 2452 } 2453 2454 func addTestPort(array []v1.ServicePort, name string, protocol v1.Protocol, port, nodeport int32, targetPort int) []v1.ServicePort { 2455 svcPort := v1.ServicePort{ 2456 Name: name, 2457 Protocol: protocol, 2458 Port: port, 2459 NodePort: nodeport, 2460 TargetPort: intstr.FromInt32(int32(targetPort)), 2461 } 2462 return append(array, svcPort) 2463 } 2464 2465 func TestBuildServiceMapAddRemove(t *testing.T) { 2466 ipt := iptablestest.NewFake() 2467 ipvs := ipvstest.NewFake() 2468 ipset := ipsettest.NewFake(testIPSetVersion) 2469 fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol) 2470 2471 services := []*v1.Service{ 2472 makeTestService("somewhere-else", "cluster-ip", func(svc *v1.Service) { 2473 svc.Spec.Type = v1.ServiceTypeClusterIP 2474 svc.Spec.ClusterIP = "172.16.55.4" 2475 svc.Spec.Ports = addTestPort(svc.Spec.Ports, "something", "UDP", 1234, 4321, 0) 2476 svc.Spec.Ports = addTestPort(svc.Spec.Ports, "somethingelse", "UDP", 1235, 5321, 0) 2477 svc.Spec.Ports = addTestPort(svc.Spec.Ports, "somesctp", "SCTP", 1236, 6321, 0) 2478 }), 2479 makeTestService("somewhere-else", "node-port", func(svc *v1.Service) { 2480 svc.Spec.Type = v1.ServiceTypeNodePort 2481 svc.Spec.ClusterIP = "172.16.55.10" 2482 svc.Spec.Ports = addTestPort(svc.Spec.Ports, "blahblah", "UDP", 345, 678, 0) 2483 svc.Spec.Ports = addTestPort(svc.Spec.Ports, "moreblahblah", "TCP", 344, 677, 0) 2484 svc.Spec.Ports = addTestPort(svc.Spec.Ports, "sctpblah", "SCTP", 343, 676, 0) 2485 }), 2486 makeTestService("somewhere", "load-balancer", func(svc *v1.Service) { 2487 svc.Spec.Type = v1.ServiceTypeLoadBalancer 2488 svc.Spec.ClusterIP = "172.16.55.11" 2489 svc.Spec.LoadBalancerIP = "5.6.7.8" 2490 svc.Spec.Ports = addTestPort(svc.Spec.Ports, "foobar", "UDP", 8675, 30061, 7000) 2491 svc.Spec.Ports = addTestPort(svc.Spec.Ports, "baz", "UDP", 8676, 30062, 7001) 2492 svc.Spec.Ports = addTestPort(svc.Spec.Ports, "sctpfoo", "SCTP", 8677, 30063, 7002) 2493 svc.Status.LoadBalancer = v1.LoadBalancerStatus{ 2494 Ingress: []v1.LoadBalancerIngress{ 2495 {IP: "10.1.2.4"}, 2496 }, 2497 } 2498 }), 2499 makeTestService("somewhere", "only-local-load-balancer", func(svc *v1.Service) { 2500 svc.Spec.Type = v1.ServiceTypeLoadBalancer 2501 svc.Spec.ClusterIP = "172.16.55.12" 2502 svc.Spec.LoadBalancerIP = "5.6.7.8" 2503 svc.Spec.Ports = addTestPort(svc.Spec.Ports, "foobar2", "UDP", 8677, 30063, 7002) 2504 svc.Spec.Ports = addTestPort(svc.Spec.Ports, "baz", "UDP", 8678, 30064, 7003) 2505 svc.Spec.Ports = addTestPort(svc.Spec.Ports, "sctpbaz", "SCTP", 8679, 30065, 7004) 2506 svc.Status.LoadBalancer = v1.LoadBalancerStatus{ 2507 Ingress: []v1.LoadBalancerIngress{ 2508 {IP: "10.1.2.3"}, 2509 }, 2510 } 2511 svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyLocal 2512 svc.Spec.HealthCheckNodePort = 345 2513 }), 2514 } 2515 2516 for i := range services { 2517 fp.OnServiceAdd(services[i]) 2518 } 2519 result := fp.svcPortMap.Update(fp.serviceChanges) 2520 if len(fp.svcPortMap) != 12 { 2521 t.Errorf("expected service map length 12, got %v", fp.svcPortMap) 2522 } 2523 2524 if len(result.DeletedUDPClusterIPs) != 0 { 2525 // Services only added, so nothing stale yet 2526 t.Errorf("expected stale UDP services length 0, got %d", len(result.DeletedUDPClusterIPs)) 2527 } 2528 2529 // The only-local-loadbalancer ones get added 2530 healthCheckNodePorts := fp.svcPortMap.HealthCheckNodePorts() 2531 if len(healthCheckNodePorts) != 1 { 2532 t.Errorf("expected 1 healthcheck port, got %v", healthCheckNodePorts) 2533 } else { 2534 nsn := makeNSN("somewhere", "only-local-load-balancer") 2535 if port, found := healthCheckNodePorts[nsn]; !found || port != 345 { 2536 t.Errorf("expected healthcheck port [%q]=345: got %v", nsn, healthCheckNodePorts) 2537 } 2538 } 2539 2540 // Remove some stuff 2541 // oneService is a modification of services[0] with removed first port. 2542 oneService := makeTestService("somewhere-else", "cluster-ip", func(svc *v1.Service) { 2543 svc.Spec.Type = v1.ServiceTypeClusterIP 2544 svc.Spec.ClusterIP = "172.16.55.4" 2545 svc.Spec.Ports = addTestPort(svc.Spec.Ports, "somethingelse", "UDP", 1235, 5321, 0) 2546 }) 2547 2548 fp.OnServiceUpdate(services[0], oneService) 2549 fp.OnServiceDelete(services[1]) 2550 fp.OnServiceDelete(services[2]) 2551 fp.OnServiceDelete(services[3]) 2552 2553 result = fp.svcPortMap.Update(fp.serviceChanges) 2554 if len(fp.svcPortMap) != 1 { 2555 t.Errorf("expected service map length 1, got %v", fp.svcPortMap) 2556 } 2557 2558 // All services but one were deleted. While you'd expect only the ClusterIPs 2559 // from the three deleted services here, we still have the ClusterIP for 2560 // the not-deleted service, because one of it's ServicePorts was deleted. 2561 expectedStaleUDPServices := []string{"172.16.55.10", "172.16.55.4", "172.16.55.11", "172.16.55.12"} 2562 if len(result.DeletedUDPClusterIPs) != len(expectedStaleUDPServices) { 2563 t.Errorf("expected stale UDP services length %d, got %v", len(expectedStaleUDPServices), result.DeletedUDPClusterIPs.UnsortedList()) 2564 } 2565 for _, ip := range expectedStaleUDPServices { 2566 if !result.DeletedUDPClusterIPs.Has(ip) { 2567 t.Errorf("expected stale UDP service service %s", ip) 2568 } 2569 } 2570 2571 healthCheckNodePorts = fp.svcPortMap.HealthCheckNodePorts() 2572 if len(healthCheckNodePorts) != 0 { 2573 t.Errorf("expected 0 healthcheck ports, got %v", healthCheckNodePorts) 2574 } 2575 } 2576 2577 func TestBuildServiceMapServiceHeadless(t *testing.T) { 2578 ipt := iptablestest.NewFake() 2579 ipvs := ipvstest.NewFake() 2580 ipset := ipsettest.NewFake(testIPSetVersion) 2581 fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol) 2582 2583 makeServiceMap(fp, 2584 makeTestService("somewhere-else", "headless", func(svc *v1.Service) { 2585 svc.Spec.Type = v1.ServiceTypeClusterIP 2586 svc.Spec.ClusterIP = v1.ClusterIPNone 2587 svc.Spec.Ports = addTestPort(svc.Spec.Ports, "rpc", "UDP", 1234, 0, 0) 2588 }), 2589 makeTestService("somewhere-else", "headless-without-port", func(svc *v1.Service) { 2590 svc.Spec.Type = v1.ServiceTypeClusterIP 2591 svc.Spec.ClusterIP = v1.ClusterIPNone 2592 }), 2593 makeTestService("somewhere-else", "headless-sctp", func(svc *v1.Service) { 2594 svc.Spec.Type = v1.ServiceTypeClusterIP 2595 svc.Spec.ClusterIP = v1.ClusterIPNone 2596 svc.Spec.Ports = addTestPort(svc.Spec.Ports, "sip", "SCTP", 1235, 0, 0) 2597 }), 2598 ) 2599 2600 // Headless service should be ignored 2601 result := fp.svcPortMap.Update(fp.serviceChanges) 2602 if len(fp.svcPortMap) != 0 { 2603 t.Errorf("expected service map length 0, got %d", len(fp.svcPortMap)) 2604 } 2605 2606 if len(result.DeletedUDPClusterIPs) != 0 { 2607 t.Errorf("expected stale UDP services length 0, got %d", len(result.DeletedUDPClusterIPs)) 2608 } 2609 2610 // No proxied services, so no healthchecks 2611 healthCheckNodePorts := fp.svcPortMap.HealthCheckNodePorts() 2612 if len(healthCheckNodePorts) != 0 { 2613 t.Errorf("expected healthcheck ports length 0, got %d", len(healthCheckNodePorts)) 2614 } 2615 } 2616 2617 func TestBuildServiceMapServiceTypeExternalName(t *testing.T) { 2618 ipt := iptablestest.NewFake() 2619 ipvs := ipvstest.NewFake() 2620 ipset := ipsettest.NewFake(testIPSetVersion) 2621 fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol) 2622 2623 makeServiceMap(fp, 2624 makeTestService("somewhere-else", "external-name", func(svc *v1.Service) { 2625 svc.Spec.Type = v1.ServiceTypeExternalName 2626 svc.Spec.ClusterIP = "172.16.55.4" // Should be ignored 2627 svc.Spec.ExternalName = "foo2.bar.com" 2628 svc.Spec.Ports = addTestPort(svc.Spec.Ports, "blah", "UDP", 1235, 5321, 0) 2629 }), 2630 ) 2631 2632 result := fp.svcPortMap.Update(fp.serviceChanges) 2633 if len(fp.svcPortMap) != 0 { 2634 t.Errorf("expected service map length 0, got %v", fp.svcPortMap) 2635 } 2636 if len(result.DeletedUDPClusterIPs) != 0 { 2637 t.Errorf("expected stale UDP services length 0, got %v", result.DeletedUDPClusterIPs) 2638 } 2639 2640 // No proxied services, so no healthchecks 2641 healthCheckNodePorts := fp.svcPortMap.HealthCheckNodePorts() 2642 if len(healthCheckNodePorts) != 0 { 2643 t.Errorf("expected healthcheck ports length 0, got %v", healthCheckNodePorts) 2644 } 2645 } 2646 2647 func TestBuildServiceMapServiceUpdate(t *testing.T) { 2648 ipt := iptablestest.NewFake() 2649 ipvs := ipvstest.NewFake() 2650 ipset := ipsettest.NewFake(testIPSetVersion) 2651 fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol) 2652 2653 servicev1 := makeTestService("somewhere", "some-service", func(svc *v1.Service) { 2654 svc.Spec.Type = v1.ServiceTypeClusterIP 2655 svc.Spec.ClusterIP = "172.16.55.4" 2656 svc.Spec.Ports = addTestPort(svc.Spec.Ports, "something", "UDP", 1234, 4321, 0) 2657 svc.Spec.Ports = addTestPort(svc.Spec.Ports, "somethingelse", "TCP", 1235, 5321, 0) 2658 }) 2659 servicev2 := makeTestService("somewhere", "some-service", func(svc *v1.Service) { 2660 svc.Spec.Type = v1.ServiceTypeLoadBalancer 2661 svc.Spec.ClusterIP = "172.16.55.4" 2662 svc.Spec.LoadBalancerIP = "5.6.7.8" 2663 svc.Spec.Ports = addTestPort(svc.Spec.Ports, "something", "UDP", 1234, 4321, 7002) 2664 svc.Spec.Ports = addTestPort(svc.Spec.Ports, "somethingelse", "TCP", 1235, 5321, 7003) 2665 svc.Status.LoadBalancer = v1.LoadBalancerStatus{ 2666 Ingress: []v1.LoadBalancerIngress{ 2667 {IP: "10.1.2.3"}, 2668 }, 2669 } 2670 svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyLocal 2671 svc.Spec.HealthCheckNodePort = 345 2672 }) 2673 2674 fp.OnServiceAdd(servicev1) 2675 2676 result := fp.svcPortMap.Update(fp.serviceChanges) 2677 if len(fp.svcPortMap) != 2 { 2678 t.Errorf("expected service map length 2, got %v", fp.svcPortMap) 2679 } 2680 if len(result.DeletedUDPClusterIPs) != 0 { 2681 // Services only added, so nothing stale yet 2682 t.Errorf("expected stale UDP services length 0, got %d", len(result.DeletedUDPClusterIPs)) 2683 } 2684 2685 healthCheckNodePorts := fp.svcPortMap.HealthCheckNodePorts() 2686 if len(healthCheckNodePorts) != 0 { 2687 t.Errorf("expected healthcheck ports length 0, got %v", healthCheckNodePorts) 2688 } 2689 2690 // Change service to load-balancer 2691 fp.OnServiceUpdate(servicev1, servicev2) 2692 result = fp.svcPortMap.Update(fp.serviceChanges) 2693 if len(fp.svcPortMap) != 2 { 2694 t.Errorf("expected service map length 2, got %v", fp.svcPortMap) 2695 } 2696 if len(result.DeletedUDPClusterIPs) != 0 { 2697 t.Errorf("expected stale UDP services length 0, got %v", result.DeletedUDPClusterIPs.UnsortedList()) 2698 } 2699 2700 healthCheckNodePorts = fp.svcPortMap.HealthCheckNodePorts() 2701 if len(healthCheckNodePorts) != 1 { 2702 t.Errorf("expected healthcheck ports length 1, got %v", healthCheckNodePorts) 2703 } 2704 2705 // No change; make sure the service map stays the same and there are 2706 // no health-check changes 2707 fp.OnServiceUpdate(servicev2, servicev2) 2708 result = fp.svcPortMap.Update(fp.serviceChanges) 2709 if len(fp.svcPortMap) != 2 { 2710 t.Errorf("expected service map length 2, got %v", fp.svcPortMap) 2711 } 2712 if len(result.DeletedUDPClusterIPs) != 0 { 2713 t.Errorf("expected stale UDP services length 0, got %v", result.DeletedUDPClusterIPs.UnsortedList()) 2714 } 2715 2716 healthCheckNodePorts = fp.svcPortMap.HealthCheckNodePorts() 2717 if len(healthCheckNodePorts) != 1 { 2718 t.Errorf("expected healthcheck ports length 1, got %v", healthCheckNodePorts) 2719 } 2720 2721 // And back to ClusterIP 2722 fp.OnServiceUpdate(servicev2, servicev1) 2723 result = fp.svcPortMap.Update(fp.serviceChanges) 2724 if len(fp.svcPortMap) != 2 { 2725 t.Errorf("expected service map length 2, got %v", fp.svcPortMap) 2726 } 2727 if len(result.DeletedUDPClusterIPs) != 0 { 2728 // Services only added, so nothing stale yet 2729 t.Errorf("expected stale UDP services length 0, got %d", len(result.DeletedUDPClusterIPs)) 2730 } 2731 2732 healthCheckNodePorts = fp.svcPortMap.HealthCheckNodePorts() 2733 if len(healthCheckNodePorts) != 0 { 2734 t.Errorf("expected healthcheck ports length 0, got %v", healthCheckNodePorts) 2735 } 2736 } 2737 2738 func TestSessionAffinity(t *testing.T) { 2739 ipt := iptablestest.NewFake() 2740 ipvs := ipvstest.NewFake() 2741 ipset := ipsettest.NewFake(testIPSetVersion) 2742 nodeIP := "100.101.102.103" 2743 fp := NewFakeProxier(ipt, ipvs, ipset, []string{nodeIP}, nil, v1.IPv4Protocol) 2744 svcIP := "10.20.30.41" 2745 svcPort := 80 2746 svcNodePort := 3001 2747 svcExternalIPs := "50.60.70.81" 2748 svcPortName := proxy.ServicePortName{ 2749 NamespacedName: makeNSN("ns1", "svc1"), 2750 Port: "p80", 2751 } 2752 2753 makeServiceMap(fp, 2754 makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { 2755 svc.Spec.Type = "NodePort" 2756 svc.Spec.ClusterIP = svcIP 2757 svc.Spec.ExternalIPs = []string{svcExternalIPs} 2758 svc.Spec.SessionAffinity = v1.ServiceAffinityClientIP 2759 svc.Spec.SessionAffinityConfig = &v1.SessionAffinityConfig{ 2760 ClientIP: &v1.ClientIPConfig{ 2761 TimeoutSeconds: ptr.To[int32](v1.DefaultClientIPServiceAffinitySeconds), 2762 }, 2763 } 2764 svc.Spec.Ports = []v1.ServicePort{{ 2765 Name: svcPortName.Port, 2766 Port: int32(svcPort), 2767 Protocol: v1.ProtocolTCP, 2768 NodePort: int32(svcNodePort), 2769 }} 2770 }), 2771 ) 2772 fp.syncProxyRules() 2773 2774 // check ipvs service and destinations 2775 services, err := ipvs.GetVirtualServers() 2776 if err != nil { 2777 t.Errorf("Failed to get ipvs services, err: %v", err) 2778 } 2779 for _, svc := range services { 2780 if svc.Timeout != uint32(v1.DefaultClientIPServiceAffinitySeconds) { 2781 t.Errorf("Unexpected mismatch ipvs service session affinity timeout: %d, expected: %d", svc.Timeout, v1.DefaultClientIPServiceAffinitySeconds) 2782 } 2783 } 2784 } 2785 2786 func makeServicePortName(ns, name, port string, protocol v1.Protocol) proxy.ServicePortName { 2787 return proxy.ServicePortName{ 2788 NamespacedName: makeNSN(ns, name), 2789 Port: port, 2790 Protocol: protocol, 2791 } 2792 } 2793 2794 func Test_updateEndpointsMap(t *testing.T) { 2795 emptyEndpointSlices := []*discovery.EndpointSlice{ 2796 makeTestEndpointSlice("ns1", "ep1", 1, func(*discovery.EndpointSlice) {}), 2797 } 2798 subset1 := func(eps *discovery.EndpointSlice) { 2799 eps.AddressType = discovery.AddressTypeIPv4 2800 eps.Endpoints = []discovery.Endpoint{{ 2801 Addresses: []string{"1.1.1.1"}, 2802 }} 2803 eps.Ports = []discovery.EndpointPort{{ 2804 Name: ptr.To("p11"), 2805 Port: ptr.To[int32](11), 2806 Protocol: ptr.To(v1.ProtocolUDP), 2807 }} 2808 } 2809 subset2 := func(eps *discovery.EndpointSlice) { 2810 eps.AddressType = discovery.AddressTypeIPv4 2811 eps.Endpoints = []discovery.Endpoint{{ 2812 Addresses: []string{"1.1.1.2"}, 2813 }} 2814 eps.Ports = []discovery.EndpointPort{{ 2815 Name: ptr.To("p12"), 2816 Port: ptr.To[int32](12), 2817 Protocol: ptr.To(v1.ProtocolUDP), 2818 }} 2819 } 2820 namedPortLocal := []*discovery.EndpointSlice{ 2821 makeTestEndpointSlice("ns1", "ep1", 1, 2822 func(eps *discovery.EndpointSlice) { 2823 eps.AddressType = discovery.AddressTypeIPv4 2824 eps.Endpoints = []discovery.Endpoint{{ 2825 Addresses: []string{"1.1.1.1"}, 2826 NodeName: ptr.To(testHostname), 2827 }} 2828 eps.Ports = []discovery.EndpointPort{{ 2829 Name: ptr.To("p11"), 2830 Port: ptr.To[int32](11), 2831 Protocol: ptr.To(v1.ProtocolUDP), 2832 }} 2833 }), 2834 } 2835 namedPort := []*discovery.EndpointSlice{ 2836 makeTestEndpointSlice("ns1", "ep1", 1, subset1), 2837 } 2838 namedPortRenamed := []*discovery.EndpointSlice{ 2839 makeTestEndpointSlice("ns1", "ep1", 1, 2840 func(eps *discovery.EndpointSlice) { 2841 eps.AddressType = discovery.AddressTypeIPv4 2842 eps.Endpoints = []discovery.Endpoint{{ 2843 Addresses: []string{"1.1.1.1"}, 2844 }} 2845 eps.Ports = []discovery.EndpointPort{{ 2846 Name: ptr.To("p11-2"), 2847 Port: ptr.To[int32](11), 2848 Protocol: ptr.To(v1.ProtocolUDP), 2849 }} 2850 }), 2851 } 2852 namedPortRenumbered := []*discovery.EndpointSlice{ 2853 makeTestEndpointSlice("ns1", "ep1", 1, 2854 func(eps *discovery.EndpointSlice) { 2855 eps.AddressType = discovery.AddressTypeIPv4 2856 eps.Endpoints = []discovery.Endpoint{{ 2857 Addresses: []string{"1.1.1.1"}, 2858 }} 2859 eps.Ports = []discovery.EndpointPort{{ 2860 Name: ptr.To("p11"), 2861 Port: ptr.To[int32](22), 2862 Protocol: ptr.To(v1.ProtocolUDP), 2863 }} 2864 }), 2865 } 2866 namedPortsLocalNoLocal := []*discovery.EndpointSlice{ 2867 makeTestEndpointSlice("ns1", "ep1", 1, 2868 func(eps *discovery.EndpointSlice) { 2869 eps.AddressType = discovery.AddressTypeIPv4 2870 eps.Endpoints = []discovery.Endpoint{{ 2871 Addresses: []string{"1.1.1.1"}, 2872 }, { 2873 Addresses: []string{"1.1.1.2"}, 2874 NodeName: ptr.To(testHostname), 2875 }} 2876 eps.Ports = []discovery.EndpointPort{{ 2877 Name: ptr.To("p11"), 2878 Port: ptr.To[int32](11), 2879 Protocol: ptr.To(v1.ProtocolUDP), 2880 }, { 2881 Name: ptr.To("p12"), 2882 Port: ptr.To[int32](12), 2883 Protocol: ptr.To(v1.ProtocolUDP), 2884 }} 2885 }), 2886 } 2887 multipleSubsets := []*discovery.EndpointSlice{ 2888 makeTestEndpointSlice("ns1", "ep1", 1, subset1), 2889 makeTestEndpointSlice("ns1", "ep1", 2, subset2), 2890 } 2891 subsetLocal := func(eps *discovery.EndpointSlice) { 2892 eps.AddressType = discovery.AddressTypeIPv4 2893 eps.Endpoints = []discovery.Endpoint{{ 2894 Addresses: []string{"1.1.1.2"}, 2895 NodeName: ptr.To(testHostname), 2896 }} 2897 eps.Ports = []discovery.EndpointPort{{ 2898 Name: ptr.To("p12"), 2899 Port: ptr.To[int32](12), 2900 Protocol: ptr.To(v1.ProtocolUDP), 2901 }} 2902 } 2903 multipleSubsetsWithLocal := []*discovery.EndpointSlice{ 2904 makeTestEndpointSlice("ns1", "ep1", 1, subset1), 2905 makeTestEndpointSlice("ns1", "ep1", 2, subsetLocal), 2906 } 2907 subsetMultiplePortsLocal := func(eps *discovery.EndpointSlice) { 2908 eps.AddressType = discovery.AddressTypeIPv4 2909 eps.Endpoints = []discovery.Endpoint{{ 2910 Addresses: []string{"1.1.1.1"}, 2911 NodeName: ptr.To(testHostname), 2912 }} 2913 eps.Ports = []discovery.EndpointPort{{ 2914 Name: ptr.To("p11"), 2915 Port: ptr.To[int32](11), 2916 Protocol: ptr.To(v1.ProtocolUDP), 2917 }, { 2918 Name: ptr.To("p12"), 2919 Port: ptr.To[int32](12), 2920 Protocol: ptr.To(v1.ProtocolUDP), 2921 }} 2922 } 2923 subset3 := func(eps *discovery.EndpointSlice) { 2924 eps.AddressType = discovery.AddressTypeIPv4 2925 eps.Endpoints = []discovery.Endpoint{{ 2926 Addresses: []string{"1.1.1.3"}, 2927 }} 2928 eps.Ports = []discovery.EndpointPort{{ 2929 Name: ptr.To("p13"), 2930 Port: ptr.To[int32](13), 2931 Protocol: ptr.To(v1.ProtocolUDP), 2932 }} 2933 } 2934 multipleSubsetsMultiplePortsLocal := []*discovery.EndpointSlice{ 2935 makeTestEndpointSlice("ns1", "ep1", 1, subsetMultiplePortsLocal), 2936 makeTestEndpointSlice("ns1", "ep1", 2, subset3), 2937 } 2938 subsetMultipleIPsPorts1 := func(eps *discovery.EndpointSlice) { 2939 eps.AddressType = discovery.AddressTypeIPv4 2940 eps.Endpoints = []discovery.Endpoint{{ 2941 Addresses: []string{"1.1.1.1"}, 2942 }, { 2943 Addresses: []string{"1.1.1.2"}, 2944 NodeName: ptr.To(testHostname), 2945 }} 2946 eps.Ports = []discovery.EndpointPort{{ 2947 Name: ptr.To("p11"), 2948 Port: ptr.To[int32](11), 2949 Protocol: ptr.To(v1.ProtocolUDP), 2950 }, { 2951 Name: ptr.To("p12"), 2952 Port: ptr.To[int32](12), 2953 Protocol: ptr.To(v1.ProtocolUDP), 2954 }} 2955 } 2956 subsetMultipleIPsPorts2 := func(eps *discovery.EndpointSlice) { 2957 eps.AddressType = discovery.AddressTypeIPv4 2958 eps.Endpoints = []discovery.Endpoint{{ 2959 Addresses: []string{"1.1.1.3"}, 2960 }, { 2961 Addresses: []string{"1.1.1.4"}, 2962 NodeName: ptr.To(testHostname), 2963 }} 2964 eps.Ports = []discovery.EndpointPort{{ 2965 Name: ptr.To("p13"), 2966 Port: ptr.To[int32](13), 2967 Protocol: ptr.To(v1.ProtocolUDP), 2968 }, { 2969 Name: ptr.To("p14"), 2970 Port: ptr.To[int32](14), 2971 Protocol: ptr.To(v1.ProtocolUDP), 2972 }} 2973 } 2974 subsetMultipleIPsPorts3 := func(eps *discovery.EndpointSlice) { 2975 eps.AddressType = discovery.AddressTypeIPv4 2976 eps.Endpoints = []discovery.Endpoint{{ 2977 Addresses: []string{"2.2.2.1"}, 2978 }, { 2979 Addresses: []string{"2.2.2.2"}, 2980 NodeName: ptr.To(testHostname), 2981 }} 2982 eps.Ports = []discovery.EndpointPort{{ 2983 Name: ptr.To("p21"), 2984 Port: ptr.To[int32](21), 2985 Protocol: ptr.To(v1.ProtocolUDP), 2986 }, { 2987 Name: ptr.To("p22"), 2988 Port: ptr.To[int32](22), 2989 Protocol: ptr.To(v1.ProtocolUDP), 2990 }} 2991 } 2992 multipleSubsetsIPsPorts := []*discovery.EndpointSlice{ 2993 makeTestEndpointSlice("ns1", "ep1", 1, subsetMultipleIPsPorts1), 2994 makeTestEndpointSlice("ns1", "ep1", 2, subsetMultipleIPsPorts2), 2995 makeTestEndpointSlice("ns2", "ep2", 1, subsetMultipleIPsPorts3), 2996 } 2997 complexSubset1 := func(eps *discovery.EndpointSlice) { 2998 eps.AddressType = discovery.AddressTypeIPv4 2999 eps.Endpoints = []discovery.Endpoint{{ 3000 Addresses: []string{"2.2.2.2"}, 3001 NodeName: ptr.To(testHostname), 3002 }, { 3003 Addresses: []string{"2.2.2.22"}, 3004 NodeName: ptr.To(testHostname), 3005 }} 3006 eps.Ports = []discovery.EndpointPort{{ 3007 Name: ptr.To("p22"), 3008 Port: ptr.To[int32](22), 3009 Protocol: ptr.To(v1.ProtocolUDP), 3010 }} 3011 } 3012 complexSubset2 := func(eps *discovery.EndpointSlice) { 3013 eps.AddressType = discovery.AddressTypeIPv4 3014 eps.Endpoints = []discovery.Endpoint{{ 3015 Addresses: []string{"2.2.2.3"}, 3016 NodeName: ptr.To(testHostname), 3017 }} 3018 eps.Ports = []discovery.EndpointPort{{ 3019 Name: ptr.To("p23"), 3020 Port: ptr.To[int32](23), 3021 Protocol: ptr.To(v1.ProtocolUDP), 3022 }} 3023 } 3024 complexSubset3 := func(eps *discovery.EndpointSlice) { 3025 eps.AddressType = discovery.AddressTypeIPv4 3026 eps.Endpoints = []discovery.Endpoint{{ 3027 Addresses: []string{"4.4.4.4"}, 3028 NodeName: ptr.To(testHostname), 3029 }, { 3030 Addresses: []string{"4.4.4.5"}, 3031 NodeName: ptr.To(testHostname), 3032 }} 3033 eps.Ports = []discovery.EndpointPort{{ 3034 Name: ptr.To("p44"), 3035 Port: ptr.To[int32](44), 3036 Protocol: ptr.To(v1.ProtocolUDP), 3037 }} 3038 } 3039 complexSubset4 := func(eps *discovery.EndpointSlice) { 3040 eps.AddressType = discovery.AddressTypeIPv4 3041 eps.Endpoints = []discovery.Endpoint{{ 3042 Addresses: []string{"4.4.4.6"}, 3043 NodeName: ptr.To(testHostname), 3044 }} 3045 eps.Ports = []discovery.EndpointPort{{ 3046 Name: ptr.To("p45"), 3047 Port: ptr.To[int32](45), 3048 Protocol: ptr.To(v1.ProtocolUDP), 3049 }} 3050 } 3051 complexSubset5 := func(eps *discovery.EndpointSlice) { 3052 eps.AddressType = discovery.AddressTypeIPv4 3053 eps.Endpoints = []discovery.Endpoint{{ 3054 Addresses: []string{"1.1.1.1"}, 3055 }, { 3056 Addresses: []string{"1.1.1.11"}, 3057 }} 3058 eps.Ports = []discovery.EndpointPort{{ 3059 Name: ptr.To("p11"), 3060 Port: ptr.To[int32](11), 3061 Protocol: ptr.To(v1.ProtocolUDP), 3062 }} 3063 } 3064 complexSubset6 := func(eps *discovery.EndpointSlice) { 3065 eps.AddressType = discovery.AddressTypeIPv4 3066 eps.Endpoints = []discovery.Endpoint{{ 3067 Addresses: []string{"1.1.1.2"}, 3068 }} 3069 eps.Ports = []discovery.EndpointPort{{ 3070 Name: ptr.To("p12"), 3071 Port: ptr.To[int32](12), 3072 Protocol: ptr.To(v1.ProtocolUDP), 3073 }, { 3074 Name: ptr.To("p122"), 3075 Port: ptr.To[int32](122), 3076 Protocol: ptr.To(v1.ProtocolUDP), 3077 }} 3078 } 3079 complexSubset7 := func(eps *discovery.EndpointSlice) { 3080 eps.AddressType = discovery.AddressTypeIPv4 3081 eps.Endpoints = []discovery.Endpoint{{ 3082 Addresses: []string{"3.3.3.3"}, 3083 }} 3084 eps.Ports = []discovery.EndpointPort{{ 3085 Name: ptr.To("p33"), 3086 Port: ptr.To[int32](33), 3087 Protocol: ptr.To(v1.ProtocolUDP), 3088 }} 3089 } 3090 complexSubset8 := func(eps *discovery.EndpointSlice) { 3091 eps.AddressType = discovery.AddressTypeIPv4 3092 eps.Endpoints = []discovery.Endpoint{{ 3093 Addresses: []string{"4.4.4.4"}, 3094 NodeName: ptr.To(testHostname), 3095 }} 3096 eps.Ports = []discovery.EndpointPort{{ 3097 Name: ptr.To("p44"), 3098 Port: ptr.To[int32](44), 3099 Protocol: ptr.To(v1.ProtocolUDP), 3100 }} 3101 } 3102 complexBefore := []*discovery.EndpointSlice{ 3103 makeTestEndpointSlice("ns1", "ep1", 1, subset1), 3104 nil, 3105 makeTestEndpointSlice("ns2", "ep2", 1, complexSubset1), 3106 makeTestEndpointSlice("ns2", "ep2", 2, complexSubset2), 3107 nil, 3108 makeTestEndpointSlice("ns4", "ep4", 1, complexSubset3), 3109 makeTestEndpointSlice("ns4", "ep4", 2, complexSubset4), 3110 } 3111 complexAfter := []*discovery.EndpointSlice{ 3112 makeTestEndpointSlice("ns1", "ep1", 1, complexSubset5), 3113 makeTestEndpointSlice("ns1", "ep1", 2, complexSubset6), 3114 nil, 3115 nil, 3116 makeTestEndpointSlice("ns3", "ep3", 1, complexSubset7), 3117 makeTestEndpointSlice("ns4", "ep4", 1, complexSubset8), 3118 nil, 3119 } 3120 3121 testCases := []struct { 3122 // previousEndpoints and currentEndpoints are used to call appropriate 3123 // handlers OnEndpoints* (based on whether corresponding values are nil 3124 // or non-nil) and must be of equal length. 3125 name string 3126 previousEndpoints []*discovery.EndpointSlice 3127 currentEndpoints []*discovery.EndpointSlice 3128 oldEndpoints map[proxy.ServicePortName][]endpointExpectation 3129 expectedResult map[proxy.ServicePortName][]endpointExpectation 3130 expectedDeletedUDPEndpoints []proxy.ServiceEndpoint 3131 expectedNewlyActiveUDPServices map[proxy.ServicePortName]bool 3132 expectedReadyEndpoints map[types.NamespacedName]int 3133 }{{ 3134 // Case[0]: nothing 3135 name: "nothing", 3136 oldEndpoints: map[proxy.ServicePortName][]endpointExpectation{}, 3137 expectedResult: map[proxy.ServicePortName][]endpointExpectation{}, 3138 expectedDeletedUDPEndpoints: []proxy.ServiceEndpoint{}, 3139 expectedNewlyActiveUDPServices: map[proxy.ServicePortName]bool{}, 3140 expectedReadyEndpoints: map[types.NamespacedName]int{}, 3141 }, { 3142 // Case[1]: no change, named port, local 3143 name: "no change, named port, local", 3144 previousEndpoints: namedPortLocal, 3145 currentEndpoints: namedPortLocal, 3146 oldEndpoints: map[proxy.ServicePortName][]endpointExpectation{ 3147 makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): { 3148 {endpoint: "1.1.1.1:11", isLocal: true}, 3149 }, 3150 }, 3151 expectedResult: map[proxy.ServicePortName][]endpointExpectation{ 3152 makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): { 3153 {endpoint: "1.1.1.1:11", isLocal: true}, 3154 }, 3155 }, 3156 expectedDeletedUDPEndpoints: []proxy.ServiceEndpoint{}, 3157 expectedNewlyActiveUDPServices: map[proxy.ServicePortName]bool{}, 3158 expectedReadyEndpoints: map[types.NamespacedName]int{ 3159 makeNSN("ns1", "ep1"): 1, 3160 }, 3161 }, { 3162 // Case[2]: no change, multiple subsets 3163 name: "no change, multiple subsets", 3164 previousEndpoints: multipleSubsets, 3165 currentEndpoints: multipleSubsets, 3166 oldEndpoints: map[proxy.ServicePortName][]endpointExpectation{ 3167 makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): { 3168 {endpoint: "1.1.1.1:11", isLocal: false}, 3169 }, 3170 makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP): { 3171 {endpoint: "1.1.1.2:12", isLocal: false}, 3172 }, 3173 }, 3174 expectedResult: map[proxy.ServicePortName][]endpointExpectation{ 3175 makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): { 3176 {endpoint: "1.1.1.1:11", isLocal: false}, 3177 }, 3178 makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP): { 3179 {endpoint: "1.1.1.2:12", isLocal: false}, 3180 }, 3181 }, 3182 expectedDeletedUDPEndpoints: []proxy.ServiceEndpoint{}, 3183 expectedNewlyActiveUDPServices: map[proxy.ServicePortName]bool{}, 3184 expectedReadyEndpoints: map[types.NamespacedName]int{}, 3185 }, { 3186 // Case[3]: no change, multiple subsets, multiple ports, local 3187 name: "no change, multiple subsets, multiple ports, local", 3188 previousEndpoints: multipleSubsetsMultiplePortsLocal, 3189 currentEndpoints: multipleSubsetsMultiplePortsLocal, 3190 oldEndpoints: map[proxy.ServicePortName][]endpointExpectation{ 3191 makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): { 3192 {endpoint: "1.1.1.1:11", isLocal: true}, 3193 }, 3194 makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP): { 3195 {endpoint: "1.1.1.1:12", isLocal: true}, 3196 }, 3197 makeServicePortName("ns1", "ep1", "p13", v1.ProtocolUDP): { 3198 {endpoint: "1.1.1.3:13", isLocal: false}, 3199 }, 3200 }, 3201 expectedResult: map[proxy.ServicePortName][]endpointExpectation{ 3202 makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): { 3203 {endpoint: "1.1.1.1:11", isLocal: true}, 3204 }, 3205 makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP): { 3206 {endpoint: "1.1.1.1:12", isLocal: true}, 3207 }, 3208 makeServicePortName("ns1", "ep1", "p13", v1.ProtocolUDP): { 3209 {endpoint: "1.1.1.3:13", isLocal: false}, 3210 }, 3211 }, 3212 expectedDeletedUDPEndpoints: []proxy.ServiceEndpoint{}, 3213 expectedNewlyActiveUDPServices: map[proxy.ServicePortName]bool{}, 3214 expectedReadyEndpoints: map[types.NamespacedName]int{ 3215 makeNSN("ns1", "ep1"): 1, 3216 }, 3217 }, { 3218 // Case[4]: no change, multiple endpoints, subsets, IPs, and ports 3219 name: "no change, multiple endpoints, subsets, IPs, and ports", 3220 previousEndpoints: multipleSubsetsIPsPorts, 3221 currentEndpoints: multipleSubsetsIPsPorts, 3222 oldEndpoints: map[proxy.ServicePortName][]endpointExpectation{ 3223 makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): { 3224 {endpoint: "1.1.1.1:11", isLocal: false}, 3225 {endpoint: "1.1.1.2:11", isLocal: true}, 3226 }, 3227 makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP): { 3228 {endpoint: "1.1.1.1:12", isLocal: false}, 3229 {endpoint: "1.1.1.2:12", isLocal: true}, 3230 }, 3231 makeServicePortName("ns1", "ep1", "p13", v1.ProtocolUDP): { 3232 {endpoint: "1.1.1.3:13", isLocal: false}, 3233 {endpoint: "1.1.1.4:13", isLocal: true}, 3234 }, 3235 makeServicePortName("ns1", "ep1", "p14", v1.ProtocolUDP): { 3236 {endpoint: "1.1.1.3:14", isLocal: false}, 3237 {endpoint: "1.1.1.4:14", isLocal: true}, 3238 }, 3239 makeServicePortName("ns2", "ep2", "p21", v1.ProtocolUDP): { 3240 {endpoint: "2.2.2.1:21", isLocal: false}, 3241 {endpoint: "2.2.2.2:21", isLocal: true}, 3242 }, 3243 makeServicePortName("ns2", "ep2", "p22", v1.ProtocolUDP): { 3244 {endpoint: "2.2.2.1:22", isLocal: false}, 3245 {endpoint: "2.2.2.2:22", isLocal: true}, 3246 }, 3247 }, 3248 expectedResult: map[proxy.ServicePortName][]endpointExpectation{ 3249 makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): { 3250 {endpoint: "1.1.1.1:11", isLocal: false}, 3251 {endpoint: "1.1.1.2:11", isLocal: true}, 3252 }, 3253 makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP): { 3254 {endpoint: "1.1.1.1:12", isLocal: false}, 3255 {endpoint: "1.1.1.2:12", isLocal: true}, 3256 }, 3257 makeServicePortName("ns1", "ep1", "p13", v1.ProtocolUDP): { 3258 {endpoint: "1.1.1.3:13", isLocal: false}, 3259 {endpoint: "1.1.1.4:13", isLocal: true}, 3260 }, 3261 makeServicePortName("ns1", "ep1", "p14", v1.ProtocolUDP): { 3262 {endpoint: "1.1.1.3:14", isLocal: false}, 3263 {endpoint: "1.1.1.4:14", isLocal: true}, 3264 }, 3265 makeServicePortName("ns2", "ep2", "p21", v1.ProtocolUDP): { 3266 {endpoint: "2.2.2.1:21", isLocal: false}, 3267 {endpoint: "2.2.2.2:21", isLocal: true}, 3268 }, 3269 makeServicePortName("ns2", "ep2", "p22", v1.ProtocolUDP): { 3270 {endpoint: "2.2.2.1:22", isLocal: false}, 3271 {endpoint: "2.2.2.2:22", isLocal: true}, 3272 }, 3273 }, 3274 expectedDeletedUDPEndpoints: []proxy.ServiceEndpoint{}, 3275 expectedNewlyActiveUDPServices: map[proxy.ServicePortName]bool{}, 3276 expectedReadyEndpoints: map[types.NamespacedName]int{ 3277 makeNSN("ns1", "ep1"): 2, 3278 makeNSN("ns2", "ep2"): 1, 3279 }, 3280 }, { 3281 // Case[5]: add an Endpoints 3282 name: "add an Endpoints", 3283 previousEndpoints: []*discovery.EndpointSlice{nil}, 3284 currentEndpoints: namedPortLocal, 3285 oldEndpoints: map[proxy.ServicePortName][]endpointExpectation{}, 3286 expectedResult: map[proxy.ServicePortName][]endpointExpectation{ 3287 makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): { 3288 {endpoint: "1.1.1.1:11", isLocal: true}, 3289 }, 3290 }, 3291 expectedDeletedUDPEndpoints: []proxy.ServiceEndpoint{}, 3292 expectedNewlyActiveUDPServices: map[proxy.ServicePortName]bool{ 3293 makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): true, 3294 }, 3295 expectedReadyEndpoints: map[types.NamespacedName]int{ 3296 makeNSN("ns1", "ep1"): 1, 3297 }, 3298 }, { 3299 // Case[6]: remove an Endpoints 3300 name: "remove an Endpoints", 3301 previousEndpoints: namedPortLocal, 3302 currentEndpoints: []*discovery.EndpointSlice{nil}, 3303 oldEndpoints: map[proxy.ServicePortName][]endpointExpectation{ 3304 makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): { 3305 {endpoint: "1.1.1.1:11", isLocal: true}, 3306 }, 3307 }, 3308 expectedResult: map[proxy.ServicePortName][]endpointExpectation{}, 3309 expectedDeletedUDPEndpoints: []proxy.ServiceEndpoint{{ 3310 Endpoint: "1.1.1.1:11", 3311 ServicePortName: makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP), 3312 }}, 3313 expectedNewlyActiveUDPServices: map[proxy.ServicePortName]bool{}, 3314 expectedReadyEndpoints: map[types.NamespacedName]int{}, 3315 }, { 3316 // Case[7]: add an IP and port 3317 name: "add an IP and port", 3318 previousEndpoints: namedPort, 3319 currentEndpoints: namedPortsLocalNoLocal, 3320 oldEndpoints: map[proxy.ServicePortName][]endpointExpectation{ 3321 makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): { 3322 {endpoint: "1.1.1.1:11", isLocal: false}, 3323 }, 3324 }, 3325 expectedResult: map[proxy.ServicePortName][]endpointExpectation{ 3326 makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): { 3327 {endpoint: "1.1.1.1:11", isLocal: false}, 3328 {endpoint: "1.1.1.2:11", isLocal: true}, 3329 }, 3330 makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP): { 3331 {endpoint: "1.1.1.1:12", isLocal: false}, 3332 {endpoint: "1.1.1.2:12", isLocal: true}, 3333 }, 3334 }, 3335 expectedDeletedUDPEndpoints: []proxy.ServiceEndpoint{}, 3336 expectedNewlyActiveUDPServices: map[proxy.ServicePortName]bool{ 3337 makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP): true, 3338 }, 3339 expectedReadyEndpoints: map[types.NamespacedName]int{ 3340 makeNSN("ns1", "ep1"): 1, 3341 }, 3342 }, { 3343 // Case[8]: remove an IP and port 3344 name: "remove an IP and port", 3345 previousEndpoints: namedPortsLocalNoLocal, 3346 currentEndpoints: namedPort, 3347 oldEndpoints: map[proxy.ServicePortName][]endpointExpectation{ 3348 makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): { 3349 {endpoint: "1.1.1.1:11", isLocal: false}, 3350 {endpoint: "1.1.1.2:11", isLocal: true}, 3351 }, 3352 makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP): { 3353 {endpoint: "1.1.1.1:12", isLocal: false}, 3354 {endpoint: "1.1.1.2:12", isLocal: true}, 3355 }, 3356 }, 3357 expectedResult: map[proxy.ServicePortName][]endpointExpectation{ 3358 makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): { 3359 {endpoint: "1.1.1.1:11", isLocal: false}, 3360 }, 3361 }, 3362 expectedDeletedUDPEndpoints: []proxy.ServiceEndpoint{{ 3363 Endpoint: "1.1.1.2:11", 3364 ServicePortName: makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP), 3365 }, { 3366 Endpoint: "1.1.1.1:12", 3367 ServicePortName: makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP), 3368 }, { 3369 Endpoint: "1.1.1.2:12", 3370 ServicePortName: makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP), 3371 }}, 3372 expectedNewlyActiveUDPServices: map[proxy.ServicePortName]bool{}, 3373 expectedReadyEndpoints: map[types.NamespacedName]int{}, 3374 }, { 3375 // Case[9]: add a subset 3376 name: "add a subset", 3377 previousEndpoints: []*discovery.EndpointSlice{namedPort[0], nil}, 3378 currentEndpoints: multipleSubsetsWithLocal, 3379 oldEndpoints: map[proxy.ServicePortName][]endpointExpectation{ 3380 makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): { 3381 {endpoint: "1.1.1.1:11", isLocal: false}, 3382 }, 3383 }, 3384 expectedResult: map[proxy.ServicePortName][]endpointExpectation{ 3385 makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): { 3386 {endpoint: "1.1.1.1:11", isLocal: false}, 3387 }, 3388 makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP): { 3389 {endpoint: "1.1.1.2:12", isLocal: true}, 3390 }, 3391 }, 3392 expectedDeletedUDPEndpoints: []proxy.ServiceEndpoint{}, 3393 expectedNewlyActiveUDPServices: map[proxy.ServicePortName]bool{ 3394 makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP): true, 3395 }, 3396 expectedReadyEndpoints: map[types.NamespacedName]int{ 3397 makeNSN("ns1", "ep1"): 1, 3398 }, 3399 }, { 3400 // Case[10]: remove a subset 3401 name: "remove a subset", 3402 previousEndpoints: multipleSubsets, 3403 currentEndpoints: []*discovery.EndpointSlice{namedPort[0], nil}, 3404 oldEndpoints: map[proxy.ServicePortName][]endpointExpectation{ 3405 makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): { 3406 {endpoint: "1.1.1.1:11", isLocal: false}, 3407 }, 3408 makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP): { 3409 {endpoint: "1.1.1.2:12", isLocal: false}, 3410 }, 3411 }, 3412 expectedResult: map[proxy.ServicePortName][]endpointExpectation{ 3413 makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): { 3414 {endpoint: "1.1.1.1:11", isLocal: false}, 3415 }, 3416 }, 3417 expectedDeletedUDPEndpoints: []proxy.ServiceEndpoint{{ 3418 Endpoint: "1.1.1.2:12", 3419 ServicePortName: makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP), 3420 }}, 3421 expectedNewlyActiveUDPServices: map[proxy.ServicePortName]bool{}, 3422 expectedReadyEndpoints: map[types.NamespacedName]int{}, 3423 }, { 3424 // Case[11]: rename a port 3425 name: "rename a port", 3426 previousEndpoints: namedPort, 3427 currentEndpoints: namedPortRenamed, 3428 oldEndpoints: map[proxy.ServicePortName][]endpointExpectation{ 3429 makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): { 3430 {endpoint: "1.1.1.1:11", isLocal: false}, 3431 }, 3432 }, 3433 expectedResult: map[proxy.ServicePortName][]endpointExpectation{ 3434 makeServicePortName("ns1", "ep1", "p11-2", v1.ProtocolUDP): { 3435 {endpoint: "1.1.1.1:11", isLocal: false}, 3436 }, 3437 }, 3438 expectedDeletedUDPEndpoints: []proxy.ServiceEndpoint{{ 3439 Endpoint: "1.1.1.1:11", 3440 ServicePortName: makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP), 3441 }}, 3442 expectedNewlyActiveUDPServices: map[proxy.ServicePortName]bool{ 3443 makeServicePortName("ns1", "ep1", "p11-2", v1.ProtocolUDP): true, 3444 }, 3445 expectedReadyEndpoints: map[types.NamespacedName]int{}, 3446 }, { 3447 // Case[12]: renumber a port 3448 name: "renumber a port", 3449 previousEndpoints: namedPort, 3450 currentEndpoints: namedPortRenumbered, 3451 oldEndpoints: map[proxy.ServicePortName][]endpointExpectation{ 3452 makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): { 3453 {endpoint: "1.1.1.1:11", isLocal: false}, 3454 }, 3455 }, 3456 expectedResult: map[proxy.ServicePortName][]endpointExpectation{ 3457 makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): { 3458 {endpoint: "1.1.1.1:22", isLocal: false}, 3459 }, 3460 }, 3461 expectedDeletedUDPEndpoints: []proxy.ServiceEndpoint{{ 3462 Endpoint: "1.1.1.1:11", 3463 ServicePortName: makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP), 3464 }}, 3465 expectedNewlyActiveUDPServices: map[proxy.ServicePortName]bool{}, 3466 expectedReadyEndpoints: map[types.NamespacedName]int{}, 3467 }, { 3468 // Case[13]: complex add and remove 3469 name: "complex add and remove", 3470 previousEndpoints: complexBefore, 3471 currentEndpoints: complexAfter, 3472 oldEndpoints: map[proxy.ServicePortName][]endpointExpectation{ 3473 makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): { 3474 {endpoint: "1.1.1.1:11", isLocal: false}, 3475 }, 3476 makeServicePortName("ns2", "ep2", "p22", v1.ProtocolUDP): { 3477 {endpoint: "2.2.2.22:22", isLocal: true}, 3478 {endpoint: "2.2.2.2:22", isLocal: true}, 3479 }, 3480 makeServicePortName("ns2", "ep2", "p23", v1.ProtocolUDP): { 3481 {endpoint: "2.2.2.3:23", isLocal: true}, 3482 }, 3483 makeServicePortName("ns4", "ep4", "p44", v1.ProtocolUDP): { 3484 {endpoint: "4.4.4.4:44", isLocal: true}, 3485 {endpoint: "4.4.4.5:44", isLocal: true}, 3486 }, 3487 makeServicePortName("ns4", "ep4", "p45", v1.ProtocolUDP): { 3488 {endpoint: "4.4.4.6:45", isLocal: true}, 3489 }, 3490 }, 3491 expectedResult: map[proxy.ServicePortName][]endpointExpectation{ 3492 makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): { 3493 {endpoint: "1.1.1.11:11", isLocal: false}, 3494 {endpoint: "1.1.1.1:11", isLocal: false}, 3495 }, 3496 makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP): { 3497 {endpoint: "1.1.1.2:12", isLocal: false}, 3498 }, 3499 makeServicePortName("ns1", "ep1", "p122", v1.ProtocolUDP): { 3500 {endpoint: "1.1.1.2:122", isLocal: false}, 3501 }, 3502 makeServicePortName("ns3", "ep3", "p33", v1.ProtocolUDP): { 3503 {endpoint: "3.3.3.3:33", isLocal: false}, 3504 }, 3505 makeServicePortName("ns4", "ep4", "p44", v1.ProtocolUDP): { 3506 {endpoint: "4.4.4.4:44", isLocal: true}, 3507 }, 3508 }, 3509 expectedDeletedUDPEndpoints: []proxy.ServiceEndpoint{{ 3510 Endpoint: "2.2.2.2:22", 3511 ServicePortName: makeServicePortName("ns2", "ep2", "p22", v1.ProtocolUDP), 3512 }, { 3513 Endpoint: "2.2.2.22:22", 3514 ServicePortName: makeServicePortName("ns2", "ep2", "p22", v1.ProtocolUDP), 3515 }, { 3516 Endpoint: "2.2.2.3:23", 3517 ServicePortName: makeServicePortName("ns2", "ep2", "p23", v1.ProtocolUDP), 3518 }, { 3519 Endpoint: "4.4.4.5:44", 3520 ServicePortName: makeServicePortName("ns4", "ep4", "p44", v1.ProtocolUDP), 3521 }, { 3522 Endpoint: "4.4.4.6:45", 3523 ServicePortName: makeServicePortName("ns4", "ep4", "p45", v1.ProtocolUDP), 3524 }}, 3525 expectedNewlyActiveUDPServices: map[proxy.ServicePortName]bool{ 3526 makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP): true, 3527 makeServicePortName("ns1", "ep1", "p122", v1.ProtocolUDP): true, 3528 makeServicePortName("ns3", "ep3", "p33", v1.ProtocolUDP): true, 3529 }, 3530 expectedReadyEndpoints: map[types.NamespacedName]int{ 3531 makeNSN("ns4", "ep4"): 1, 3532 }, 3533 }, { 3534 // Case[14]: change from 0 endpoint address to 1 named port 3535 name: "change from 0 endpoint address to 1 named port", 3536 previousEndpoints: emptyEndpointSlices, 3537 currentEndpoints: namedPort, 3538 oldEndpoints: map[proxy.ServicePortName][]endpointExpectation{}, 3539 expectedResult: map[proxy.ServicePortName][]endpointExpectation{ 3540 makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): { 3541 {endpoint: "1.1.1.1:11", isLocal: false}, 3542 }, 3543 }, 3544 expectedDeletedUDPEndpoints: []proxy.ServiceEndpoint{}, 3545 expectedNewlyActiveUDPServices: map[proxy.ServicePortName]bool{ 3546 makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): true, 3547 }, 3548 expectedReadyEndpoints: map[types.NamespacedName]int{}, 3549 }, 3550 } 3551 3552 for tci, tc := range testCases { 3553 t.Run(tc.name, func(t *testing.T) { 3554 ipt := iptablestest.NewFake() 3555 ipvs := ipvstest.NewFake() 3556 ipset := ipsettest.NewFake(testIPSetVersion) 3557 fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol) 3558 fp.hostname = testHostname 3559 3560 // First check that after adding all previous versions of endpoints, 3561 // the fp.oldEndpoints is as we expect. 3562 for i := range tc.previousEndpoints { 3563 if tc.previousEndpoints[i] != nil { 3564 fp.OnEndpointSliceAdd(tc.previousEndpoints[i]) 3565 } 3566 } 3567 fp.endpointsMap.Update(fp.endpointsChanges) 3568 checkEndpointExpectations(t, tci, fp.endpointsMap, tc.oldEndpoints) 3569 3570 // Now let's call appropriate handlers to get to state we want to be. 3571 if len(tc.previousEndpoints) != len(tc.currentEndpoints) { 3572 t.Fatalf("[%d] different lengths of previous and current endpoints", tci) 3573 } 3574 3575 for i := range tc.previousEndpoints { 3576 prev, curr := tc.previousEndpoints[i], tc.currentEndpoints[i] 3577 switch { 3578 case prev == nil: 3579 fp.OnEndpointSliceAdd(curr) 3580 case curr == nil: 3581 fp.OnEndpointSliceDelete(prev) 3582 default: 3583 fp.OnEndpointSliceUpdate(prev, curr) 3584 } 3585 } 3586 result := fp.endpointsMap.Update(fp.endpointsChanges) 3587 newMap := fp.endpointsMap 3588 checkEndpointExpectations(t, tci, newMap, tc.expectedResult) 3589 if len(result.DeletedUDPEndpoints) != len(tc.expectedDeletedUDPEndpoints) { 3590 t.Errorf("[%d] expected %d staleEndpoints, got %d: %v", tci, len(tc.expectedDeletedUDPEndpoints), len(result.DeletedUDPEndpoints), result.DeletedUDPEndpoints) 3591 } 3592 for _, x := range tc.expectedDeletedUDPEndpoints { 3593 found := false 3594 for _, stale := range result.DeletedUDPEndpoints { 3595 if stale == x { 3596 found = true 3597 break 3598 } 3599 } 3600 if !found { 3601 t.Errorf("[%d] expected staleEndpoints[%v], but didn't find it: %v", tci, x, result.DeletedUDPEndpoints) 3602 } 3603 } 3604 if len(result.NewlyActiveUDPServices) != len(tc.expectedNewlyActiveUDPServices) { 3605 t.Errorf("[%d] expected %d staleServiceNames, got %d: %v", tci, len(tc.expectedNewlyActiveUDPServices), len(result.NewlyActiveUDPServices), result.NewlyActiveUDPServices) 3606 } 3607 for svcName := range tc.expectedNewlyActiveUDPServices { 3608 found := false 3609 for _, stale := range result.NewlyActiveUDPServices { 3610 if stale == svcName { 3611 found = true 3612 break 3613 } 3614 } 3615 if !found { 3616 t.Errorf("[%d] expected staleServiceNames[%v], but didn't find it: %v", tci, svcName, result.NewlyActiveUDPServices) 3617 } 3618 } 3619 localReadyEndpoints := fp.endpointsMap.LocalReadyEndpoints() 3620 if !reflect.DeepEqual(localReadyEndpoints, tc.expectedReadyEndpoints) { 3621 t.Errorf("[%d] expected healthchecks %v, got %v", tci, tc.expectedReadyEndpoints, localReadyEndpoints) 3622 } 3623 }) 3624 } 3625 } 3626 3627 type endpointExpectation struct { 3628 endpoint string 3629 isLocal bool 3630 } 3631 3632 func checkEndpointExpectations(t *testing.T, tci int, newMap proxy.EndpointsMap, expected map[proxy.ServicePortName][]endpointExpectation) { 3633 if len(newMap) != len(expected) { 3634 t.Errorf("[%d] expected %d results, got %d: %v", tci, len(expected), len(newMap), newMap) 3635 } 3636 for x := range expected { 3637 if len(newMap[x]) != len(expected[x]) { 3638 t.Errorf("[%d] expected %d endpoints for %v, got %d", tci, len(expected[x]), x, len(newMap[x])) 3639 } else { 3640 for i := range expected[x] { 3641 newEp := newMap[x][i] 3642 if newEp.String() != expected[x][i].endpoint || 3643 newEp.IsLocal() != expected[x][i].isLocal { 3644 t.Errorf("[%d] expected new[%v][%d] to be %v, got %v", tci, x, i, expected[x][i], newEp) 3645 } 3646 } 3647 } 3648 } 3649 } 3650 3651 func Test_syncService(t *testing.T) { 3652 testCases := []struct { 3653 oldVirtualServer *utilipvs.VirtualServer 3654 svcName string 3655 newVirtualServer *utilipvs.VirtualServer 3656 bindAddr bool 3657 alreadyBoundAddrs sets.Set[string] 3658 }{ 3659 { 3660 // case 0, old virtual server is same as new virtual server 3661 oldVirtualServer: &utilipvs.VirtualServer{ 3662 Address: netutils.ParseIPSloppy("1.2.3.4"), 3663 Protocol: string(v1.ProtocolTCP), 3664 Port: 80, 3665 Scheduler: "rr", 3666 Flags: utilipvs.FlagHashed, 3667 }, 3668 svcName: "foo", 3669 newVirtualServer: &utilipvs.VirtualServer{ 3670 Address: netutils.ParseIPSloppy("1.2.3.4"), 3671 Protocol: string(v1.ProtocolTCP), 3672 Port: 80, 3673 Scheduler: "rr", 3674 Flags: utilipvs.FlagHashed, 3675 }, 3676 bindAddr: false, 3677 alreadyBoundAddrs: nil, 3678 }, 3679 { 3680 // case 1, old virtual server is different from new virtual server 3681 oldVirtualServer: &utilipvs.VirtualServer{ 3682 Address: netutils.ParseIPSloppy("1.2.3.4"), 3683 Protocol: string(v1.ProtocolTCP), 3684 Port: 8080, 3685 Scheduler: "rr", 3686 Flags: utilipvs.FlagHashed, 3687 }, 3688 svcName: "bar", 3689 newVirtualServer: &utilipvs.VirtualServer{ 3690 Address: netutils.ParseIPSloppy("1.2.3.4"), 3691 Protocol: string(v1.ProtocolTCP), 3692 Port: 8080, 3693 Scheduler: "rr", 3694 Flags: utilipvs.FlagPersistent, 3695 }, 3696 bindAddr: false, 3697 alreadyBoundAddrs: nil, 3698 }, 3699 { 3700 // case 2, old virtual server is different from new virtual server 3701 oldVirtualServer: &utilipvs.VirtualServer{ 3702 Address: netutils.ParseIPSloppy("1.2.3.4"), 3703 Protocol: string(v1.ProtocolTCP), 3704 Port: 8080, 3705 Scheduler: "rr", 3706 Flags: utilipvs.FlagHashed, 3707 }, 3708 svcName: "bar", 3709 newVirtualServer: &utilipvs.VirtualServer{ 3710 Address: netutils.ParseIPSloppy("1.2.3.4"), 3711 Protocol: string(v1.ProtocolTCP), 3712 Port: 8080, 3713 Scheduler: "wlc", 3714 Flags: utilipvs.FlagHashed, 3715 }, 3716 bindAddr: false, 3717 alreadyBoundAddrs: nil, 3718 }, 3719 { 3720 // case 3, old virtual server is nil, and create new virtual server 3721 oldVirtualServer: nil, 3722 svcName: "baz", 3723 newVirtualServer: &utilipvs.VirtualServer{ 3724 Address: netutils.ParseIPSloppy("1.2.3.4"), 3725 Protocol: string(v1.ProtocolUDP), 3726 Port: 53, 3727 Scheduler: "rr", 3728 Flags: utilipvs.FlagHashed, 3729 }, 3730 bindAddr: true, 3731 alreadyBoundAddrs: nil, 3732 }, 3733 { 3734 // case 4, SCTP, old virtual server is same as new virtual server 3735 oldVirtualServer: &utilipvs.VirtualServer{ 3736 Address: netutils.ParseIPSloppy("1.2.3.4"), 3737 Protocol: string(v1.ProtocolSCTP), 3738 Port: 80, 3739 Scheduler: "rr", 3740 Flags: utilipvs.FlagHashed, 3741 }, 3742 svcName: "foo", 3743 newVirtualServer: &utilipvs.VirtualServer{ 3744 Address: netutils.ParseIPSloppy("1.2.3.4"), 3745 Protocol: string(v1.ProtocolSCTP), 3746 Port: 80, 3747 Scheduler: "rr", 3748 Flags: utilipvs.FlagHashed, 3749 }, 3750 bindAddr: false, 3751 alreadyBoundAddrs: nil, 3752 }, 3753 { 3754 // case 5, old virtual server is different from new virtual server 3755 oldVirtualServer: &utilipvs.VirtualServer{ 3756 Address: netutils.ParseIPSloppy("1.2.3.4"), 3757 Protocol: string(v1.ProtocolSCTP), 3758 Port: 8080, 3759 Scheduler: "rr", 3760 Flags: utilipvs.FlagHashed, 3761 }, 3762 svcName: "bar", 3763 newVirtualServer: &utilipvs.VirtualServer{ 3764 Address: netutils.ParseIPSloppy("1.2.3.4"), 3765 Protocol: string(v1.ProtocolSCTP), 3766 Port: 8080, 3767 Scheduler: "rr", 3768 Flags: utilipvs.FlagPersistent, 3769 }, 3770 bindAddr: false, 3771 alreadyBoundAddrs: nil, 3772 }, 3773 { 3774 // case 6, old virtual server is different from new virtual server 3775 oldVirtualServer: &utilipvs.VirtualServer{ 3776 Address: netutils.ParseIPSloppy("1.2.3.4"), 3777 Protocol: string(v1.ProtocolSCTP), 3778 Port: 8080, 3779 Scheduler: "rr", 3780 Flags: utilipvs.FlagHashed, 3781 }, 3782 svcName: "bar", 3783 newVirtualServer: &utilipvs.VirtualServer{ 3784 Address: netutils.ParseIPSloppy("1.2.3.4"), 3785 Protocol: string(v1.ProtocolSCTP), 3786 Port: 8080, 3787 Scheduler: "wlc", 3788 Flags: utilipvs.FlagHashed, 3789 }, 3790 bindAddr: false, 3791 alreadyBoundAddrs: nil, 3792 }, 3793 { 3794 // case 7, old virtual server is nil, and create new virtual server 3795 oldVirtualServer: nil, 3796 svcName: "baz", 3797 newVirtualServer: &utilipvs.VirtualServer{ 3798 Address: netutils.ParseIPSloppy("1.2.3.4"), 3799 Protocol: string(v1.ProtocolSCTP), 3800 Port: 53, 3801 Scheduler: "rr", 3802 Flags: utilipvs.FlagHashed, 3803 }, 3804 bindAddr: true, 3805 alreadyBoundAddrs: sets.New[string](), 3806 }, 3807 { 3808 // case 8, virtual server address already binded, skip sync 3809 oldVirtualServer: &utilipvs.VirtualServer{ 3810 Address: netutils.ParseIPSloppy("1.2.3.4"), 3811 Protocol: string(v1.ProtocolSCTP), 3812 Port: 53, 3813 Scheduler: "rr", 3814 Flags: utilipvs.FlagHashed, 3815 }, 3816 svcName: "baz", 3817 newVirtualServer: &utilipvs.VirtualServer{ 3818 Address: netutils.ParseIPSloppy("1.2.3.4"), 3819 Protocol: string(v1.ProtocolSCTP), 3820 Port: 53, 3821 Scheduler: "rr", 3822 Flags: utilipvs.FlagHashed, 3823 }, 3824 bindAddr: true, 3825 alreadyBoundAddrs: sets.New("1.2.3.4"), 3826 }, 3827 } 3828 3829 for i := range testCases { 3830 ipt := iptablestest.NewFake() 3831 ipvs := ipvstest.NewFake() 3832 ipset := ipsettest.NewFake(testIPSetVersion) 3833 proxier := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol) 3834 3835 proxier.netlinkHandle.EnsureDummyDevice(defaultDummyDevice) 3836 if testCases[i].oldVirtualServer != nil { 3837 if err := proxier.ipvs.AddVirtualServer(testCases[i].oldVirtualServer); err != nil { 3838 t.Errorf("Case [%d], unexpected add IPVS virtual server error: %v", i, err) 3839 } 3840 } 3841 if err := proxier.syncService(testCases[i].svcName, testCases[i].newVirtualServer, testCases[i].bindAddr, testCases[i].alreadyBoundAddrs); err != nil { 3842 t.Errorf("Case [%d], unexpected sync IPVS virtual server error: %v", i, err) 3843 } 3844 // check 3845 list, err := proxier.ipvs.GetVirtualServers() 3846 if err != nil { 3847 t.Errorf("Case [%d], unexpected list IPVS virtual server error: %v", i, err) 3848 } 3849 if len(list) != 1 { 3850 t.Errorf("Case [%d], expect %d virtual servers, got %d", i, 1, len(list)) 3851 continue 3852 } 3853 if !list[0].Equal(testCases[i].newVirtualServer) { 3854 t.Errorf("Case [%d], unexpected mismatch, expect: %#v, got: %#v", i, testCases[i].newVirtualServer, list[0]) 3855 } 3856 } 3857 } 3858 3859 func buildFakeProxier() (*iptablestest.FakeIPTables, *Proxier) { 3860 ipt := iptablestest.NewFake() 3861 ipvs := ipvstest.NewFake() 3862 ipset := ipsettest.NewFake(testIPSetVersion) 3863 return ipt, NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol) 3864 } 3865 3866 func getRules(ipt *iptablestest.FakeIPTables, chain utiliptables.Chain) []*iptablestest.Rule { 3867 var rules []*iptablestest.Rule 3868 3869 buf := bytes.NewBuffer(nil) 3870 _ = ipt.SaveInto(utiliptables.TableNAT, buf) 3871 _ = ipt.SaveInto(utiliptables.TableFilter, buf) 3872 lines := strings.Split(buf.String(), "\n") 3873 for _, l := range lines { 3874 if !strings.HasPrefix(l, "-A ") { 3875 continue 3876 } 3877 rule, _ := iptablestest.ParseRule(l, false) 3878 if rule != nil && rule.Chain == chain { 3879 rules = append(rules, rule) 3880 } 3881 } 3882 return rules 3883 } 3884 3885 // checkIptables to check expected iptables chain and rules. The got rules must have same number and order as the 3886 // expected rules. 3887 func checkIptables(t *testing.T, ipt *iptablestest.FakeIPTables, epIpt netlinktest.ExpectedIptablesChain) { 3888 for epChain, epRules := range epIpt { 3889 rules := getRules(ipt, utiliptables.Chain(epChain)) 3890 if len(rules) != len(epRules) { 3891 t.Errorf("Expected %d iptables rule in chain %s, got %d", len(epRules), epChain, len(rules)) 3892 continue 3893 } 3894 for i, epRule := range epRules { 3895 rule := rules[i] 3896 if rule.Jump == nil || rule.Jump.Value != epRule.JumpChain { 3897 t.Errorf("Expected MatchSet=%s JumpChain=%s, got %s", epRule.MatchSet, epRule.JumpChain, rule.Raw) 3898 } 3899 if (epRule.MatchSet == "" && rule.MatchSet != nil) || (epRule.MatchSet != "" && (rule.MatchSet == nil || rule.MatchSet.Value != epRule.MatchSet)) { 3900 t.Errorf("Expected MatchSet=%s JumpChain=%s, got %s", epRule.MatchSet, epRule.JumpChain, rule.Raw) 3901 } 3902 } 3903 } 3904 } 3905 3906 // checkIPSet to check expected ipset and entries 3907 func checkIPSet(t *testing.T, fp *Proxier, ipSet netlinktest.ExpectedIPSet) { 3908 for set, entries := range ipSet { 3909 ents, err := fp.ipset.ListEntries(set) 3910 if err != nil || len(ents) != len(entries) { 3911 t.Errorf("Check ipset entries failed for ipset: %q, expect %d, got %d", set, len(entries), len(ents)) 3912 continue 3913 } 3914 expectedEntries := []string{} 3915 for _, entry := range entries { 3916 expectedEntries = append(expectedEntries, entry.String()) 3917 } 3918 sort.Strings(ents) 3919 sort.Strings(expectedEntries) 3920 if !reflect.DeepEqual(ents, expectedEntries) { 3921 t.Errorf("Check ipset entries failed for ipset: %q", set) 3922 } 3923 } 3924 } 3925 3926 // checkIPVS to check expected ipvs service and destination 3927 func checkIPVS(t *testing.T, fp *Proxier, vs *netlinktest.ExpectedVirtualServer) { 3928 t.Helper() 3929 services, err := fp.ipvs.GetVirtualServers() 3930 if err != nil { 3931 t.Errorf("Failed to get ipvs services, err: %v", err) 3932 } 3933 if len(services) != vs.VSNum { 3934 t.Errorf("Expect %d ipvs services, got %d", vs.VSNum, len(services)) 3935 } 3936 for _, svc := range services { 3937 if svc.Address.String() == vs.IP && svc.Port == vs.Port && svc.Protocol == vs.Protocol { 3938 destinations, _ := fp.ipvs.GetRealServers(svc) 3939 if len(destinations) != len(vs.RS) { 3940 t.Errorf("Expected %d destinations, got %d destinations", len(vs.RS), len(destinations)) 3941 } 3942 if len(vs.RS) == 1 { 3943 if destinations[0].Address.String() != vs.RS[0].IP || destinations[0].Port != vs.RS[0].Port { 3944 t.Errorf("Unexpected mismatch destinations") 3945 } 3946 } 3947 } 3948 } 3949 } 3950 3951 func TestCleanLegacyService(t *testing.T) { 3952 ipt := iptablestest.NewFake() 3953 ipvs := ipvstest.NewFake() 3954 ipset := ipsettest.NewFake(testIPSetVersion) 3955 excludeCIDRs, _ := netutils.ParseCIDRs([]string{"3.3.3.0/24", "4.4.4.0/24"}) 3956 fp := NewFakeProxier(ipt, ipvs, ipset, nil, excludeCIDRs, v1.IPv4Protocol) 3957 3958 // All ipvs services that were processed in the latest sync loop. 3959 activeServices := sets.New("ipvs0", "ipvs1") 3960 // All ipvs services in the system. 3961 currentServices := map[string]*utilipvs.VirtualServer{ 3962 // Created by kube-proxy. 3963 "ipvs0": { 3964 Address: netutils.ParseIPSloppy("1.1.1.1"), 3965 Protocol: string(v1.ProtocolUDP), 3966 Port: 53, 3967 Scheduler: "rr", 3968 Flags: utilipvs.FlagHashed, 3969 }, 3970 // Created by kube-proxy. 3971 "ipvs1": { 3972 Address: netutils.ParseIPSloppy("2.2.2.2"), 3973 Protocol: string(v1.ProtocolUDP), 3974 Port: 54, 3975 Scheduler: "rr", 3976 Flags: utilipvs.FlagHashed, 3977 }, 3978 // Created by an external party. 3979 "ipvs2": { 3980 Address: netutils.ParseIPSloppy("3.3.3.3"), 3981 Protocol: string(v1.ProtocolUDP), 3982 Port: 55, 3983 Scheduler: "rr", 3984 Flags: utilipvs.FlagHashed, 3985 }, 3986 // Created by an external party. 3987 "ipvs3": { 3988 Address: netutils.ParseIPSloppy("4.4.4.4"), 3989 Protocol: string(v1.ProtocolUDP), 3990 Port: 56, 3991 Scheduler: "rr", 3992 Flags: utilipvs.FlagHashed, 3993 }, 3994 // Created by an external party. 3995 "ipvs4": { 3996 Address: netutils.ParseIPSloppy("5.5.5.5"), 3997 Protocol: string(v1.ProtocolUDP), 3998 Port: 57, 3999 Scheduler: "rr", 4000 Flags: utilipvs.FlagHashed, 4001 }, 4002 // Created by kube-proxy, but now stale. 4003 "ipvs5": { 4004 Address: netutils.ParseIPSloppy("6.6.6.6"), 4005 Protocol: string(v1.ProtocolUDP), 4006 Port: 58, 4007 Scheduler: "rr", 4008 Flags: utilipvs.FlagHashed, 4009 }, 4010 } 4011 for v := range currentServices { 4012 fp.ipvs.AddVirtualServer(currentServices[v]) 4013 } 4014 4015 fp.cleanLegacyService(activeServices, currentServices) 4016 // ipvs4 and ipvs5 should have been cleaned. 4017 remainingVirtualServers, _ := fp.ipvs.GetVirtualServers() 4018 if len(remainingVirtualServers) != 4 { 4019 t.Errorf("Expected number of remaining IPVS services after cleanup to be %v. Got %v", 4, len(remainingVirtualServers)) 4020 } 4021 for _, vs := range remainingVirtualServers { 4022 // Checking that ipvs4 and ipvs5 were removed. 4023 if vs.Port == 57 { 4024 t.Errorf("Expected ipvs4 to be removed after cleanup. It still remains") 4025 } 4026 if vs.Port == 58 { 4027 t.Errorf("Expected ipvs5 to be removed after cleanup. It still remains") 4028 } 4029 } 4030 } 4031 4032 func TestCleanLegacyServiceWithRealServers(t *testing.T) { 4033 ipt := iptablestest.NewFake() 4034 ipvs := ipvstest.NewFake() 4035 ipset := ipsettest.NewFake(testIPSetVersion) 4036 fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol) 4037 4038 // all deleted expect ipvs2 4039 activeServices := sets.New("ipvs2") 4040 // All ipvs services in the system. 4041 currentServices := map[string]*utilipvs.VirtualServer{ 4042 "ipvs0": { // deleted with real servers 4043 Address: netutils.ParseIPSloppy("1.1.1.1"), 4044 Protocol: string(v1.ProtocolUDP), 4045 Port: 53, 4046 Scheduler: "rr", 4047 Flags: utilipvs.FlagHashed, 4048 }, 4049 "ipvs1": { // deleted no real server 4050 Address: netutils.ParseIPSloppy("2.2.2.2"), 4051 Protocol: string(v1.ProtocolUDP), 4052 Port: 54, 4053 Scheduler: "rr", 4054 Flags: utilipvs.FlagHashed, 4055 }, 4056 "ipvs2": { // not deleted 4057 Address: netutils.ParseIPSloppy("3.3.3.3"), 4058 Protocol: string(v1.ProtocolUDP), 4059 Port: 54, 4060 Scheduler: "rr", 4061 Flags: utilipvs.FlagHashed, 4062 }, 4063 } 4064 4065 // "ipvs0" has a real server, but it should still be deleted since the Service is deleted 4066 realServers := map[*utilipvs.VirtualServer]*utilipvs.RealServer{ 4067 { 4068 Address: netutils.ParseIPSloppy("1.1.1.1"), 4069 Protocol: string(v1.ProtocolUDP), 4070 Port: 53, 4071 Scheduler: "rr", 4072 Flags: utilipvs.FlagHashed, 4073 }: { 4074 Address: netutils.ParseIPSloppy("10.180.0.1"), 4075 Port: uint16(53), 4076 Weight: 1, 4077 }, 4078 } 4079 4080 for v := range currentServices { 4081 fp.ipvs.AddVirtualServer(currentServices[v]) 4082 } 4083 4084 for v, r := range realServers { 4085 fp.ipvs.AddRealServer(v, r) 4086 } 4087 4088 fp.cleanLegacyService(activeServices, currentServices) 4089 remainingVirtualServers, _ := fp.ipvs.GetVirtualServers() 4090 if len(remainingVirtualServers) != 1 { 4091 t.Errorf("Expected number of remaining IPVS services after cleanup to be %v. Got %v", 1, len(remainingVirtualServers)) 4092 } 4093 4094 if remainingVirtualServers[0] != currentServices["ipvs2"] { 4095 t.Logf("actual virtual server: %v", remainingVirtualServers[0]) 4096 t.Logf("expected virtual server: %v", currentServices["ipvs0"]) 4097 t.Errorf("unexpected IPVS service") 4098 } 4099 } 4100 4101 func TestCleanLegacyRealServersExcludeCIDRs(t *testing.T) { 4102 ipt := iptablestest.NewFake() 4103 ipvs := ipvstest.NewFake() 4104 ipset := ipsettest.NewFake(testIPSetVersion) 4105 gtm := NewGracefulTerminationManager(ipvs) 4106 excludeCIDRs, _ := netutils.ParseCIDRs([]string{"4.4.4.4/32"}) 4107 fp := NewFakeProxier(ipt, ipvs, ipset, nil, excludeCIDRs, v1.IPv4Protocol) 4108 fp.gracefuldeleteManager = gtm 4109 4110 vs := &utilipvs.VirtualServer{ 4111 Address: netutils.ParseIPSloppy("4.4.4.4"), 4112 Protocol: string(v1.ProtocolUDP), 4113 Port: 56, 4114 Scheduler: "rr", 4115 Flags: utilipvs.FlagHashed, 4116 } 4117 4118 fp.ipvs.AddVirtualServer(vs) 4119 4120 rss := []*utilipvs.RealServer{ 4121 { 4122 Address: netutils.ParseIPSloppy("10.10.10.10"), 4123 Port: 56, 4124 ActiveConn: 0, 4125 InactiveConn: 0, 4126 }, 4127 { 4128 Address: netutils.ParseIPSloppy("11.11.11.11"), 4129 Port: 56, 4130 ActiveConn: 0, 4131 InactiveConn: 0, 4132 }, 4133 } 4134 for _, rs := range rss { 4135 fp.ipvs.AddRealServer(vs, rs) 4136 } 4137 4138 fp.netlinkHandle.EnsureDummyDevice(defaultDummyDevice) 4139 4140 fp.netlinkHandle.EnsureAddressBind("4.4.4.4", defaultDummyDevice) 4141 4142 fp.cleanLegacyService(nil, map[string]*utilipvs.VirtualServer{"ipvs0": vs}) 4143 4144 fp.gracefuldeleteManager.tryDeleteRs() 4145 4146 remainingRealServers, _ := fp.ipvs.GetRealServers(vs) 4147 4148 if len(remainingRealServers) != 2 { 4149 t.Errorf("Expected number of remaining IPVS real servers after cleanup should be %v. Got %v", 2, len(remainingRealServers)) 4150 } 4151 } 4152 4153 func TestCleanLegacyService6(t *testing.T) { 4154 ipt := iptablestest.NewFake() 4155 ipvs := ipvstest.NewFake() 4156 ipset := ipsettest.NewFake(testIPSetVersion) 4157 excludeCIDRs, _ := netutils.ParseCIDRs([]string{"3000::/64", "4000::/64"}) 4158 fp := NewFakeProxier(ipt, ipvs, ipset, nil, excludeCIDRs, v1.IPv6Protocol) 4159 fp.nodeIP = netutils.ParseIPSloppy("::1") 4160 4161 // All ipvs services that were processed in the latest sync loop. 4162 activeServices := sets.New("ipvs0", "ipvs1") 4163 // All ipvs services in the system. 4164 currentServices := map[string]*utilipvs.VirtualServer{ 4165 // Created by kube-proxy. 4166 "ipvs0": { 4167 Address: netutils.ParseIPSloppy("1000::1"), 4168 Protocol: string(v1.ProtocolUDP), 4169 Port: 53, 4170 Scheduler: "rr", 4171 Flags: utilipvs.FlagHashed, 4172 }, 4173 // Created by kube-proxy. 4174 "ipvs1": { 4175 Address: netutils.ParseIPSloppy("1000::2"), 4176 Protocol: string(v1.ProtocolUDP), 4177 Port: 54, 4178 Scheduler: "rr", 4179 Flags: utilipvs.FlagHashed, 4180 }, 4181 // Created by an external party. 4182 "ipvs2": { 4183 Address: netutils.ParseIPSloppy("3000::1"), 4184 Protocol: string(v1.ProtocolUDP), 4185 Port: 55, 4186 Scheduler: "rr", 4187 Flags: utilipvs.FlagHashed, 4188 }, 4189 // Created by an external party. 4190 "ipvs3": { 4191 Address: netutils.ParseIPSloppy("4000::1"), 4192 Protocol: string(v1.ProtocolUDP), 4193 Port: 56, 4194 Scheduler: "rr", 4195 Flags: utilipvs.FlagHashed, 4196 }, 4197 // Created by an external party. 4198 "ipvs4": { 4199 Address: netutils.ParseIPSloppy("5000::1"), 4200 Protocol: string(v1.ProtocolUDP), 4201 Port: 57, 4202 Scheduler: "rr", 4203 Flags: utilipvs.FlagHashed, 4204 }, 4205 // Created by kube-proxy, but now stale. 4206 "ipvs5": { 4207 Address: netutils.ParseIPSloppy("1000::6"), 4208 Protocol: string(v1.ProtocolUDP), 4209 Port: 58, 4210 Scheduler: "rr", 4211 Flags: utilipvs.FlagHashed, 4212 }, 4213 } 4214 for v := range currentServices { 4215 fp.ipvs.AddVirtualServer(currentServices[v]) 4216 } 4217 4218 fp.cleanLegacyService(activeServices, currentServices) 4219 // ipvs4 and ipvs5 should have been cleaned. 4220 remainingVirtualServers, _ := fp.ipvs.GetVirtualServers() 4221 if len(remainingVirtualServers) != 4 { 4222 t.Errorf("Expected number of remaining IPVS services after cleanup to be %v. Got %v", 4, len(remainingVirtualServers)) 4223 } 4224 for _, vs := range remainingVirtualServers { 4225 // Checking that ipvs4 and ipvs5 were removed. 4226 if vs.Port == 57 { 4227 t.Errorf("Expected ipvs4 to be removed after cleanup. It still remains") 4228 } 4229 if vs.Port == 58 { 4230 t.Errorf("Expected ipvs5 to be removed after cleanup. It still remains") 4231 } 4232 } 4233 } 4234 4235 func TestMultiPortServiceBindAddr(t *testing.T) { 4236 ipt := iptablestest.NewFake() 4237 ipvs := ipvstest.NewFake() 4238 ipset := ipsettest.NewFake(testIPSetVersion) 4239 fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol) 4240 4241 service1 := makeTestService("ns1", "svc1", func(svc *v1.Service) { 4242 svc.Spec.Type = v1.ServiceTypeClusterIP 4243 svc.Spec.ClusterIP = "172.16.55.4" 4244 svc.Spec.Ports = addTestPort(svc.Spec.Ports, "port1", "TCP", 1234, 0, 0) 4245 svc.Spec.Ports = addTestPort(svc.Spec.Ports, "port2", "TCP", 1235, 0, 0) 4246 }) 4247 service2 := makeTestService("ns1", "svc1", func(svc *v1.Service) { 4248 svc.Spec.Type = v1.ServiceTypeClusterIP 4249 svc.Spec.ClusterIP = "172.16.55.4" 4250 svc.Spec.Ports = addTestPort(svc.Spec.Ports, "port1", "TCP", 1234, 0, 0) 4251 }) 4252 service3 := makeTestService("ns1", "svc1", func(svc *v1.Service) { 4253 svc.Spec.Type = v1.ServiceTypeClusterIP 4254 svc.Spec.ClusterIP = "172.16.55.4" 4255 svc.Spec.Ports = addTestPort(svc.Spec.Ports, "port1", "TCP", 1234, 0, 0) 4256 svc.Spec.Ports = addTestPort(svc.Spec.Ports, "port2", "TCP", 1235, 0, 0) 4257 svc.Spec.Ports = addTestPort(svc.Spec.Ports, "port3", "UDP", 1236, 0, 0) 4258 }) 4259 4260 fp.servicesSynced = true 4261 4262 // first, add multi-port service1 4263 fp.OnServiceAdd(service1) 4264 fp.syncProxyRules() 4265 remainingAddrs, _ := fp.netlinkHandle.ListBindAddress(defaultDummyDevice) 4266 // should only remain address "172.16.55.4" 4267 if len(remainingAddrs) != 1 { 4268 t.Errorf("Expected number of remaining bound addrs after cleanup to be %v. Got %v", 1, len(remainingAddrs)) 4269 } 4270 if remainingAddrs[0] != "172.16.55.4" { 4271 t.Errorf("Expected remaining address should be %s, got %s", "172.16.55.4", remainingAddrs[0]) 4272 } 4273 4274 // update multi-port service1 to single-port service2 4275 fp.OnServiceUpdate(service1, service2) 4276 fp.syncProxyRules() 4277 remainingAddrs, _ = fp.netlinkHandle.ListBindAddress(defaultDummyDevice) 4278 // should still only remain address "172.16.55.4" 4279 if len(remainingAddrs) != 1 { 4280 t.Errorf("Expected number of remaining bound addrs after cleanup to be %v. Got %v", 1, len(remainingAddrs)) 4281 } else if remainingAddrs[0] != "172.16.55.4" { 4282 t.Errorf("Expected remaining address should be %s, got %s", "172.16.55.4", remainingAddrs[0]) 4283 } 4284 4285 // update single-port service2 to multi-port service3 4286 fp.OnServiceUpdate(service2, service3) 4287 fp.syncProxyRules() 4288 remainingAddrs, _ = fp.netlinkHandle.ListBindAddress(defaultDummyDevice) 4289 // should still only remain address "172.16.55.4" 4290 if len(remainingAddrs) != 1 { 4291 t.Errorf("Expected number of remaining bound addrs after cleanup to be %v. Got %v", 1, len(remainingAddrs)) 4292 } else if remainingAddrs[0] != "172.16.55.4" { 4293 t.Errorf("Expected remaining address should be %s, got %s", "172.16.55.4", remainingAddrs[0]) 4294 } 4295 4296 // delete multi-port service3 4297 fp.OnServiceDelete(service3) 4298 fp.syncProxyRules() 4299 remainingAddrs, _ = fp.netlinkHandle.ListBindAddress(defaultDummyDevice) 4300 // all addresses should be unbound 4301 if len(remainingAddrs) != 0 { 4302 t.Errorf("Expected number of remaining bound addrs after cleanup to be %v. Got %v", 0, len(remainingAddrs)) 4303 } 4304 } 4305 4306 func Test_getFirstColumn(t *testing.T) { 4307 testCases := []struct { 4308 name string 4309 fileContent string 4310 want []string 4311 wantErr bool 4312 }{ 4313 { 4314 name: "valid content", 4315 fileContent: `libiscsi_tcp 28672 1 iscsi_tcp, Live 0xffffffffc07ae000 4316 libiscsi 57344 3 ib_iser,iscsi_tcp,libiscsi_tcp, Live 0xffffffffc079a000 4317 raid10 57344 0 - Live 0xffffffffc0597000`, 4318 want: []string{"libiscsi_tcp", "libiscsi", "raid10"}, 4319 wantErr: false, 4320 }, 4321 } 4322 for _, test := range testCases { 4323 t.Run(test.name, func(t *testing.T) { 4324 got, err := getFirstColumn(strings.NewReader(test.fileContent)) 4325 if (err != nil) != test.wantErr { 4326 t.Errorf("getFirstColumn() error = %v, wantErr %v", err, test.wantErr) 4327 return 4328 } 4329 if !reflect.DeepEqual(got, test.want) { 4330 t.Errorf("getFirstColumn() = %v, want %v", got, test.want) 4331 } 4332 }) 4333 } 4334 } 4335 4336 // The majority of EndpointSlice specific tests are not ipvs specific and focus on 4337 // the shared EndpointsChangeTracker and EndpointSliceCache. This test ensures that the 4338 // ipvs proxier supports translating EndpointSlices to ipvs output. 4339 func TestEndpointSliceE2E(t *testing.T) { 4340 ipt := iptablestest.NewFake() 4341 ipvs := ipvstest.NewFake() 4342 ipset := ipsettest.NewFake(testIPSetVersion) 4343 fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol) 4344 fp.servicesSynced = true 4345 fp.endpointSlicesSynced = true 4346 4347 // Add initial service 4348 serviceName := "svc1" 4349 namespaceName := "ns1" 4350 fp.OnServiceAdd(&v1.Service{ 4351 ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: namespaceName}, 4352 Spec: v1.ServiceSpec{ 4353 ClusterIP: "172.20.1.1", 4354 Selector: map[string]string{"foo": "bar"}, 4355 Ports: []v1.ServicePort{{Name: "", TargetPort: intstr.FromInt32(80), Protocol: v1.ProtocolTCP}}, 4356 }, 4357 }) 4358 4359 // Add initial endpoint slice 4360 endpointSlice := &discovery.EndpointSlice{ 4361 ObjectMeta: metav1.ObjectMeta{ 4362 Name: fmt.Sprintf("%s-1", serviceName), 4363 Namespace: namespaceName, 4364 Labels: map[string]string{discovery.LabelServiceName: serviceName}, 4365 }, 4366 Ports: []discovery.EndpointPort{{ 4367 Name: ptr.To(""), 4368 Port: ptr.To[int32](80), 4369 Protocol: ptr.To(v1.ProtocolTCP), 4370 }}, 4371 AddressType: discovery.AddressTypeIPv4, 4372 Endpoints: []discovery.Endpoint{{ 4373 Addresses: []string{"10.0.1.1"}, 4374 Conditions: discovery.EndpointConditions{Ready: ptr.To(true)}, 4375 NodeName: ptr.To(testHostname), 4376 }, { 4377 Addresses: []string{"10.0.1.2"}, 4378 Conditions: discovery.EndpointConditions{Ready: ptr.To(true)}, 4379 NodeName: ptr.To("node2"), 4380 }, { 4381 Addresses: []string{"10.0.1.3"}, 4382 Conditions: discovery.EndpointConditions{Ready: ptr.To(true)}, 4383 NodeName: ptr.To("node3"), 4384 }, { // not ready endpoints should be ignored 4385 Addresses: []string{"10.0.1.4"}, 4386 Conditions: discovery.EndpointConditions{Ready: ptr.To(false)}, 4387 NodeName: ptr.To("node3"), 4388 }}, 4389 } 4390 4391 fp.OnEndpointSliceAdd(endpointSlice) 4392 fp.syncProxyRules() 4393 4394 // Ensure that Proxier updates ipvs appropriately after EndpointSlice update 4395 assert.NotNil(t, fp.ipsetList["KUBE-LOOP-BACK"]) 4396 activeEntries1 := fp.ipsetList["KUBE-LOOP-BACK"].activeEntries 4397 assert.Equal(t, 1, activeEntries1.Len(), "Expected 1 active entry in KUBE-LOOP-BACK") 4398 assert.Equal(t, true, activeEntries1.Has("10.0.1.1,tcp:80,10.0.1.1"), "Expected activeEntries to reference first (local) pod") 4399 virtualServers1, vsErr1 := ipvs.GetVirtualServers() 4400 assert.Nil(t, vsErr1, "Expected no error getting virtual servers") 4401 assert.Len(t, virtualServers1, 1, "Expected 1 virtual server") 4402 realServers1, rsErr1 := ipvs.GetRealServers(virtualServers1[0]) 4403 assert.Nil(t, rsErr1, "Expected no error getting real servers") 4404 assert.Len(t, realServers1, 3, "Expected 3 real servers") 4405 assert.Equal(t, realServers1[0].String(), "10.0.1.1:80") 4406 assert.Equal(t, realServers1[1].String(), "10.0.1.2:80") 4407 assert.Equal(t, realServers1[2].String(), "10.0.1.3:80") 4408 4409 fp.OnEndpointSliceDelete(endpointSlice) 4410 fp.syncProxyRules() 4411 4412 // Ensure that Proxier updates ipvs appropriately after EndpointSlice delete 4413 assert.NotNil(t, fp.ipsetList["KUBE-LOOP-BACK"]) 4414 activeEntries2 := fp.ipsetList["KUBE-LOOP-BACK"].activeEntries 4415 assert.Equal(t, 0, activeEntries2.Len(), "Expected 0 active entries in KUBE-LOOP-BACK") 4416 virtualServers2, vsErr2 := ipvs.GetVirtualServers() 4417 assert.Nil(t, vsErr2, "Expected no error getting virtual servers") 4418 assert.Len(t, virtualServers2, 1, "Expected 1 virtual server") 4419 realServers2, rsErr2 := ipvs.GetRealServers(virtualServers2[0]) 4420 assert.Nil(t, rsErr2, "Expected no error getting real servers") 4421 assert.Len(t, realServers2, 0, "Expected 0 real servers") 4422 } 4423 4424 func TestHealthCheckNodePortE2E(t *testing.T) { 4425 ipt := iptablestest.NewFake() 4426 ipvs := ipvstest.NewFake() 4427 ipset := ipsettest.NewFake(testIPSetVersion) 4428 fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol) 4429 fp.servicesSynced = true 4430 fp.endpointSlicesSynced = true 4431 4432 // Add initial service 4433 serviceName := "svc1" 4434 namespaceName := "ns1" 4435 4436 svc := v1.Service{ 4437 ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: namespaceName}, 4438 Spec: v1.ServiceSpec{ 4439 ClusterIP: "172.20.1.1", 4440 Selector: map[string]string{"foo": "bar"}, 4441 Ports: []v1.ServicePort{{Name: "", TargetPort: intstr.FromInt32(80), Protocol: v1.ProtocolTCP}}, 4442 Type: "LoadBalancer", 4443 HealthCheckNodePort: 30000, 4444 ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyLocal, 4445 }, 4446 } 4447 fp.OnServiceAdd(&svc) 4448 fp.syncProxyRules() 4449 4450 // Ensure that Proxier updates ipvs appropriately after service's being created 4451 assert.NotNil(t, fp.ipsetList["KUBE-HEALTH-CHECK-NODE-PORT"]) 4452 activeEntries1 := fp.ipsetList["KUBE-HEALTH-CHECK-NODE-PORT"].activeEntries 4453 assert.Equal(t, 1, activeEntries1.Len(), "Expected 1 active entry in KUBE-HEALTH-CHECK-NODE-PORT") 4454 assert.Equal(t, true, activeEntries1.Has("30000"), "Expected activeEntries to reference hc node port in spec") 4455 4456 // Update health check node port in the spec 4457 newSvc := svc 4458 newSvc.Spec.HealthCheckNodePort = 30001 4459 fp.OnServiceUpdate(&svc, &newSvc) 4460 fp.syncProxyRules() 4461 4462 // Ensure that Proxier updates ipvs appropriately after service's being updated 4463 assert.NotNil(t, fp.ipsetList["KUBE-HEALTH-CHECK-NODE-PORT"]) 4464 activeEntries2 := fp.ipsetList["KUBE-HEALTH-CHECK-NODE-PORT"].activeEntries 4465 assert.Equal(t, 1, activeEntries2.Len(), "Expected 1 active entry in KUBE-HEALTH-CHECK-NODE-PORT") 4466 assert.Equal(t, true, activeEntries2.Has("30001"), "Expected activeEntries to reference updated hc node port in spec") 4467 4468 fp.OnServiceDelete(&svc) 4469 fp.syncProxyRules() 4470 4471 // Ensure that Proxier updates ipvs appropriately after EndpointSlice delete 4472 assert.NotNil(t, fp.ipsetList["KUBE-HEALTH-CHECK-NODE-PORT"]) 4473 activeEntries3 := fp.ipsetList["KUBE-HEALTH-CHECK-NODE-PORT"].activeEntries 4474 assert.Equal(t, 0, activeEntries3.Len(), "Expected 0 active entries in KUBE-HEALTH-CHECK-NODE-PORT") 4475 } 4476 4477 // Test_HealthCheckNodePortWhenTerminating tests that health check node ports are not enabled when all local endpoints are terminating 4478 func Test_HealthCheckNodePortWhenTerminating(t *testing.T) { 4479 ipt := iptablestest.NewFake() 4480 ipvs := ipvstest.NewFake() 4481 ipset := ipsettest.NewFake(testIPSetVersion) 4482 fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol) 4483 fp.servicesSynced = true 4484 // fp.endpointsSynced = true 4485 fp.endpointSlicesSynced = true 4486 4487 serviceName := "svc1" 4488 namespaceName := "ns1" 4489 4490 fp.OnServiceAdd(&v1.Service{ 4491 ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: namespaceName}, 4492 Spec: v1.ServiceSpec{ 4493 ClusterIP: "172.20.1.1", 4494 Selector: map[string]string{"foo": "bar"}, 4495 Ports: []v1.ServicePort{{Name: "", TargetPort: intstr.FromInt32(80), Protocol: v1.ProtocolTCP}}, 4496 }, 4497 }) 4498 4499 endpointSlice := &discovery.EndpointSlice{ 4500 ObjectMeta: metav1.ObjectMeta{ 4501 Name: fmt.Sprintf("%s-1", serviceName), 4502 Namespace: namespaceName, 4503 Labels: map[string]string{discovery.LabelServiceName: serviceName}, 4504 }, 4505 Ports: []discovery.EndpointPort{{ 4506 Name: ptr.To(""), 4507 Port: ptr.To[int32](80), 4508 Protocol: ptr.To(v1.ProtocolTCP), 4509 }}, 4510 AddressType: discovery.AddressTypeIPv4, 4511 Endpoints: []discovery.Endpoint{{ 4512 Addresses: []string{"10.0.1.1"}, 4513 Conditions: discovery.EndpointConditions{Ready: ptr.To(true)}, 4514 NodeName: ptr.To(testHostname), 4515 }, { 4516 Addresses: []string{"10.0.1.2"}, 4517 Conditions: discovery.EndpointConditions{Ready: ptr.To(true)}, 4518 NodeName: ptr.To(testHostname), 4519 }, { 4520 Addresses: []string{"10.0.1.3"}, 4521 Conditions: discovery.EndpointConditions{Ready: ptr.To(true)}, 4522 }, { // not ready endpoints should be ignored 4523 Addresses: []string{"10.0.1.4"}, 4524 Conditions: discovery.EndpointConditions{Ready: ptr.To(false)}, 4525 NodeName: ptr.To(testHostname), 4526 }}, 4527 } 4528 4529 fp.OnEndpointSliceAdd(endpointSlice) 4530 _ = fp.endpointsMap.Update(fp.endpointsChanges) 4531 localReadyEndpoints := fp.endpointsMap.LocalReadyEndpoints() 4532 if len(localReadyEndpoints) != 1 { 4533 t.Errorf("unexpected number of health check node ports, expected 1 but got: %d", len(localReadyEndpoints)) 4534 } 4535 4536 // set all endpoints to terminating 4537 endpointSliceTerminating := &discovery.EndpointSlice{ 4538 ObjectMeta: metav1.ObjectMeta{ 4539 Name: fmt.Sprintf("%s-1", serviceName), 4540 Namespace: namespaceName, 4541 Labels: map[string]string{discovery.LabelServiceName: serviceName}, 4542 }, 4543 Ports: []discovery.EndpointPort{{ 4544 Name: ptr.To(""), 4545 Port: ptr.To[int32](80), 4546 Protocol: ptr.To(v1.ProtocolTCP), 4547 }}, 4548 AddressType: discovery.AddressTypeIPv4, 4549 Endpoints: []discovery.Endpoint{{ 4550 Addresses: []string{"10.0.1.1"}, 4551 Conditions: discovery.EndpointConditions{ 4552 Ready: ptr.To(false), 4553 Serving: ptr.To(true), 4554 Terminating: ptr.To(false), 4555 }, 4556 NodeName: ptr.To(testHostname), 4557 }, { 4558 Addresses: []string{"10.0.1.2"}, 4559 Conditions: discovery.EndpointConditions{ 4560 Ready: ptr.To(false), 4561 Serving: ptr.To(true), 4562 Terminating: ptr.To(true), 4563 }, 4564 NodeName: ptr.To(testHostname), 4565 }, { 4566 Addresses: []string{"10.0.1.3"}, 4567 Conditions: discovery.EndpointConditions{ 4568 Ready: ptr.To(false), 4569 Serving: ptr.To(true), 4570 Terminating: ptr.To(true), 4571 }, 4572 NodeName: ptr.To(testHostname), 4573 }, { // not ready endpoints should be ignored 4574 Addresses: []string{"10.0.1.4"}, 4575 Conditions: discovery.EndpointConditions{ 4576 Ready: ptr.To(false), 4577 Serving: ptr.To(false), 4578 Terminating: ptr.To(true), 4579 }, 4580 NodeName: ptr.To(testHostname), 4581 }}, 4582 } 4583 4584 fp.OnEndpointSliceUpdate(endpointSlice, endpointSliceTerminating) 4585 _ = fp.endpointsMap.Update(fp.endpointsChanges) 4586 localReadyEndpoints = fp.endpointsMap.LocalReadyEndpoints() 4587 if len(localReadyEndpoints) != 0 { 4588 t.Errorf("unexpected number of health check node ports, expected 0 but got: %d", len(localReadyEndpoints)) 4589 } 4590 } 4591 4592 func TestFilterCIDRs(t *testing.T) { 4593 var cidrList []string 4594 var cidrs []string 4595 var expected []string 4596 cidrs = filterCIDRs(true, []string{}) 4597 if len(cidrs) > 0 { 4598 t.Errorf("An empty list produces a non-empty return %v", cidrs) 4599 } 4600 4601 cidrList = []string{"1000::/64", "10.0.0.0/16", "11.0.0.0/16", "2000::/64"} 4602 expected = []string{"1000::/64", "2000::/64"} 4603 cidrs = filterCIDRs(true, cidrList) 4604 if !reflect.DeepEqual(cidrs, expected) { 4605 t.Errorf("cidrs %v is not expected %v", cidrs, expected) 4606 } 4607 4608 expected = []string{"10.0.0.0/16", "11.0.0.0/16"} 4609 cidrs = filterCIDRs(false, cidrList) 4610 if !reflect.DeepEqual(cidrs, expected) { 4611 t.Errorf("cidrs %v is not expected %v", cidrs, expected) 4612 } 4613 4614 cidrList = []string{"1000::/64", "2000::/64"} 4615 expected = []string{} 4616 cidrs = filterCIDRs(false, cidrList) 4617 if len(cidrs) > 0 { 4618 t.Errorf("cidrs %v is not expected %v", cidrs, expected) 4619 } 4620 } 4621 4622 func TestCreateAndLinkKubeChain(t *testing.T) { 4623 ipt := iptablestest.NewFake() 4624 ipvs := ipvstest.NewFake() 4625 ipset := ipsettest.NewFake(testIPSetVersion) 4626 fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol) 4627 fp.createAndLinkKubeChain() 4628 expectedNATChains := `:KUBE-SERVICES - [0:0] 4629 :KUBE-POSTROUTING - [0:0] 4630 :KUBE-NODE-PORT - [0:0] 4631 :KUBE-LOAD-BALANCER - [0:0] 4632 :KUBE-MARK-MASQ - [0:0] 4633 ` 4634 expectedFilterChains := `:KUBE-FORWARD - [0:0] 4635 :KUBE-NODE-PORT - [0:0] 4636 :KUBE-PROXY-FIREWALL - [0:0] 4637 :KUBE-SOURCE-RANGES-FIREWALL - [0:0] 4638 :KUBE-IPVS-FILTER - [0:0] 4639 :KUBE-IPVS-OUT-FILTER - [0:0] 4640 ` 4641 assert.Equal(t, expectedNATChains, fp.natChains.String()) 4642 assert.Equal(t, expectedFilterChains, fp.filterChains.String()) 4643 } 4644 4645 // This test ensures that the iptables proxier supports translating Endpoints to 4646 // iptables output when internalTrafficPolicy is specified 4647 func TestTestInternalTrafficPolicyE2E(t *testing.T) { 4648 type endpoint struct { 4649 ip string 4650 hostname string 4651 } 4652 4653 testCases := []struct { 4654 name string 4655 internalTrafficPolicy *v1.ServiceInternalTrafficPolicy 4656 endpoints []endpoint 4657 expectVirtualServer bool 4658 expectLocalEntries bool 4659 expectLocalRealServerNum int 4660 expectLocalRealServers []string 4661 }{ 4662 { 4663 name: "internalTrafficPolicy is cluster with non-zero local endpoints", 4664 internalTrafficPolicy: ptr.To(v1.ServiceInternalTrafficPolicyCluster), 4665 endpoints: []endpoint{ 4666 {"10.0.1.1", testHostname}, 4667 {"10.0.1.2", "host1"}, 4668 {"10.0.1.3", "host2"}, 4669 }, 4670 expectVirtualServer: true, 4671 expectLocalEntries: true, 4672 expectLocalRealServerNum: 3, 4673 expectLocalRealServers: []string{ 4674 "10.0.1.1:80", 4675 "10.0.1.2:80", 4676 "10.0.1.3:80", 4677 }, 4678 }, 4679 { 4680 name: "internalTrafficPolicy is cluster with zero local endpoints", 4681 internalTrafficPolicy: ptr.To(v1.ServiceInternalTrafficPolicyCluster), 4682 endpoints: []endpoint{ 4683 {"10.0.1.1", "host0"}, 4684 {"10.0.1.2", "host1"}, 4685 {"10.0.1.3", "host2"}, 4686 }, 4687 expectVirtualServer: false, 4688 expectLocalEntries: false, 4689 expectLocalRealServerNum: 3, 4690 expectLocalRealServers: []string{ 4691 "10.0.1.1:80", 4692 "10.0.1.2:80", 4693 "10.0.1.3:80", 4694 }, 4695 }, 4696 { 4697 name: "internalTrafficPolicy is local with non-zero local endpoints", 4698 internalTrafficPolicy: ptr.To(v1.ServiceInternalTrafficPolicyLocal), 4699 endpoints: []endpoint{ 4700 {"10.0.1.1", testHostname}, 4701 {"10.0.1.2", "host1"}, 4702 {"10.0.1.3", "host2"}, 4703 }, 4704 expectVirtualServer: true, 4705 expectLocalEntries: true, 4706 expectLocalRealServerNum: 1, 4707 expectLocalRealServers: []string{ 4708 "10.0.1.1:80", 4709 }, 4710 }, 4711 { 4712 name: "internalTrafficPolicy is local with zero local endpoints", 4713 internalTrafficPolicy: ptr.To(v1.ServiceInternalTrafficPolicyLocal), 4714 endpoints: []endpoint{ 4715 {"10.0.1.1", "host0"}, 4716 {"10.0.1.2", "host1"}, 4717 {"10.0.1.3", "host2"}, 4718 }, 4719 expectVirtualServer: false, 4720 expectLocalEntries: false, 4721 expectLocalRealServerNum: 0, 4722 expectLocalRealServers: []string{}, 4723 }, 4724 } 4725 for _, tc := range testCases { 4726 ipt := iptablestest.NewFake() 4727 ipvs := ipvstest.NewFake() 4728 ipset := ipsettest.NewFake(testIPSetVersion) 4729 fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol) 4730 fp.servicesSynced = true 4731 // fp.endpointsSynced = true 4732 fp.endpointSlicesSynced = true 4733 4734 // Add initial service 4735 serviceName := "svc1" 4736 namespaceName := "ns1" 4737 4738 svc := &v1.Service{ 4739 ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: namespaceName}, 4740 Spec: v1.ServiceSpec{ 4741 ClusterIP: "172.20.1.1", 4742 Selector: map[string]string{"foo": "bar"}, 4743 Ports: []v1.ServicePort{{Name: "", TargetPort: intstr.FromInt32(80), Protocol: v1.ProtocolTCP}}, 4744 }, 4745 } 4746 if tc.internalTrafficPolicy != nil { 4747 svc.Spec.InternalTrafficPolicy = tc.internalTrafficPolicy 4748 } 4749 4750 fp.OnServiceAdd(svc) 4751 4752 // Add initial endpoint slice 4753 endpointSlice := &discovery.EndpointSlice{ 4754 ObjectMeta: metav1.ObjectMeta{ 4755 Name: fmt.Sprintf("%s-1", serviceName), 4756 Namespace: namespaceName, 4757 Labels: map[string]string{discovery.LabelServiceName: serviceName}, 4758 }, 4759 Ports: []discovery.EndpointPort{{ 4760 Name: ptr.To(""), 4761 Port: ptr.To[int32](80), 4762 Protocol: ptr.To(v1.ProtocolTCP), 4763 }}, 4764 AddressType: discovery.AddressTypeIPv4, 4765 } 4766 4767 for _, ep := range tc.endpoints { 4768 endpointSlice.Endpoints = append(endpointSlice.Endpoints, discovery.Endpoint{ 4769 Addresses: []string{ep.ip}, 4770 Conditions: discovery.EndpointConditions{Ready: ptr.To(true)}, 4771 NodeName: ptr.To(ep.hostname), 4772 }) 4773 } 4774 4775 fp.OnEndpointSliceAdd(endpointSlice) 4776 fp.syncProxyRules() 4777 4778 // Ensure that Proxier updates ipvs appropriately after EndpointSlice update 4779 assert.NotNil(t, fp.ipsetList["KUBE-LOOP-BACK"]) 4780 4781 activeEntries1 := fp.ipsetList["KUBE-LOOP-BACK"].activeEntries 4782 4783 if tc.expectLocalEntries { 4784 assert.Equal(t, 1, activeEntries1.Len(), "Expected 1 active entry in KUBE-LOOP-BACK") 4785 } else { 4786 assert.Equal(t, 0, activeEntries1.Len(), "Expected no active entry in KUBE-LOOP-BACK") 4787 } 4788 4789 if tc.expectVirtualServer { 4790 virtualServers1, vsErr1 := ipvs.GetVirtualServers() 4791 assert.Nil(t, vsErr1, "Expected no error getting virtual servers") 4792 4793 assert.Len(t, virtualServers1, 1, "Expected 1 virtual server") 4794 realServers1, rsErr1 := ipvs.GetRealServers(virtualServers1[0]) 4795 assert.Nil(t, rsErr1, "Expected no error getting real servers") 4796 4797 assert.Len(t, realServers1, tc.expectLocalRealServerNum, fmt.Sprintf("Expected %d real servers", tc.expectLocalRealServerNum)) 4798 for i := 0; i < tc.expectLocalRealServerNum; i++ { 4799 assert.Equal(t, realServers1[i].String(), tc.expectLocalRealServers[i]) 4800 } 4801 } 4802 4803 fp.OnEndpointSliceDelete(endpointSlice) 4804 fp.syncProxyRules() 4805 4806 // Ensure that Proxier updates ipvs appropriately after EndpointSlice delete 4807 assert.NotNil(t, fp.ipsetList["KUBE-LOOP-BACK"]) 4808 activeEntries3 := fp.ipsetList["KUBE-LOOP-BACK"].activeEntries 4809 assert.Equal(t, 0, activeEntries3.Len(), "Expected 0 active entries in KUBE-LOOP-BACK") 4810 virtualServers2, vsErr2 := ipvs.GetVirtualServers() 4811 assert.Nil(t, vsErr2, "Expected no error getting virtual servers") 4812 assert.Len(t, virtualServers2, 1, "Expected 1 virtual server") 4813 realServers2, rsErr2 := ipvs.GetRealServers(virtualServers2[0]) 4814 assert.Nil(t, rsErr2, "Expected no error getting real servers") 4815 assert.Len(t, realServers2, 0, "Expected 0 real servers") 4816 } 4817 } 4818 4819 // Test_EndpointSliceReadyAndTerminatingCluster tests that when there are ready and ready + terminating 4820 // endpoints and the traffic policy is "Cluster", only the ready endpoints are used. 4821 func Test_EndpointSliceReadyAndTerminatingCluster(t *testing.T) { 4822 4823 ipt := iptablestest.NewFake() 4824 ipvs := ipvstest.NewFake() 4825 ipset := ipsettest.NewFake(testIPSetVersion) 4826 fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol) 4827 fp.servicesSynced = true 4828 // fp.endpointsSynced = true 4829 fp.endpointSlicesSynced = true 4830 4831 serviceName := "svc1" 4832 // Add initial service 4833 namespaceName := "ns1" 4834 fp.OnServiceAdd(&v1.Service{ 4835 ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: namespaceName}, 4836 Spec: v1.ServiceSpec{ 4837 ClusterIP: "172.20.1.1", 4838 Selector: map[string]string{"foo": "bar"}, 4839 Type: v1.ServiceTypeNodePort, 4840 ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyCluster, 4841 InternalTrafficPolicy: ptr.To(v1.ServiceInternalTrafficPolicyCluster), 4842 ExternalIPs: []string{ 4843 "1.2.3.4", 4844 }, 4845 Ports: []v1.ServicePort{ 4846 { 4847 Name: "", 4848 Port: 80, 4849 TargetPort: intstr.FromInt32(80), 4850 Protocol: v1.ProtocolTCP, 4851 }, 4852 }, 4853 }, 4854 }) 4855 4856 // Add initial endpoint slice 4857 endpointSlice := &discovery.EndpointSlice{ 4858 ObjectMeta: metav1.ObjectMeta{ 4859 Name: fmt.Sprintf("%s-1", serviceName), 4860 Namespace: namespaceName, 4861 Labels: map[string]string{discovery.LabelServiceName: serviceName}, 4862 }, 4863 Ports: []discovery.EndpointPort{{ 4864 Name: ptr.To(""), 4865 Port: ptr.To[int32](80), 4866 Protocol: ptr.To(v1.ProtocolTCP), 4867 }}, 4868 AddressType: discovery.AddressTypeIPv4, 4869 Endpoints: []discovery.Endpoint{ 4870 { 4871 Addresses: []string{"10.0.1.1"}, 4872 Conditions: discovery.EndpointConditions{ 4873 Ready: ptr.To(true), 4874 Serving: ptr.To(true), 4875 Terminating: ptr.To(false), 4876 }, 4877 NodeName: ptr.To(testHostname), 4878 }, 4879 { 4880 Addresses: []string{"10.0.1.2"}, 4881 Conditions: discovery.EndpointConditions{ 4882 Ready: ptr.To(true), 4883 Serving: ptr.To(true), 4884 Terminating: ptr.To(false), 4885 }, 4886 NodeName: ptr.To(testHostname), 4887 }, 4888 { 4889 Addresses: []string{"10.0.1.3"}, 4890 Conditions: discovery.EndpointConditions{ 4891 Ready: ptr.To(false), 4892 Serving: ptr.To(true), 4893 Terminating: ptr.To(true), 4894 }, 4895 NodeName: ptr.To(testHostname), 4896 }, 4897 { 4898 Addresses: []string{"10.0.1.4"}, 4899 Conditions: discovery.EndpointConditions{ 4900 Ready: ptr.To(false), 4901 Serving: ptr.To(false), 4902 Terminating: ptr.To(true), 4903 }, 4904 NodeName: ptr.To(testHostname), 4905 }, 4906 { 4907 Addresses: []string{"10.0.1.5"}, 4908 Conditions: discovery.EndpointConditions{ 4909 Ready: ptr.To(true), 4910 Serving: ptr.To(true), 4911 Terminating: ptr.To(false), 4912 }, 4913 NodeName: ptr.To("another-host"), 4914 }, 4915 }, 4916 } 4917 4918 fp.OnEndpointSliceAdd(endpointSlice) 4919 fp.syncProxyRules() 4920 4921 // Ensure that Proxier updates ipvs appropriately after EndpointSlice update 4922 assert.NotNil(t, fp.ipsetList["KUBE-LOOP-BACK"]) 4923 activeEntries1 := fp.ipsetList["KUBE-LOOP-BACK"].activeEntries 4924 assert.Equal(t, 4, activeEntries1.Len(), "Expected 4 active entry in KUBE-LOOP-BACK") 4925 assert.Equal(t, true, activeEntries1.Has("10.0.1.1,tcp:80,10.0.1.1"), "Expected activeEntries to reference first pod") 4926 assert.Equal(t, true, activeEntries1.Has("10.0.1.2,tcp:80,10.0.1.2"), "Expected activeEntries to reference second pod") 4927 assert.Equal(t, true, activeEntries1.Has("10.0.1.3,tcp:80,10.0.1.3"), "Expected activeEntries to reference third pod") 4928 assert.Equal(t, true, activeEntries1.Has("10.0.1.4,tcp:80,10.0.1.4"), "Expected activeEntries to reference fourth pod") 4929 4930 virtualServers, vsErr := ipvs.GetVirtualServers() 4931 assert.Nil(t, vsErr, "Expected no error getting virtual servers") 4932 assert.Len(t, virtualServers, 2, "Expected 2 virtual server") 4933 4934 var clusterIPServer, externalIPServer *utilipvs.VirtualServer 4935 for _, virtualServer := range virtualServers { 4936 if virtualServer.Address.String() == "172.20.1.1" { 4937 clusterIPServer = virtualServer 4938 } 4939 4940 if virtualServer.Address.String() == "1.2.3.4" { 4941 externalIPServer = virtualServer 4942 } 4943 } 4944 4945 // clusterIP should route to cluster-wide ready endpoints 4946 realServers1, rsErr1 := ipvs.GetRealServers(clusterIPServer) 4947 assert.Nil(t, rsErr1, "Expected no error getting real servers") 4948 assert.Len(t, realServers1, 3, "Expected 3 real servers") 4949 assert.Equal(t, realServers1[0].String(), "10.0.1.1:80") 4950 assert.Equal(t, realServers1[1].String(), "10.0.1.2:80") 4951 assert.Equal(t, realServers1[2].String(), "10.0.1.5:80") 4952 4953 // externalIP should route to cluster-wide ready endpoints 4954 realServers2, rsErr2 := ipvs.GetRealServers(externalIPServer) 4955 assert.Nil(t, rsErr2, "Expected no error getting real servers") 4956 assert.Len(t, realServers2, 3, "Expected 3 real servers") 4957 assert.Equal(t, realServers2[0].String(), "10.0.1.1:80") 4958 assert.Equal(t, realServers2[1].String(), "10.0.1.2:80") 4959 assert.Equal(t, realServers1[2].String(), "10.0.1.5:80") 4960 4961 fp.OnEndpointSliceDelete(endpointSlice) 4962 fp.syncProxyRules() 4963 4964 // Ensure that Proxier updates ipvs appropriately after EndpointSlice delete 4965 assert.NotNil(t, fp.ipsetList["KUBE-LOOP-BACK"]) 4966 activeEntries2 := fp.ipsetList["KUBE-LOOP-BACK"].activeEntries 4967 assert.Equal(t, 0, activeEntries2.Len(), "Expected 0 active entries in KUBE-LOOP-BACK") 4968 4969 virtualServers, vsErr = ipvs.GetVirtualServers() 4970 assert.Nil(t, vsErr, "Expected no error getting virtual servers") 4971 assert.Len(t, virtualServers, 2, "Expected 2 virtual server") 4972 4973 for _, virtualServer := range virtualServers { 4974 if virtualServer.Address.String() == "172.20.1.1" { 4975 clusterIPServer = virtualServer 4976 } 4977 4978 if virtualServer.Address.String() == "1.2.3.4" { 4979 externalIPServer = virtualServer 4980 } 4981 } 4982 4983 realServers1, rsErr1 = ipvs.GetRealServers(clusterIPServer) 4984 assert.Nil(t, rsErr1, "Expected no error getting real servers") 4985 assert.Len(t, realServers1, 0, "Expected 0 real servers") 4986 4987 realServers2, rsErr2 = ipvs.GetRealServers(externalIPServer) 4988 assert.Nil(t, rsErr2, "Expected no error getting real servers") 4989 assert.Len(t, realServers2, 0, "Expected 0 real servers") 4990 } 4991 4992 // Test_EndpointSliceReadyAndTerminatingLocal tests that when there are local ready and ready + terminating 4993 // endpoints, only the ready endpoints are used. 4994 func Test_EndpointSliceReadyAndTerminatingLocal(t *testing.T) { 4995 4996 ipt := iptablestest.NewFake() 4997 ipvs := ipvstest.NewFake() 4998 ipset := ipsettest.NewFake(testIPSetVersion) 4999 fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol) 5000 fp.servicesSynced = true 5001 // fp.endpointsSynced = true 5002 fp.endpointSlicesSynced = true 5003 5004 serviceName := "svc1" 5005 // Add initial service 5006 namespaceName := "ns1" 5007 fp.OnServiceAdd(&v1.Service{ 5008 ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: namespaceName}, 5009 Spec: v1.ServiceSpec{ 5010 ClusterIP: "172.20.1.1", 5011 Selector: map[string]string{"foo": "bar"}, 5012 Type: v1.ServiceTypeNodePort, 5013 ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyLocal, 5014 InternalTrafficPolicy: ptr.To(v1.ServiceInternalTrafficPolicyCluster), 5015 ExternalIPs: []string{ 5016 "1.2.3.4", 5017 }, 5018 Ports: []v1.ServicePort{ 5019 { 5020 Name: "", 5021 Port: 80, 5022 TargetPort: intstr.FromInt32(80), 5023 Protocol: v1.ProtocolTCP, 5024 }, 5025 }, 5026 }, 5027 }) 5028 5029 // Add initial endpoint slice 5030 endpointSlice := &discovery.EndpointSlice{ 5031 ObjectMeta: metav1.ObjectMeta{ 5032 Name: fmt.Sprintf("%s-1", serviceName), 5033 Namespace: namespaceName, 5034 Labels: map[string]string{discovery.LabelServiceName: serviceName}, 5035 }, 5036 Ports: []discovery.EndpointPort{{ 5037 Name: ptr.To(""), 5038 Port: ptr.To[int32](80), 5039 Protocol: ptr.To(v1.ProtocolTCP), 5040 }}, 5041 AddressType: discovery.AddressTypeIPv4, 5042 Endpoints: []discovery.Endpoint{ 5043 { 5044 Addresses: []string{"10.0.1.1"}, 5045 Conditions: discovery.EndpointConditions{ 5046 Ready: ptr.To(true), 5047 Serving: ptr.To(true), 5048 Terminating: ptr.To(false), 5049 }, 5050 NodeName: ptr.To(testHostname), 5051 }, 5052 { 5053 Addresses: []string{"10.0.1.2"}, 5054 Conditions: discovery.EndpointConditions{ 5055 Ready: ptr.To(true), 5056 Serving: ptr.To(true), 5057 Terminating: ptr.To(false), 5058 }, 5059 NodeName: ptr.To(testHostname), 5060 }, 5061 { 5062 Addresses: []string{"10.0.1.3"}, 5063 Conditions: discovery.EndpointConditions{ 5064 Ready: ptr.To(false), 5065 Serving: ptr.To(true), 5066 Terminating: ptr.To(true), 5067 }, 5068 NodeName: ptr.To(testHostname), 5069 }, 5070 { 5071 Addresses: []string{"10.0.1.4"}, 5072 Conditions: discovery.EndpointConditions{ 5073 Ready: ptr.To(false), 5074 Serving: ptr.To(false), 5075 Terminating: ptr.To(true), 5076 }, 5077 NodeName: ptr.To(testHostname), 5078 }, 5079 { 5080 Addresses: []string{"10.0.1.5"}, 5081 Conditions: discovery.EndpointConditions{ 5082 Ready: ptr.To(true), 5083 Serving: ptr.To(true), 5084 Terminating: ptr.To(false), 5085 }, 5086 NodeName: ptr.To("another-host"), 5087 }, 5088 }, 5089 } 5090 5091 fp.OnEndpointSliceAdd(endpointSlice) 5092 fp.syncProxyRules() 5093 5094 // Ensure that Proxier updates ipvs appropriately after EndpointSlice update 5095 assert.NotNil(t, fp.ipsetList["KUBE-LOOP-BACK"]) 5096 activeEntries1 := fp.ipsetList["KUBE-LOOP-BACK"].activeEntries 5097 assert.Equal(t, 4, activeEntries1.Len(), "Expected 3 active entry in KUBE-LOOP-BACK") 5098 assert.Equal(t, true, activeEntries1.Has("10.0.1.1,tcp:80,10.0.1.1"), "Expected activeEntries to reference first (local) pod") 5099 assert.Equal(t, true, activeEntries1.Has("10.0.1.2,tcp:80,10.0.1.2"), "Expected activeEntries to reference second (local) pod") 5100 assert.Equal(t, true, activeEntries1.Has("10.0.1.3,tcp:80,10.0.1.3"), "Expected activeEntries to reference second (local) pod") 5101 assert.Equal(t, true, activeEntries1.Has("10.0.1.4,tcp:80,10.0.1.4"), "Expected activeEntries to reference second (local) pod") 5102 5103 virtualServers, vsErr := ipvs.GetVirtualServers() 5104 assert.Nil(t, vsErr, "Expected no error getting virtual servers") 5105 assert.Len(t, virtualServers, 2, "Expected 2 virtual server") 5106 5107 var clusterIPServer, externalIPServer *utilipvs.VirtualServer 5108 for _, virtualServer := range virtualServers { 5109 if virtualServer.Address.String() == "172.20.1.1" { 5110 clusterIPServer = virtualServer 5111 } 5112 5113 if virtualServer.Address.String() == "1.2.3.4" { 5114 externalIPServer = virtualServer 5115 } 5116 } 5117 5118 // clusterIP should route to cluster-wide ready endpoints 5119 realServers1, rsErr1 := ipvs.GetRealServers(clusterIPServer) 5120 assert.Nil(t, rsErr1, "Expected no error getting real servers") 5121 assert.Len(t, realServers1, 3, "Expected 3 real servers") 5122 assert.Equal(t, realServers1[0].String(), "10.0.1.1:80") 5123 assert.Equal(t, realServers1[1].String(), "10.0.1.2:80") 5124 assert.Equal(t, realServers1[2].String(), "10.0.1.5:80") 5125 5126 // externalIP should route to local ready + non-terminating endpoints if they exist 5127 realServers2, rsErr2 := ipvs.GetRealServers(externalIPServer) 5128 assert.Nil(t, rsErr2, "Expected no error getting real servers") 5129 assert.Len(t, realServers2, 2, "Expected 2 real servers") 5130 assert.Equal(t, realServers2[0].String(), "10.0.1.1:80") 5131 assert.Equal(t, realServers2[1].String(), "10.0.1.2:80") 5132 5133 fp.OnEndpointSliceDelete(endpointSlice) 5134 fp.syncProxyRules() 5135 5136 // Ensure that Proxier updates ipvs appropriately after EndpointSlice delete 5137 assert.NotNil(t, fp.ipsetList["KUBE-LOOP-BACK"]) 5138 activeEntries2 := fp.ipsetList["KUBE-LOOP-BACK"].activeEntries 5139 assert.Equal(t, 0, activeEntries2.Len(), "Expected 0 active entries in KUBE-LOOP-BACK") 5140 5141 virtualServers, vsErr = ipvs.GetVirtualServers() 5142 assert.Nil(t, vsErr, "Expected no error getting virtual servers") 5143 assert.Len(t, virtualServers, 2, "Expected 2 virtual server") 5144 5145 for _, virtualServer := range virtualServers { 5146 if virtualServer.Address.String() == "172.20.1.1" { 5147 clusterIPServer = virtualServer 5148 } 5149 5150 if virtualServer.Address.String() == "1.2.3.4" { 5151 externalIPServer = virtualServer 5152 } 5153 } 5154 5155 realServers1, rsErr1 = ipvs.GetRealServers(clusterIPServer) 5156 assert.Nil(t, rsErr1, "Expected no error getting real servers") 5157 assert.Len(t, realServers1, 0, "Expected 0 real servers") 5158 5159 realServers2, rsErr2 = ipvs.GetRealServers(externalIPServer) 5160 assert.Nil(t, rsErr2, "Expected no error getting real servers") 5161 assert.Len(t, realServers2, 0, "Expected 0 real servers") 5162 } 5163 5164 // Test_EndpointSliceOnlyReadyTerminatingCluster tests that when there are only ready terminating 5165 // endpoints and the traffic policy is "Cluster", we fall back to terminating endpoints. 5166 func Test_EndpointSliceOnlyReadyAndTerminatingCluster(t *testing.T) { 5167 5168 ipt := iptablestest.NewFake() 5169 ipvs := ipvstest.NewFake() 5170 ipset := ipsettest.NewFake(testIPSetVersion) 5171 fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol) 5172 fp.servicesSynced = true 5173 // fp.endpointsSynced = true 5174 fp.endpointSlicesSynced = true 5175 5176 // Add initial service 5177 serviceName := "svc1" 5178 namespaceName := "ns1" 5179 fp.OnServiceAdd(&v1.Service{ 5180 ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: namespaceName}, 5181 Spec: v1.ServiceSpec{ 5182 ClusterIP: "172.20.1.1", 5183 Selector: map[string]string{"foo": "bar"}, 5184 Type: v1.ServiceTypeNodePort, 5185 ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyCluster, 5186 InternalTrafficPolicy: ptr.To(v1.ServiceInternalTrafficPolicyCluster), 5187 ExternalIPs: []string{ 5188 "1.2.3.4", 5189 }, 5190 Ports: []v1.ServicePort{ 5191 { 5192 Name: "", 5193 Port: 80, 5194 TargetPort: intstr.FromInt32(80), 5195 Protocol: v1.ProtocolTCP, 5196 }, 5197 }, 5198 }, 5199 }) 5200 5201 // Add initial endpoint slice 5202 endpointSlice := &discovery.EndpointSlice{ 5203 ObjectMeta: metav1.ObjectMeta{ 5204 Name: fmt.Sprintf("%s-1", serviceName), 5205 Namespace: namespaceName, 5206 Labels: map[string]string{discovery.LabelServiceName: serviceName}, 5207 }, 5208 Ports: []discovery.EndpointPort{{ 5209 Name: ptr.To(""), 5210 Port: ptr.To[int32](80), 5211 Protocol: ptr.To(v1.ProtocolTCP), 5212 }}, 5213 AddressType: discovery.AddressTypeIPv4, 5214 Endpoints: []discovery.Endpoint{ 5215 { 5216 Addresses: []string{"10.0.1.1"}, 5217 Conditions: discovery.EndpointConditions{ 5218 Ready: ptr.To(false), 5219 Serving: ptr.To(true), 5220 Terminating: ptr.To(true), 5221 }, 5222 NodeName: ptr.To(testHostname), 5223 }, 5224 { 5225 Addresses: []string{"10.0.1.2"}, 5226 Conditions: discovery.EndpointConditions{ 5227 Ready: ptr.To(false), 5228 Serving: ptr.To(true), 5229 Terminating: ptr.To(true), 5230 }, 5231 NodeName: ptr.To(testHostname), 5232 }, 5233 { 5234 Addresses: []string{"10.0.1.3"}, 5235 Conditions: discovery.EndpointConditions{ 5236 Ready: ptr.To(false), 5237 Serving: ptr.To(false), 5238 Terminating: ptr.To(true), 5239 }, 5240 NodeName: ptr.To(testHostname), 5241 }, 5242 { 5243 Addresses: []string{"10.0.1.4"}, 5244 Conditions: discovery.EndpointConditions{ 5245 Ready: ptr.To(false), 5246 Serving: ptr.To(true), 5247 Terminating: ptr.To(true), 5248 }, 5249 NodeName: ptr.To("another-host"), 5250 }, 5251 { 5252 Addresses: []string{"10.0.1.5"}, 5253 Conditions: discovery.EndpointConditions{ 5254 Ready: ptr.To(false), 5255 Serving: ptr.To(false), 5256 Terminating: ptr.To(false), 5257 }, 5258 NodeName: ptr.To("another-host"), 5259 }, 5260 }, 5261 } 5262 5263 fp.OnEndpointSliceAdd(endpointSlice) 5264 fp.syncProxyRules() 5265 5266 // Ensure that Proxier updates ipvs appropriately after EndpointSlice update 5267 assert.NotNil(t, fp.ipsetList["KUBE-LOOP-BACK"]) 5268 activeEntries1 := fp.ipsetList["KUBE-LOOP-BACK"].activeEntries 5269 assert.Equal(t, 3, activeEntries1.Len(), "Expected 3 active entry in KUBE-LOOP-BACK") 5270 assert.Equal(t, true, activeEntries1.Has("10.0.1.1,tcp:80,10.0.1.1"), "Expected activeEntries to reference first (local) pod") 5271 assert.Equal(t, true, activeEntries1.Has("10.0.1.2,tcp:80,10.0.1.2"), "Expected activeEntries to reference second (local) pod") 5272 assert.Equal(t, true, activeEntries1.Has("10.0.1.3,tcp:80,10.0.1.3"), "Expected activeEntries to reference second (local) pod") 5273 5274 virtualServers, vsErr := ipvs.GetVirtualServers() 5275 assert.Nil(t, vsErr, "Expected no error getting virtual servers") 5276 assert.Len(t, virtualServers, 2, "Expected 2 virtual server") 5277 5278 var clusterIPServer, externalIPServer *utilipvs.VirtualServer 5279 for _, virtualServer := range virtualServers { 5280 if virtualServer.Address.String() == "172.20.1.1" { 5281 clusterIPServer = virtualServer 5282 } 5283 5284 if virtualServer.Address.String() == "1.2.3.4" { 5285 externalIPServer = virtualServer 5286 } 5287 } 5288 5289 // clusterIP should fall back to cluster-wide ready + terminating endpoints 5290 realServers1, rsErr1 := ipvs.GetRealServers(clusterIPServer) 5291 assert.Nil(t, rsErr1, "Expected no error getting real servers") 5292 assert.Len(t, realServers1, 3, "Expected 1 real servers") 5293 assert.Equal(t, realServers1[0].String(), "10.0.1.1:80") 5294 assert.Equal(t, realServers1[1].String(), "10.0.1.2:80") 5295 assert.Equal(t, realServers1[2].String(), "10.0.1.4:80") 5296 5297 // externalIP should fall back to ready + terminating endpoints 5298 realServers2, rsErr2 := ipvs.GetRealServers(externalIPServer) 5299 assert.Nil(t, rsErr2, "Expected no error getting real servers") 5300 assert.Len(t, realServers2, 3, "Expected 2 real servers") 5301 assert.Equal(t, realServers2[0].String(), "10.0.1.1:80") 5302 assert.Equal(t, realServers2[1].String(), "10.0.1.2:80") 5303 assert.Equal(t, realServers2[2].String(), "10.0.1.4:80") 5304 5305 fp.OnEndpointSliceDelete(endpointSlice) 5306 fp.syncProxyRules() 5307 5308 // Ensure that Proxier updates ipvs appropriately after EndpointSlice delete 5309 assert.NotNil(t, fp.ipsetList["KUBE-LOOP-BACK"]) 5310 activeEntries2 := fp.ipsetList["KUBE-LOOP-BACK"].activeEntries 5311 assert.Equal(t, 0, activeEntries2.Len(), "Expected 0 active entries in KUBE-LOOP-BACK") 5312 5313 virtualServers, vsErr = ipvs.GetVirtualServers() 5314 assert.Nil(t, vsErr, "Expected no error getting virtual servers") 5315 assert.Len(t, virtualServers, 2, "Expected 2 virtual server") 5316 5317 for _, virtualServer := range virtualServers { 5318 if virtualServer.Address.String() == "172.20.1.1" { 5319 clusterIPServer = virtualServer 5320 } 5321 5322 if virtualServer.Address.String() == "1.2.3.4" { 5323 externalIPServer = virtualServer 5324 } 5325 } 5326 5327 realServers1, rsErr1 = ipvs.GetRealServers(clusterIPServer) 5328 assert.Nil(t, rsErr1, "Expected no error getting real servers") 5329 assert.Len(t, realServers1, 0, "Expected 0 real servers") 5330 5331 realServers2, rsErr2 = ipvs.GetRealServers(externalIPServer) 5332 assert.Nil(t, rsErr2, "Expected no error getting real servers") 5333 assert.Len(t, realServers2, 0, "Expected 0 real servers") 5334 } 5335 5336 // Test_EndpointSliceOnlyReadyTerminatingLocal tests that when there are only local ready terminating 5337 // endpoints, we fall back to those endpoints. 5338 func Test_EndpointSliceOnlyReadyAndTerminatingLocal(t *testing.T) { 5339 5340 ipt := iptablestest.NewFake() 5341 ipvs := ipvstest.NewFake() 5342 ipset := ipsettest.NewFake(testIPSetVersion) 5343 fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol) 5344 fp.servicesSynced = true 5345 // fp.endpointsSynced = true 5346 fp.endpointSlicesSynced = true 5347 5348 // Add initial service 5349 serviceName := "svc1" 5350 namespaceName := "ns1" 5351 fp.OnServiceAdd(&v1.Service{ 5352 ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: namespaceName}, 5353 Spec: v1.ServiceSpec{ 5354 ClusterIP: "172.20.1.1", 5355 Selector: map[string]string{"foo": "bar"}, 5356 Type: v1.ServiceTypeNodePort, 5357 ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyLocal, 5358 InternalTrafficPolicy: ptr.To(v1.ServiceInternalTrafficPolicyCluster), 5359 ExternalIPs: []string{ 5360 "1.2.3.4", 5361 }, 5362 Ports: []v1.ServicePort{ 5363 { 5364 Name: "", 5365 Port: 80, 5366 TargetPort: intstr.FromInt32(80), 5367 Protocol: v1.ProtocolTCP, 5368 }, 5369 }, 5370 }, 5371 }) 5372 5373 // Add initial endpoint slice 5374 endpointSlice := &discovery.EndpointSlice{ 5375 ObjectMeta: metav1.ObjectMeta{ 5376 Name: fmt.Sprintf("%s-1", serviceName), 5377 Namespace: namespaceName, 5378 Labels: map[string]string{discovery.LabelServiceName: serviceName}, 5379 }, 5380 Ports: []discovery.EndpointPort{{ 5381 Name: ptr.To(""), 5382 Port: ptr.To[int32](80), 5383 Protocol: ptr.To(v1.ProtocolTCP), 5384 }}, 5385 AddressType: discovery.AddressTypeIPv4, 5386 Endpoints: []discovery.Endpoint{ 5387 { 5388 Addresses: []string{"10.0.1.1"}, 5389 Conditions: discovery.EndpointConditions{ 5390 Ready: ptr.To(false), 5391 Serving: ptr.To(true), 5392 Terminating: ptr.To(true), 5393 }, 5394 NodeName: ptr.To(testHostname), 5395 }, 5396 { 5397 Addresses: []string{"10.0.1.2"}, 5398 Conditions: discovery.EndpointConditions{ 5399 Ready: ptr.To(false), 5400 Serving: ptr.To(true), 5401 Terminating: ptr.To(true), 5402 }, 5403 NodeName: ptr.To(testHostname), 5404 }, 5405 { 5406 Addresses: []string{"10.0.1.3"}, 5407 Conditions: discovery.EndpointConditions{ 5408 Ready: ptr.To(false), 5409 Serving: ptr.To(false), 5410 Terminating: ptr.To(true), 5411 }, 5412 NodeName: ptr.To(testHostname), 5413 }, 5414 { 5415 Addresses: []string{"10.0.1.4"}, 5416 Conditions: discovery.EndpointConditions{ 5417 Ready: ptr.To(false), 5418 Serving: ptr.To(true), 5419 Terminating: ptr.To(true), 5420 }, 5421 NodeName: ptr.To("another-host"), 5422 }, 5423 { 5424 Addresses: []string{"10.0.1.5"}, 5425 Conditions: discovery.EndpointConditions{ 5426 Ready: ptr.To(true), 5427 Serving: ptr.To(true), 5428 Terminating: ptr.To(false), 5429 }, 5430 NodeName: ptr.To("another-host"), 5431 }, 5432 }, 5433 } 5434 5435 fp.OnEndpointSliceAdd(endpointSlice) 5436 fp.syncProxyRules() 5437 5438 // Ensure that Proxier updates ipvs appropriately after EndpointSlice update 5439 assert.NotNil(t, fp.ipsetList["KUBE-LOOP-BACK"]) 5440 activeEntries1 := fp.ipsetList["KUBE-LOOP-BACK"].activeEntries 5441 assert.Equal(t, 3, activeEntries1.Len(), "Expected 3 active entry in KUBE-LOOP-BACK") 5442 assert.Equal(t, true, activeEntries1.Has("10.0.1.1,tcp:80,10.0.1.1"), "Expected activeEntries to reference first (local) pod") 5443 assert.Equal(t, true, activeEntries1.Has("10.0.1.2,tcp:80,10.0.1.2"), "Expected activeEntries to reference second (local) pod") 5444 assert.Equal(t, true, activeEntries1.Has("10.0.1.3,tcp:80,10.0.1.3"), "Expected activeEntries to reference second (local) pod") 5445 5446 virtualServers, vsErr := ipvs.GetVirtualServers() 5447 assert.Nil(t, vsErr, "Expected no error getting virtual servers") 5448 assert.Len(t, virtualServers, 2, "Expected 2 virtual server") 5449 5450 var clusterIPServer, externalIPServer *utilipvs.VirtualServer 5451 for _, virtualServer := range virtualServers { 5452 if virtualServer.Address.String() == "172.20.1.1" { 5453 clusterIPServer = virtualServer 5454 } 5455 5456 if virtualServer.Address.String() == "1.2.3.4" { 5457 externalIPServer = virtualServer 5458 } 5459 } 5460 5461 // clusterIP should route to cluster-wide Ready endpoints 5462 realServers1, rsErr1 := ipvs.GetRealServers(clusterIPServer) 5463 assert.Nil(t, rsErr1, "Expected no error getting real servers") 5464 assert.Len(t, realServers1, 1, "Expected 1 real servers") 5465 assert.Equal(t, realServers1[0].String(), "10.0.1.5:80") 5466 5467 // externalIP should fall back to local ready + terminating endpoints 5468 realServers2, rsErr2 := ipvs.GetRealServers(externalIPServer) 5469 assert.Nil(t, rsErr2, "Expected no error getting real servers") 5470 assert.Len(t, realServers2, 2, "Expected 2 real servers") 5471 assert.Equal(t, realServers2[0].String(), "10.0.1.1:80") 5472 assert.Equal(t, realServers2[1].String(), "10.0.1.2:80") 5473 5474 fp.OnEndpointSliceDelete(endpointSlice) 5475 fp.syncProxyRules() 5476 5477 // Ensure that Proxier updates ipvs appropriately after EndpointSlice delete 5478 assert.NotNil(t, fp.ipsetList["KUBE-LOOP-BACK"]) 5479 activeEntries2 := fp.ipsetList["KUBE-LOOP-BACK"].activeEntries 5480 assert.Equal(t, 0, activeEntries2.Len(), "Expected 0 active entries in KUBE-LOOP-BACK") 5481 5482 virtualServers, vsErr = ipvs.GetVirtualServers() 5483 assert.Nil(t, vsErr, "Expected no error getting virtual servers") 5484 assert.Len(t, virtualServers, 2, "Expected 2 virtual server") 5485 5486 for _, virtualServer := range virtualServers { 5487 if virtualServer.Address.String() == "172.20.1.1" { 5488 clusterIPServer = virtualServer 5489 } 5490 5491 if virtualServer.Address.String() == "1.2.3.4" { 5492 externalIPServer = virtualServer 5493 } 5494 } 5495 5496 realServers1, rsErr1 = ipvs.GetRealServers(clusterIPServer) 5497 assert.Nil(t, rsErr1, "Expected no error getting real servers") 5498 assert.Len(t, realServers1, 0, "Expected 0 real servers") 5499 5500 realServers2, rsErr2 = ipvs.GetRealServers(externalIPServer) 5501 assert.Nil(t, rsErr2, "Expected no error getting real servers") 5502 assert.Len(t, realServers2, 0, "Expected 0 real servers") 5503 } 5504 5505 func TestIpIsValidForSet(t *testing.T) { 5506 testCases := []struct { 5507 isIPv6 bool 5508 ip string 5509 res bool 5510 }{ 5511 { 5512 false, 5513 "127.0.0.1", 5514 false, 5515 }, 5516 { 5517 false, 5518 "127.0.0.0", 5519 false, 5520 }, 5521 { 5522 false, 5523 "127.6.7.8", 5524 false, 5525 }, 5526 { 5527 false, 5528 "8.8.8.8", 5529 true, 5530 }, 5531 { 5532 false, 5533 "192.168.0.1", 5534 true, 5535 }, 5536 { 5537 false, 5538 "169.254.0.0", 5539 true, 5540 }, 5541 { 5542 false, 5543 "::ffff:169.254.0.0", // IPv6 mapped IPv4 5544 true, 5545 }, 5546 { 5547 false, 5548 "1000::", 5549 false, 5550 }, 5551 // IPv6 5552 { 5553 true, 5554 "::1", 5555 false, 5556 }, 5557 { 5558 true, 5559 "1000::", 5560 true, 5561 }, 5562 { 5563 true, 5564 "fe80::200:ff:fe01:1", 5565 false, 5566 }, 5567 { 5568 true, 5569 "8.8.8.8", 5570 false, 5571 }, 5572 { 5573 true, 5574 "::ffff:8.8.8.8", 5575 false, 5576 }, 5577 } 5578 5579 for _, tc := range testCases { 5580 v := &netlinkHandle{} 5581 v.isIPv6 = tc.isIPv6 5582 ip := netutils.ParseIPSloppy(tc.ip) 5583 if ip == nil { 5584 t.Errorf("Parse error: %s", tc.ip) 5585 } 5586 if v.isValidForSet(ip) != tc.res { 5587 if tc.isIPv6 { 5588 t.Errorf("IPv6: %s", tc.ip) 5589 } else { 5590 t.Errorf("IPv4: %s", tc.ip) 5591 } 5592 } 5593 } 5594 } 5595 5596 func TestNoEndpointsMetric(t *testing.T) { 5597 type endpoint struct { 5598 ip string 5599 hostname string 5600 } 5601 5602 metrics.RegisterMetrics() 5603 5604 testCases := []struct { 5605 name string 5606 internalTrafficPolicy *v1.ServiceInternalTrafficPolicy 5607 externalTrafficPolicy v1.ServiceExternalTrafficPolicy 5608 endpoints []endpoint 5609 expectedSyncProxyRulesNoLocalEndpointsTotalInternal int 5610 expectedSyncProxyRulesNoLocalEndpointsTotalExternal int 5611 }{ 5612 { 5613 name: "internalTrafficPolicy is set and there are local endpoints", 5614 internalTrafficPolicy: ptr.To(v1.ServiceInternalTrafficPolicyLocal), 5615 endpoints: []endpoint{ 5616 {"10.0.1.1", testHostname}, 5617 {"10.0.1.2", "host1"}, 5618 {"10.0.1.3", "host2"}, 5619 }, 5620 }, 5621 { 5622 name: "externalTrafficPolicy is set and there are local endpoints", 5623 externalTrafficPolicy: v1.ServiceExternalTrafficPolicyLocal, 5624 endpoints: []endpoint{ 5625 {"10.0.1.1", testHostname}, 5626 {"10.0.1.2", "host1"}, 5627 {"10.0.1.3", "host2"}, 5628 }, 5629 }, 5630 { 5631 name: "both policies are set and there are local endpoints", 5632 internalTrafficPolicy: ptr.To(v1.ServiceInternalTrafficPolicyLocal), 5633 externalTrafficPolicy: v1.ServiceExternalTrafficPolicyLocal, 5634 endpoints: []endpoint{ 5635 {"10.0.1.1", testHostname}, 5636 {"10.0.1.2", "host1"}, 5637 {"10.0.1.3", "host2"}, 5638 }, 5639 }, 5640 { 5641 name: "internalTrafficPolicy is set and there are no local endpoints", 5642 internalTrafficPolicy: ptr.To(v1.ServiceInternalTrafficPolicyLocal), 5643 endpoints: []endpoint{ 5644 {"10.0.1.1", "host0"}, 5645 {"10.0.1.2", "host1"}, 5646 {"10.0.1.3", "host2"}, 5647 }, 5648 expectedSyncProxyRulesNoLocalEndpointsTotalInternal: 1, 5649 }, 5650 { 5651 name: "externalTrafficPolicy is set and there are no local endpoints", 5652 externalTrafficPolicy: v1.ServiceExternalTrafficPolicyLocal, 5653 endpoints: []endpoint{ 5654 {"10.0.1.1", "host0"}, 5655 {"10.0.1.2", "host1"}, 5656 {"10.0.1.3", "host2"}, 5657 }, 5658 expectedSyncProxyRulesNoLocalEndpointsTotalExternal: 1, 5659 }, 5660 { 5661 name: "Both policies are set and there are no local endpoints", 5662 internalTrafficPolicy: ptr.To(v1.ServiceInternalTrafficPolicyLocal), 5663 externalTrafficPolicy: v1.ServiceExternalTrafficPolicyLocal, 5664 endpoints: []endpoint{ 5665 {"10.0.1.1", "host0"}, 5666 {"10.0.1.2", "host1"}, 5667 {"10.0.1.3", "host2"}, 5668 }, 5669 expectedSyncProxyRulesNoLocalEndpointsTotalInternal: 1, 5670 expectedSyncProxyRulesNoLocalEndpointsTotalExternal: 1, 5671 }, 5672 { 5673 name: "Both policies are set and there are no endpoints at all", 5674 internalTrafficPolicy: ptr.To(v1.ServiceInternalTrafficPolicyLocal), 5675 externalTrafficPolicy: v1.ServiceExternalTrafficPolicyLocal, 5676 endpoints: []endpoint{}, 5677 expectedSyncProxyRulesNoLocalEndpointsTotalInternal: 0, 5678 expectedSyncProxyRulesNoLocalEndpointsTotalExternal: 0, 5679 }, 5680 } 5681 for _, tc := range testCases { 5682 ipt := iptablestest.NewFake() 5683 ipvs := ipvstest.NewFake() 5684 ipset := ipsettest.NewFake(testIPSetVersion) 5685 fp := NewFakeProxier(ipt, ipvs, ipset, []string{"10.0.0.1"}, nil, v1.IPv4Protocol) 5686 fp.servicesSynced = true 5687 // fp.endpointsSynced = true 5688 fp.endpointSlicesSynced = true 5689 5690 // Add initial service 5691 serviceName := "svc1" 5692 namespaceName := "ns1" 5693 5694 svc := &v1.Service{ 5695 ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: namespaceName}, 5696 Spec: v1.ServiceSpec{ 5697 ClusterIP: "172.20.1.1", 5698 Selector: map[string]string{"foo": "bar"}, 5699 Ports: []v1.ServicePort{{Name: "p80", Port: 80, TargetPort: intstr.FromInt32(80), Protocol: v1.ProtocolTCP, NodePort: 30000}}, 5700 }, 5701 } 5702 if tc.internalTrafficPolicy != nil { 5703 svc.Spec.InternalTrafficPolicy = tc.internalTrafficPolicy 5704 } 5705 if tc.externalTrafficPolicy != "" { 5706 svc.Spec.Type = v1.ServiceTypeNodePort 5707 svc.Spec.ExternalTrafficPolicy = tc.externalTrafficPolicy 5708 } 5709 5710 fp.OnServiceAdd(svc) 5711 5712 // Add initial endpoint slice 5713 endpointSlice := &discovery.EndpointSlice{ 5714 ObjectMeta: metav1.ObjectMeta{ 5715 Name: fmt.Sprintf("%s-1", serviceName), 5716 Namespace: namespaceName, 5717 Labels: map[string]string{discovery.LabelServiceName: serviceName}, 5718 }, 5719 Ports: []discovery.EndpointPort{{ 5720 Name: ptr.To("p80"), 5721 Port: ptr.To[int32](80), 5722 Protocol: ptr.To(v1.ProtocolTCP), 5723 }}, 5724 AddressType: discovery.AddressTypeIPv4, 5725 } 5726 5727 for _, ep := range tc.endpoints { 5728 endpointSlice.Endpoints = append(endpointSlice.Endpoints, discovery.Endpoint{ 5729 Addresses: []string{ep.ip}, 5730 Conditions: discovery.EndpointConditions{Ready: ptr.To(true)}, 5731 NodeName: ptr.To(ep.hostname), 5732 }) 5733 } 5734 5735 fp.OnEndpointSliceAdd(endpointSlice) 5736 fp.syncProxyRules() 5737 5738 syncProxyRulesNoLocalEndpointsTotalInternal, err := testutil.GetGaugeMetricValue(metrics.SyncProxyRulesNoLocalEndpointsTotal.WithLabelValues("internal")) 5739 if err != nil { 5740 t.Errorf("failed to get %s value(internal), err: %v", metrics.SyncProxyRulesNoLocalEndpointsTotal.Name, err) 5741 } 5742 5743 if tc.expectedSyncProxyRulesNoLocalEndpointsTotalInternal != int(syncProxyRulesNoLocalEndpointsTotalInternal) { 5744 t.Errorf("sync_proxy_rules_no_endpoints_total metric mismatch(internal): got=%d, expected %d", int(syncProxyRulesNoLocalEndpointsTotalInternal), tc.expectedSyncProxyRulesNoLocalEndpointsTotalInternal) 5745 } 5746 5747 syncProxyRulesNoLocalEndpointsTotalExternal, err := testutil.GetGaugeMetricValue(metrics.SyncProxyRulesNoLocalEndpointsTotal.WithLabelValues("external")) 5748 if err != nil { 5749 t.Errorf("failed to get %s value(external), err: %v", metrics.SyncProxyRulesNoLocalEndpointsTotal.Name, err) 5750 } 5751 5752 if tc.expectedSyncProxyRulesNoLocalEndpointsTotalExternal != int(syncProxyRulesNoLocalEndpointsTotalExternal) { 5753 t.Errorf("sync_proxy_rules_no_endpoints_total metric mismatch(external): got=%d, expected %d", int(syncProxyRulesNoLocalEndpointsTotalExternal), tc.expectedSyncProxyRulesNoLocalEndpointsTotalExternal) 5754 } 5755 } 5756 } 5757 5758 func TestDismissLocalhostRuleExist(t *testing.T) { 5759 tests := []struct { 5760 name string 5761 ipFamily v1.IPFamily 5762 src string 5763 }{ 5764 { 5765 name: "ipv4 rule", 5766 ipFamily: v1.IPv4Protocol, 5767 src: "127.0.0.0/8", 5768 }, 5769 { 5770 name: "ipv6 rule", 5771 ipFamily: v1.IPv6Protocol, 5772 src: "::1/128", 5773 }, 5774 } 5775 for _, test := range tests { 5776 t.Run(test.name, func(t *testing.T) { 5777 ipt := iptablestest.NewFake() 5778 if test.ipFamily == v1.IPv6Protocol { 5779 ipt = iptablestest.NewIPv6Fake() 5780 } 5781 ipvs := ipvstest.NewFake() 5782 ipset := ipsettest.NewFake(testIPSetVersion) 5783 fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, test.ipFamily) 5784 5785 fp.syncProxyRules() 5786 5787 rules := getRules(ipt, kubeServicesChain) 5788 if len(rules) <= 0 { 5789 t.Errorf("skip loop back ip in kubeservice chain not exist") 5790 return 5791 } 5792 if !rules[0].Jump.Matches("RETURN") || !rules[0].SourceAddress.Matches(test.src) { 5793 t.Errorf("rules not match, expect jump: %s, got: %s; expect source address: %s, got: %s", "RETURN", rules[0].Jump.String(), test.src, rules[0].SourceAddress.String()) 5794 } 5795 }) 5796 } 5797 } 5798 5799 func TestLoadBalancerIngressRouteTypeProxy(t *testing.T) { 5800 testCases := []struct { 5801 name string 5802 ipModeEnabled bool 5803 svcIP string 5804 svcLBIP string 5805 ipMode *v1.LoadBalancerIPMode 5806 expectedServices int 5807 }{ 5808 /* LoadBalancerIPMode disabled */ 5809 { 5810 name: "LoadBalancerIPMode disabled, ipMode Proxy", 5811 ipModeEnabled: false, 5812 svcIP: "10.20.30.41", 5813 svcLBIP: "1.2.3.4", 5814 ipMode: ptr.To(v1.LoadBalancerIPModeProxy), 5815 expectedServices: 2, 5816 }, 5817 { 5818 name: "LoadBalancerIPMode disabled, ipMode VIP", 5819 ipModeEnabled: false, 5820 svcIP: "10.20.30.42", 5821 svcLBIP: "1.2.3.5", 5822 ipMode: ptr.To(v1.LoadBalancerIPModeVIP), 5823 expectedServices: 2, 5824 }, 5825 { 5826 name: "LoadBalancerIPMode disabled, ipMode nil", 5827 ipModeEnabled: false, 5828 svcIP: "10.20.30.43", 5829 svcLBIP: "1.2.3.6", 5830 ipMode: nil, 5831 expectedServices: 2, 5832 }, 5833 /* LoadBalancerIPMode enabled */ 5834 { 5835 name: "LoadBalancerIPMode enabled, ipMode Proxy", 5836 ipModeEnabled: true, 5837 svcIP: "10.20.30.41", 5838 svcLBIP: "1.2.3.4", 5839 ipMode: ptr.To(v1.LoadBalancerIPModeProxy), 5840 expectedServices: 1, 5841 }, 5842 { 5843 name: "LoadBalancerIPMode enabled, ipMode VIP", 5844 ipModeEnabled: true, 5845 svcIP: "10.20.30.42", 5846 svcLBIP: "1.2.3.5", 5847 ipMode: ptr.To(v1.LoadBalancerIPModeVIP), 5848 expectedServices: 2, 5849 }, 5850 { 5851 name: "LoadBalancerIPMode enabled, ipMode nil", 5852 ipModeEnabled: true, 5853 svcIP: "10.20.30.43", 5854 svcLBIP: "1.2.3.6", 5855 ipMode: nil, 5856 expectedServices: 2, 5857 }, 5858 } 5859 5860 svcPort := 80 5861 svcNodePort := 3001 5862 svcPortName := proxy.ServicePortName{ 5863 NamespacedName: makeNSN("ns1", "svc1"), 5864 Port: "p80", 5865 } 5866 5867 for _, testCase := range testCases { 5868 t.Run(testCase.name, func(t *testing.T) { 5869 defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LoadBalancerIPMode, testCase.ipModeEnabled)() 5870 _, fp := buildFakeProxier() 5871 makeServiceMap(fp, 5872 makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { 5873 svc.Spec.Type = "LoadBalancer" 5874 svc.Spec.ClusterIP = testCase.svcIP 5875 svc.Spec.Ports = []v1.ServicePort{{ 5876 Name: svcPortName.Port, 5877 Port: int32(svcPort), 5878 Protocol: v1.ProtocolTCP, 5879 NodePort: int32(svcNodePort), 5880 }} 5881 svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{ 5882 IP: testCase.svcLBIP, 5883 IPMode: testCase.ipMode, 5884 }} 5885 }), 5886 ) 5887 5888 makeEndpointSliceMap(fp, 5889 makeTestEndpointSlice("ns1", "svc1", 1, func(eps *discovery.EndpointSlice) { 5890 eps.AddressType = discovery.AddressTypeIPv4 5891 eps.Endpoints = []discovery.Endpoint{{ 5892 Addresses: []string{"10.180.0.1"}, 5893 }} 5894 eps.Ports = []discovery.EndpointPort{{ 5895 Name: ptr.To("p80"), 5896 Port: ptr.To[int32](80), 5897 Protocol: ptr.To(v1.ProtocolTCP), 5898 }} 5899 }), 5900 ) 5901 5902 fp.syncProxyRules() 5903 5904 services, err := fp.ipvs.GetVirtualServers() 5905 if err != nil { 5906 t.Errorf("Failed to get ipvs services, err: %v", err) 5907 } 5908 if len(services) != testCase.expectedServices { 5909 t.Errorf("Expected %d ipvs services, got %d", testCase.expectedServices, len(services)) 5910 } 5911 }) 5912 } 5913 }