k8s.io/kubernetes@v1.29.3/pkg/proxy/ipvs/proxier_test.go (about)

     1  //go:build !windows
     2  // +build !windows
     3  
     4  /*
     5  Copyright 2017 The Kubernetes Authors.
     6  
     7  Licensed under the Apache License, Version 2.0 (the "License");
     8  you may not use this file except in compliance with the License.
     9  You may obtain a copy of the License at
    10  
    11      http://www.apache.org/licenses/LICENSE-2.0
    12  
    13  Unless required by applicable law or agreed to in writing, software
    14  distributed under the License is distributed on an "AS IS" BASIS,
    15  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    16  See the License for the specific language governing permissions and
    17  limitations under the License.
    18  */
    19  
    20  package ipvs
    21  
    22  import (
    23  	"bytes"
    24  	"fmt"
    25  	"net"
    26  	"reflect"
    27  	"sort"
    28  	"strings"
    29  	"testing"
    30  	"time"
    31  
    32  	"github.com/stretchr/testify/assert"
    33  	v1 "k8s.io/api/core/v1"
    34  	discovery "k8s.io/api/discovery/v1"
    35  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    36  	"k8s.io/apimachinery/pkg/types"
    37  	"k8s.io/apimachinery/pkg/util/intstr"
    38  	"k8s.io/apimachinery/pkg/util/sets"
    39  	utilfeature "k8s.io/apiserver/pkg/util/feature"
    40  	featuregatetesting "k8s.io/component-base/featuregate/testing"
    41  	"k8s.io/component-base/metrics/testutil"
    42  	"k8s.io/kubernetes/pkg/features"
    43  	"k8s.io/kubernetes/pkg/proxy"
    44  	"k8s.io/kubernetes/pkg/proxy/healthcheck"
    45  	utilipset "k8s.io/kubernetes/pkg/proxy/ipvs/ipset"
    46  	ipsettest "k8s.io/kubernetes/pkg/proxy/ipvs/ipset/testing"
    47  	netlinktest "k8s.io/kubernetes/pkg/proxy/ipvs/testing"
    48  	utilipvs "k8s.io/kubernetes/pkg/proxy/ipvs/util"
    49  	ipvstest "k8s.io/kubernetes/pkg/proxy/ipvs/util/testing"
    50  	"k8s.io/kubernetes/pkg/proxy/metrics"
    51  	proxyutil "k8s.io/kubernetes/pkg/proxy/util"
    52  	proxyutiliptables "k8s.io/kubernetes/pkg/proxy/util/iptables"
    53  	proxyutiltest "k8s.io/kubernetes/pkg/proxy/util/testing"
    54  	"k8s.io/kubernetes/pkg/util/async"
    55  	utiliptables "k8s.io/kubernetes/pkg/util/iptables"
    56  	iptablestest "k8s.io/kubernetes/pkg/util/iptables/testing"
    57  	"k8s.io/utils/exec"
    58  	fakeexec "k8s.io/utils/exec/testing"
    59  	netutils "k8s.io/utils/net"
    60  	"k8s.io/utils/ptr"
    61  )
    62  
    63  const testHostname = "test-hostname"
    64  
    65  // fakeIpvs implements utilipvs.Interface
    66  type fakeIpvs struct {
    67  	ipvsErr   string
    68  	vsCreated bool
    69  }
    70  
    71  func (f *fakeIpvs) Flush() error {
    72  	return nil
    73  }
    74  func (f *fakeIpvs) AddVirtualServer(*utilipvs.VirtualServer) error {
    75  	if f.ipvsErr == "AddVirtualServer" {
    76  		return fmt.Errorf("oops")
    77  	}
    78  	f.vsCreated = true
    79  	return nil
    80  }
    81  func (f *fakeIpvs) UpdateVirtualServer(*utilipvs.VirtualServer) error {
    82  	return nil
    83  }
    84  func (f *fakeIpvs) DeleteVirtualServer(*utilipvs.VirtualServer) error {
    85  	if f.ipvsErr == "DeleteVirtualServer" {
    86  		return fmt.Errorf("oops")
    87  	}
    88  	return nil
    89  }
    90  func (f *fakeIpvs) GetVirtualServer(*utilipvs.VirtualServer) (*utilipvs.VirtualServer, error) {
    91  	return nil, nil
    92  }
    93  func (f *fakeIpvs) GetVirtualServers() ([]*utilipvs.VirtualServer, error) {
    94  	if f.ipvsErr == "GetVirtualServers" {
    95  		return nil, fmt.Errorf("oops")
    96  	}
    97  	if f.vsCreated {
    98  		vs := []*utilipvs.VirtualServer{{}}
    99  		return vs, nil
   100  	}
   101  	return nil, nil
   102  }
   103  func (f *fakeIpvs) AddRealServer(*utilipvs.VirtualServer, *utilipvs.RealServer) error {
   104  	return nil
   105  }
   106  func (f *fakeIpvs) GetRealServers(*utilipvs.VirtualServer) ([]*utilipvs.RealServer, error) {
   107  	return nil, nil
   108  }
   109  func (f *fakeIpvs) DeleteRealServer(*utilipvs.VirtualServer, *utilipvs.RealServer) error {
   110  	return nil
   111  }
   112  func (f *fakeIpvs) UpdateRealServer(*utilipvs.VirtualServer, *utilipvs.RealServer) error {
   113  	return nil
   114  }
   115  func (f *fakeIpvs) ConfigureTimeouts(time.Duration, time.Duration, time.Duration) error {
   116  	return nil
   117  }
   118  
   119  // fakeIPSetVersioner implements IPSetVersioner.
   120  type fakeIPSetVersioner struct {
   121  	version string
   122  	err     error
   123  }
   124  
   125  func (fake *fakeIPSetVersioner) GetVersion() (string, error) {
   126  	return fake.version, fake.err
   127  }
   128  
   129  func NewFakeProxier(ipt utiliptables.Interface, ipvs utilipvs.Interface, ipset utilipset.Interface, nodeIPs []string, excludeCIDRs []*net.IPNet, ipFamily v1.IPFamily) *Proxier {
   130  
   131  	netlinkHandle := netlinktest.NewFakeNetlinkHandle(ipFamily == v1.IPv6Protocol)
   132  	netlinkHandle.SetLocalAddresses("eth0", nodeIPs...)
   133  
   134  	fcmd := fakeexec.FakeCmd{
   135  		CombinedOutputScript: []fakeexec.FakeAction{
   136  			func() ([]byte, []byte, error) { return []byte("dummy device have been created"), nil, nil },
   137  			func() ([]byte, []byte, error) { return []byte(""), nil, nil },
   138  		},
   139  	}
   140  	fexec := &fakeexec.FakeExec{
   141  		CommandScript: []fakeexec.FakeCommandAction{
   142  			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
   143  			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
   144  		},
   145  		LookPathFunc: func(cmd string) (string, error) { return cmd, nil },
   146  	}
   147  	// initialize ipsetList with all sets we needed
   148  	ipsetList := make(map[string]*IPSet)
   149  	for _, is := range ipsetInfo {
   150  		ipsetList[is.name] = NewIPSet(ipset, is.name, is.setType, false, is.comment)
   151  	}
   152  	p := &Proxier{
   153  		exec:                  fexec,
   154  		svcPortMap:            make(proxy.ServicePortMap),
   155  		serviceChanges:        proxy.NewServiceChangeTracker(newServiceInfo, ipFamily, nil, nil),
   156  		endpointsMap:          make(proxy.EndpointsMap),
   157  		endpointsChanges:      proxy.NewEndpointsChangeTracker(testHostname, nil, ipFamily, nil, nil),
   158  		excludeCIDRs:          excludeCIDRs,
   159  		iptables:              ipt,
   160  		ipvs:                  ipvs,
   161  		ipset:                 ipset,
   162  		strictARP:             false,
   163  		localDetector:         proxyutiliptables.NewNoOpLocalDetector(),
   164  		hostname:              testHostname,
   165  		serviceHealthServer:   healthcheck.NewFakeServiceHealthServer(),
   166  		ipvsScheduler:         defaultScheduler,
   167  		iptablesData:          bytes.NewBuffer(nil),
   168  		filterChainsData:      bytes.NewBuffer(nil),
   169  		natChains:             proxyutil.NewLineBuffer(),
   170  		natRules:              proxyutil.NewLineBuffer(),
   171  		filterChains:          proxyutil.NewLineBuffer(),
   172  		filterRules:           proxyutil.NewLineBuffer(),
   173  		netlinkHandle:         netlinkHandle,
   174  		ipsetList:             ipsetList,
   175  		nodePortAddresses:     proxyutil.NewNodePortAddresses(ipFamily, nil),
   176  		networkInterfacer:     proxyutiltest.NewFakeNetwork(),
   177  		gracefuldeleteManager: NewGracefulTerminationManager(ipvs),
   178  		ipFamily:              ipFamily,
   179  	}
   180  	p.setInitialized(true)
   181  	p.syncRunner = async.NewBoundedFrequencyRunner("test-sync-runner", p.syncProxyRules, 0, time.Minute, 1)
   182  	return p
   183  }
   184  
   185  func makeNSN(namespace, name string) types.NamespacedName {
   186  	return types.NamespacedName{Namespace: namespace, Name: name}
   187  }
   188  
   189  func makeServiceMap(proxier *Proxier, allServices ...*v1.Service) {
   190  	for i := range allServices {
   191  		proxier.OnServiceAdd(allServices[i])
   192  	}
   193  
   194  	proxier.mu.Lock()
   195  	defer proxier.mu.Unlock()
   196  	proxier.servicesSynced = true
   197  }
   198  
   199  func makeEndpointSliceMap(proxier *Proxier, allEpSlices ...*discovery.EndpointSlice) {
   200  	for i := range allEpSlices {
   201  		proxier.OnEndpointSliceAdd(allEpSlices[i])
   202  	}
   203  	proxier.mu.Lock()
   204  	defer proxier.mu.Unlock()
   205  	proxier.endpointSlicesSynced = true
   206  }
   207  
   208  func makeTestService(namespace, name string, svcFunc func(*v1.Service)) *v1.Service {
   209  	svc := &v1.Service{
   210  		ObjectMeta: metav1.ObjectMeta{
   211  			Name:        name,
   212  			Namespace:   namespace,
   213  			Annotations: map[string]string{},
   214  		},
   215  		Spec:   v1.ServiceSpec{},
   216  		Status: v1.ServiceStatus{},
   217  	}
   218  	svcFunc(svc)
   219  	return svc
   220  }
   221  
   222  func populateEndpointSlices(proxier *Proxier, allEndpointSlices ...*discovery.EndpointSlice) {
   223  	for i := range allEndpointSlices {
   224  		proxier.OnEndpointSliceAdd(allEndpointSlices[i])
   225  	}
   226  }
   227  
   228  func makeTestEndpointSlice(namespace, name string, sliceNum int, epsFunc func(*discovery.EndpointSlice)) *discovery.EndpointSlice {
   229  	eps := &discovery.EndpointSlice{
   230  		ObjectMeta: metav1.ObjectMeta{
   231  			Name:      fmt.Sprintf("%s-%d", name, sliceNum),
   232  			Namespace: namespace,
   233  			Labels:    map[string]string{discovery.LabelServiceName: name},
   234  		},
   235  	}
   236  	epsFunc(eps)
   237  	return eps
   238  }
   239  
   240  func TestCleanupLeftovers(t *testing.T) {
   241  	ipt := iptablestest.NewFake()
   242  	ipvs := ipvstest.NewFake()
   243  	ipset := ipsettest.NewFake(testIPSetVersion)
   244  	fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol)
   245  	svcIP := "10.20.30.41"
   246  	svcPort := 80
   247  	svcNodePort := 3001
   248  	svcPortName := proxy.ServicePortName{
   249  		NamespacedName: makeNSN("ns1", "svc1"),
   250  		Port:           "p80",
   251  	}
   252  
   253  	makeServiceMap(fp,
   254  		makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) {
   255  			svc.Spec.Type = "NodePort"
   256  			svc.Spec.ClusterIP = svcIP
   257  			svc.Spec.Ports = []v1.ServicePort{{
   258  				Name:     svcPortName.Port,
   259  				Port:     int32(svcPort),
   260  				Protocol: v1.ProtocolTCP,
   261  				NodePort: int32(svcNodePort),
   262  			}}
   263  		}),
   264  	)
   265  	epIP := "10.180.0.1"
   266  	populateEndpointSlices(fp,
   267  		makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) {
   268  			eps.AddressType = discovery.AddressTypeIPv4
   269  			eps.Endpoints = []discovery.Endpoint{{
   270  				Addresses: []string{epIP},
   271  			}}
   272  			eps.Ports = []discovery.EndpointPort{{
   273  				Name:     ptr.To(svcPortName.Port),
   274  				Port:     ptr.To(int32(svcPort)),
   275  				Protocol: ptr.To(v1.ProtocolTCP),
   276  			}}
   277  		}),
   278  	)
   279  
   280  	fp.syncProxyRules()
   281  
   282  	// test cleanup left over
   283  	if CleanupLeftovers(ipvs, ipt, ipset) {
   284  		t.Errorf("Cleanup leftovers failed")
   285  	}
   286  }
   287  
   288  func TestCanUseIPVSProxier(t *testing.T) {
   289  	testCases := []struct {
   290  		name         string
   291  		scheduler    string
   292  		ipsetVersion string
   293  		ipsetErr     error
   294  		ipvsErr      string
   295  		ok           bool
   296  	}{
   297  		{
   298  			name:         "happy days",
   299  			ipsetVersion: MinIPSetCheckVersion,
   300  			ok:           true,
   301  		},
   302  		{
   303  			name:         "ipset error",
   304  			scheduler:    "",
   305  			ipsetVersion: MinIPSetCheckVersion,
   306  			ipsetErr:     fmt.Errorf("oops"),
   307  			ok:           false,
   308  		},
   309  		{
   310  			name:         "ipset version too low",
   311  			scheduler:    "rr",
   312  			ipsetVersion: "4.3.0",
   313  			ok:           false,
   314  		},
   315  		{
   316  			name:         "GetVirtualServers fail",
   317  			ipsetVersion: MinIPSetCheckVersion,
   318  			ipvsErr:      "GetVirtualServers",
   319  			ok:           false,
   320  		},
   321  		{
   322  			name:         "AddVirtualServer fail",
   323  			ipsetVersion: MinIPSetCheckVersion,
   324  			ipvsErr:      "AddVirtualServer",
   325  			ok:           false,
   326  		},
   327  		{
   328  			name:         "DeleteVirtualServer fail",
   329  			ipsetVersion: MinIPSetCheckVersion,
   330  			ipvsErr:      "DeleteVirtualServer",
   331  			ok:           false,
   332  		},
   333  	}
   334  
   335  	for _, tc := range testCases {
   336  		ipvs := &fakeIpvs{tc.ipvsErr, false}
   337  		versioner := &fakeIPSetVersioner{version: tc.ipsetVersion, err: tc.ipsetErr}
   338  		err := CanUseIPVSProxier(ipvs, versioner, tc.scheduler)
   339  		if (err == nil) != tc.ok {
   340  			t.Errorf("Case [%s], expect %v, got err: %v", tc.name, tc.ok, err)
   341  		}
   342  	}
   343  }
   344  
   345  func TestGetNodeIPs(t *testing.T) {
   346  	testCases := []struct {
   347  		isIPv6       bool
   348  		devAddresses map[string][]string
   349  		expectIPs    []string
   350  	}{
   351  		// case 0
   352  		{
   353  			devAddresses: map[string][]string{"eth0": {"1.2.3.4"}, "lo": {"127.0.0.1"}},
   354  			expectIPs:    []string{"1.2.3.4"},
   355  		},
   356  		// case 1
   357  		{
   358  			devAddresses: map[string][]string{"lo": {"127.0.0.1"}},
   359  			expectIPs:    []string{},
   360  		},
   361  		// case 2
   362  		{
   363  			devAddresses: map[string][]string{},
   364  			expectIPs:    []string{},
   365  		},
   366  		// case 3
   367  		{
   368  			devAddresses: map[string][]string{"encap0": {"10.20.30.40", "fe80::200:ff:fe01:1"}, "lo": {"127.0.0.1", "::1"}, "docker0": {"172.17.0.1"}},
   369  			expectIPs:    []string{"10.20.30.40", "172.17.0.1"},
   370  		},
   371  		// case 4
   372  		{
   373  			devAddresses: map[string][]string{"encaps9": {"10.20.30.40"}, "lo": {"127.0.0.1", "::1"}, "encap7": {"1000::", "10.20.30.31"}},
   374  			expectIPs:    []string{"10.20.30.40", "10.20.30.31"},
   375  		},
   376  		// case 5
   377  		{
   378  			devAddresses: map[string][]string{"kube-ipvs0": {"2000::", "1.2.3.4"}, "lo": {"127.0.0.1", "::1"}, "encap7": {"1000::", "10.20.30.31"}},
   379  			expectIPs:    []string{"10.20.30.31"},
   380  		},
   381  		// case 6
   382  		{
   383  			devAddresses: map[string][]string{"kube-ipvs0": {"1.2.3.4", "2.3.4.5"}, "lo": {"127.0.0.1", "::1"}},
   384  			expectIPs:    []string{},
   385  		},
   386  		// case 7
   387  		{
   388  			devAddresses: map[string][]string{"kube-ipvs0": {"1.2.3.4", "2.3.4.5"}},
   389  			expectIPs:    []string{},
   390  		},
   391  		// case 8
   392  		{
   393  			devAddresses: map[string][]string{"kube-ipvs0": {"1.2.3.4", "2.3.4.5"}, "eth5": {"3.4.5.6"}, "lo": {"127.0.0.1", "::1"}},
   394  			expectIPs:    []string{"3.4.5.6"},
   395  		},
   396  		// case 9
   397  		{
   398  			devAddresses: map[string][]string{"ipvs0": {"1.2.3.4"}, "lo": {"127.0.0.1", "::1"}, "encap7": {"10.20.30.31"}},
   399  			expectIPs:    []string{"10.20.30.31", "1.2.3.4"},
   400  		},
   401  		// case 10
   402  		{
   403  			isIPv6:       true,
   404  			devAddresses: map[string][]string{"ipvs0": {"1.2.3.4", "1000::"}, "lo": {"127.0.0.1", "::1"}, "encap7": {"10.20.30.31", "2000::", "fe80::200:ff:fe01:1"}},
   405  			expectIPs:    []string{"1000::", "2000::"},
   406  		},
   407  		// case 11
   408  		{
   409  			isIPv6:       true,
   410  			devAddresses: map[string][]string{"ipvs0": {"1.2.3.4", "1000::"}, "lo": {"127.0.0.1", "::1"}, "encap7": {"10.20.30.31", "2000::", "fe80::200:ff:fe01:1"}, "kube-ipvs0": {"1.2.3.4", "2.3.4.5", "2000::"}},
   411  			expectIPs:    []string{"1000::"},
   412  		},
   413  	}
   414  
   415  	for i, tc := range testCases {
   416  		fake := netlinktest.NewFakeNetlinkHandle(tc.isIPv6)
   417  		for dev, addresses := range testCases[i].devAddresses {
   418  			fake.SetLocalAddresses(dev, addresses...)
   419  		}
   420  		ips, err := fake.GetAllLocalAddresses()
   421  		if err != nil {
   422  			t.Errorf("Unexpected error: %v", err)
   423  		}
   424  		devIps, err := fake.GetLocalAddresses("kube-ipvs0")
   425  		if err != nil {
   426  			t.Errorf("Unexpected error: %v", err)
   427  		}
   428  		ips = ips.Difference(devIps)
   429  		if !ips.Equal(sets.New(tc.expectIPs...)) {
   430  			t.Errorf("case[%d], unexpected mismatch, expected: %v, got: %v", i, tc.expectIPs, ips)
   431  		}
   432  	}
   433  }
   434  
   435  func TestNodePortIPv4(t *testing.T) {
   436  	tests := []struct {
   437  		name                   string
   438  		services               []*v1.Service
   439  		endpoints              []*discovery.EndpointSlice
   440  		nodeIPs                []string
   441  		nodePortAddresses      []string
   442  		expectedIPVS           *ipvstest.FakeIPVS
   443  		expectedIPSets         netlinktest.ExpectedIPSet
   444  		expectedIptablesChains netlinktest.ExpectedIptablesChain
   445  	}{
   446  		{
   447  			name: "1 service with node port, has 2 endpoints",
   448  			services: []*v1.Service{
   449  				makeTestService("ns1", "svc1", func(svc *v1.Service) {
   450  					svc.Spec.Type = "NodePort"
   451  					svc.Spec.ClusterIP = "10.20.30.41"
   452  					svc.Spec.Ports = []v1.ServicePort{{
   453  						Name:     "p80",
   454  						Port:     int32(80),
   455  						Protocol: v1.ProtocolTCP,
   456  						NodePort: int32(3001),
   457  					}}
   458  				}),
   459  			},
   460  			endpoints: []*discovery.EndpointSlice{
   461  				makeTestEndpointSlice("ns1", "svc1", 1, func(eps *discovery.EndpointSlice) {
   462  					eps.AddressType = discovery.AddressTypeIPv4
   463  					eps.Endpoints = []discovery.Endpoint{{
   464  						Addresses: []string{"10.180.0.1"},
   465  					}}
   466  					eps.Ports = []discovery.EndpointPort{{
   467  						Name:     ptr.To("p80"),
   468  						Port:     ptr.To[int32](80),
   469  						Protocol: ptr.To(v1.ProtocolTCP),
   470  					}}
   471  				}),
   472  				makeTestEndpointSlice("ns1", "svc1", 2, func(eps *discovery.EndpointSlice) {
   473  					eps.AddressType = discovery.AddressTypeIPv6
   474  					eps.Endpoints = []discovery.Endpoint{{
   475  						Addresses: []string{"1002:ab8::2:10"},
   476  					}}
   477  					eps.Ports = []discovery.EndpointPort{{
   478  						Name:     ptr.To("p80"),
   479  						Port:     ptr.To[int32](80),
   480  						Protocol: ptr.To(v1.ProtocolTCP),
   481  					}}
   482  				}),
   483  			},
   484  			nodeIPs:           []string{"100.101.102.103", "2001:db8::1:1"},
   485  			nodePortAddresses: []string{},
   486  			expectedIPVS: &ipvstest.FakeIPVS{
   487  				Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{
   488  					{
   489  						IP:       "10.20.30.41",
   490  						Port:     80,
   491  						Protocol: "TCP",
   492  					}: {
   493  						Address:   netutils.ParseIPSloppy("10.20.30.41"),
   494  						Protocol:  "TCP",
   495  						Port:      uint16(80),
   496  						Scheduler: "rr",
   497  					},
   498  					{
   499  						IP:       "100.101.102.103",
   500  						Port:     3001,
   501  						Protocol: "TCP",
   502  					}: {
   503  						Address:   netutils.ParseIPSloppy("100.101.102.103"),
   504  						Protocol:  "TCP",
   505  						Port:      uint16(3001),
   506  						Scheduler: "rr",
   507  					},
   508  				},
   509  				Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{
   510  					{
   511  						IP:       "10.20.30.41",
   512  						Port:     80,
   513  						Protocol: "TCP",
   514  					}: {
   515  						{
   516  							Address: netutils.ParseIPSloppy("10.180.0.1"),
   517  							Port:    uint16(80),
   518  							Weight:  1,
   519  						},
   520  					},
   521  					{
   522  						IP:       "100.101.102.103",
   523  						Port:     3001,
   524  						Protocol: "TCP",
   525  					}: {
   526  						{
   527  							Address: netutils.ParseIPSloppy("10.180.0.1"),
   528  							Port:    uint16(80),
   529  							Weight:  1,
   530  						},
   531  					},
   532  				},
   533  			},
   534  		},
   535  		{
   536  			name: "1 UDP service with node port, has endpoints",
   537  			services: []*v1.Service{
   538  				makeTestService("ns1", "svc1", func(svc *v1.Service) {
   539  					svc.Spec.Type = "NodePort"
   540  					svc.Spec.ClusterIP = "10.20.30.41"
   541  					svc.Spec.Ports = []v1.ServicePort{{
   542  						Name:     "p80",
   543  						Port:     int32(80),
   544  						Protocol: v1.ProtocolUDP,
   545  						NodePort: int32(3001),
   546  					}}
   547  				}),
   548  			},
   549  			endpoints: []*discovery.EndpointSlice{
   550  				makeTestEndpointSlice("ns1", "svc1", 1, func(eps *discovery.EndpointSlice) {
   551  					eps.AddressType = discovery.AddressTypeIPv4
   552  					eps.Endpoints = []discovery.Endpoint{{
   553  						Addresses: []string{"10.180.0.1"},
   554  					}}
   555  					eps.Ports = []discovery.EndpointPort{{
   556  						Name:     ptr.To("p80"),
   557  						Port:     ptr.To[int32](80),
   558  						Protocol: ptr.To(v1.ProtocolUDP),
   559  					}}
   560  				}),
   561  			},
   562  			nodeIPs:           []string{"100.101.102.103"},
   563  			nodePortAddresses: []string{"0.0.0.0/0"},
   564  			expectedIPVS: &ipvstest.FakeIPVS{
   565  				Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{
   566  					{
   567  						IP:       "10.20.30.41",
   568  						Port:     80,
   569  						Protocol: "UDP",
   570  					}: {
   571  						Address:   netutils.ParseIPSloppy("10.20.30.41"),
   572  						Protocol:  "UDP",
   573  						Port:      uint16(80),
   574  						Scheduler: "rr",
   575  					},
   576  					{
   577  						IP:       "100.101.102.103",
   578  						Port:     3001,
   579  						Protocol: "UDP",
   580  					}: {
   581  						Address:   netutils.ParseIPSloppy("100.101.102.103"),
   582  						Protocol:  "UDP",
   583  						Port:      uint16(3001),
   584  						Scheduler: "rr",
   585  					},
   586  				},
   587  				Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{
   588  					{
   589  						IP:       "10.20.30.41",
   590  						Port:     80,
   591  						Protocol: "UDP",
   592  					}: {
   593  						{
   594  							Address: netutils.ParseIPSloppy("10.180.0.1"),
   595  							Port:    uint16(80),
   596  							Weight:  1,
   597  						},
   598  					},
   599  					{
   600  						IP:       "100.101.102.103",
   601  						Port:     3001,
   602  						Protocol: "UDP",
   603  					}: {
   604  						{
   605  							Address: netutils.ParseIPSloppy("10.180.0.1"),
   606  							Port:    uint16(80),
   607  							Weight:  1,
   608  						},
   609  					},
   610  				},
   611  			},
   612  			expectedIPSets: netlinktest.ExpectedIPSet{
   613  				kubeNodePortSetUDP: {{
   614  					Port:     3001,
   615  					Protocol: strings.ToLower(string(v1.ProtocolUDP)),
   616  					SetType:  utilipset.BitmapPort,
   617  				}},
   618  			},
   619  			expectedIptablesChains: netlinktest.ExpectedIptablesChain{
   620  				string(kubeNodePortChain): {{
   621  					JumpChain: string(kubeMarkMasqChain), MatchSet: kubeNodePortSetUDP,
   622  				}, {
   623  					JumpChain: "ACCEPT", MatchSet: kubeHealthCheckNodePortSet,
   624  				}},
   625  				string(kubeServicesChain): {{
   626  					JumpChain: "RETURN", SourceAddress: "127.0.0.0/8",
   627  				}, {
   628  					JumpChain: string(kubeMarkMasqChain), MatchSet: kubeClusterIPSet,
   629  				}, {
   630  					JumpChain: string(kubeNodePortChain), MatchSet: "",
   631  				}, {
   632  					JumpChain: "ACCEPT", MatchSet: kubeClusterIPSet,
   633  				}},
   634  			},
   635  		},
   636  		{
   637  			name: "service has node port but no endpoints",
   638  			services: []*v1.Service{
   639  				makeTestService("ns1", "svc1", func(svc *v1.Service) {
   640  					svc.Spec.Type = "NodePort"
   641  					svc.Spec.ClusterIP = "10.20.30.41"
   642  					svc.Spec.Ports = []v1.ServicePort{{
   643  						Name:     "p80",
   644  						Port:     int32(80),
   645  						Protocol: v1.ProtocolTCP,
   646  						NodePort: int32(3001),
   647  					}}
   648  				}),
   649  			},
   650  			endpoints:         []*discovery.EndpointSlice{},
   651  			nodeIPs:           []string{"100.101.102.103"},
   652  			nodePortAddresses: []string{},
   653  			expectedIPVS: &ipvstest.FakeIPVS{
   654  				Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{
   655  					{
   656  						IP:       "10.20.30.41",
   657  						Port:     80,
   658  						Protocol: "TCP",
   659  					}: {
   660  						Address:   netutils.ParseIPSloppy("10.20.30.41"),
   661  						Protocol:  "TCP",
   662  						Port:      uint16(80),
   663  						Scheduler: "rr",
   664  					},
   665  					{
   666  						IP:       "100.101.102.103",
   667  						Port:     3001,
   668  						Protocol: "TCP",
   669  					}: {
   670  						Address:   netutils.ParseIPSloppy("100.101.102.103"),
   671  						Protocol:  "TCP",
   672  						Port:      uint16(3001),
   673  						Scheduler: "rr",
   674  					},
   675  				},
   676  				Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{
   677  					{
   678  						IP:       "10.20.30.41",
   679  						Port:     80,
   680  						Protocol: "TCP",
   681  					}: {}, // no real servers corresponding to no endpoints
   682  					{
   683  						IP:       "100.101.102.103",
   684  						Port:     3001,
   685  						Protocol: "TCP",
   686  					}: {}, // no real servers corresponding to no endpoints
   687  				},
   688  			},
   689  		},
   690  		{
   691  			name: "node port service with protocol sctp on a node with multiple nodeIPs",
   692  			services: []*v1.Service{
   693  				makeTestService("ns1", "svc1", func(svc *v1.Service) {
   694  					svc.Spec.Type = "NodePort"
   695  					svc.Spec.ClusterIP = "10.20.30.41"
   696  					svc.Spec.Ports = []v1.ServicePort{{
   697  						Name:     "p80",
   698  						Port:     int32(80),
   699  						Protocol: v1.ProtocolSCTP,
   700  						NodePort: int32(3001),
   701  					}}
   702  				}),
   703  			},
   704  			endpoints: []*discovery.EndpointSlice{
   705  				makeTestEndpointSlice("ns1", "svc1", 1, func(eps *discovery.EndpointSlice) {
   706  					eps.AddressType = discovery.AddressTypeIPv4
   707  					eps.Endpoints = []discovery.Endpoint{{
   708  						Addresses: []string{"10.180.0.1"},
   709  					}}
   710  					eps.Ports = []discovery.EndpointPort{{
   711  						Name:     ptr.To("p80"),
   712  						Port:     ptr.To[int32](80),
   713  						Protocol: ptr.To(v1.ProtocolSCTP),
   714  					}}
   715  				}),
   716  			},
   717  			nodeIPs: []string{
   718  				"100.101.102.103",
   719  				"100.101.102.104",
   720  				"100.101.102.105",
   721  				"2001:db8::1:1",
   722  				"2001:db8::1:2",
   723  				"2001:db8::1:3",
   724  			},
   725  			nodePortAddresses: []string{},
   726  			expectedIPVS: &ipvstest.FakeIPVS{
   727  				Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{
   728  					{
   729  						IP:       "10.20.30.41",
   730  						Port:     80,
   731  						Protocol: "SCTP",
   732  					}: {
   733  						Address:   netutils.ParseIPSloppy("10.20.30.41"),
   734  						Protocol:  "SCTP",
   735  						Port:      uint16(80),
   736  						Scheduler: "rr",
   737  					},
   738  					{
   739  						IP:       "100.101.102.103",
   740  						Port:     3001,
   741  						Protocol: "SCTP",
   742  					}: {
   743  						Address:   netutils.ParseIPSloppy("100.101.102.103"),
   744  						Protocol:  "SCTP",
   745  						Port:      uint16(3001),
   746  						Scheduler: "rr",
   747  					},
   748  					{
   749  						IP:       "100.101.102.104",
   750  						Port:     3001,
   751  						Protocol: "SCTP",
   752  					}: {
   753  						Address:   netutils.ParseIPSloppy("100.101.102.104"),
   754  						Protocol:  "SCTP",
   755  						Port:      uint16(3001),
   756  						Scheduler: "rr",
   757  					},
   758  					{
   759  						IP:       "100.101.102.105",
   760  						Port:     3001,
   761  						Protocol: "SCTP",
   762  					}: {
   763  						Address:   netutils.ParseIPSloppy("100.101.102.105"),
   764  						Protocol:  "SCTP",
   765  						Port:      uint16(3001),
   766  						Scheduler: "rr",
   767  					},
   768  				},
   769  				Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{
   770  					{
   771  						IP:       "10.20.30.41",
   772  						Port:     80,
   773  						Protocol: "SCTP",
   774  					}: {
   775  						{
   776  							Address: netutils.ParseIPSloppy("10.180.0.1"),
   777  							Port:    uint16(80),
   778  							Weight:  1,
   779  						},
   780  					},
   781  					{
   782  						IP:       "100.101.102.103",
   783  						Port:     3001,
   784  						Protocol: "SCTP",
   785  					}: {
   786  						{
   787  							Address: netutils.ParseIPSloppy("10.180.0.1"),
   788  							Port:    uint16(80),
   789  							Weight:  1,
   790  						},
   791  					},
   792  					{
   793  						IP:       "100.101.102.104",
   794  						Port:     3001,
   795  						Protocol: "SCTP",
   796  					}: {
   797  						{
   798  							Address: netutils.ParseIPSloppy("10.180.0.1"),
   799  							Port:    uint16(80),
   800  							Weight:  1,
   801  						},
   802  					},
   803  					{
   804  						IP:       "100.101.102.105",
   805  						Port:     3001,
   806  						Protocol: "SCTP",
   807  					}: {
   808  						{
   809  							Address: netutils.ParseIPSloppy("10.180.0.1"),
   810  							Port:    uint16(80),
   811  							Weight:  1,
   812  						},
   813  					},
   814  				},
   815  			},
   816  			expectedIPSets: netlinktest.ExpectedIPSet{
   817  				kubeNodePortSetSCTP: {
   818  					{
   819  						IP:       "100.101.102.103",
   820  						Port:     3001,
   821  						Protocol: strings.ToLower(string(v1.ProtocolSCTP)),
   822  						SetType:  utilipset.HashIPPort,
   823  					},
   824  					{
   825  						IP:       "100.101.102.104",
   826  						Port:     3001,
   827  						Protocol: strings.ToLower(string(v1.ProtocolSCTP)),
   828  						SetType:  utilipset.HashIPPort,
   829  					},
   830  					{
   831  						IP:       "100.101.102.105",
   832  						Port:     3001,
   833  						Protocol: strings.ToLower(string(v1.ProtocolSCTP)),
   834  						SetType:  utilipset.HashIPPort,
   835  					},
   836  				},
   837  			},
   838  		},
   839  		{
   840  			name: "node port service with protocol sctp and externalTrafficPolicy local",
   841  			services: []*v1.Service{
   842  				makeTestService("ns1", "svc1", func(svc *v1.Service) {
   843  					svc.Spec.Type = "NodePort"
   844  					svc.Spec.ClusterIP = "10.20.30.41"
   845  					svc.Spec.Ports = []v1.ServicePort{{
   846  						Name:     "p80",
   847  						Port:     int32(80),
   848  						Protocol: v1.ProtocolSCTP,
   849  						NodePort: int32(3001),
   850  					}}
   851  					svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyLocal
   852  				}),
   853  			},
   854  			endpoints: []*discovery.EndpointSlice{
   855  				makeTestEndpointSlice("ns1", "svc1", 1, func(eps *discovery.EndpointSlice) {
   856  					eps.AddressType = discovery.AddressTypeIPv4
   857  					eps.Endpoints = []discovery.Endpoint{{
   858  						Addresses: []string{"10.180.0.1"},
   859  						NodeName:  ptr.To(testHostname),
   860  					}, {
   861  						Addresses: []string{"10.180.1.1"},
   862  						NodeName:  ptr.To("otherHost"),
   863  					}}
   864  					eps.Ports = []discovery.EndpointPort{{
   865  						Name:     ptr.To("p80"),
   866  						Port:     ptr.To[int32](80),
   867  						Protocol: ptr.To(v1.ProtocolSCTP),
   868  					}}
   869  				}),
   870  			},
   871  			nodeIPs:           []string{"100.101.102.103"},
   872  			nodePortAddresses: []string{},
   873  			expectedIPVS: &ipvstest.FakeIPVS{
   874  				Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{
   875  					{
   876  						IP:       "10.20.30.41",
   877  						Port:     80,
   878  						Protocol: "SCTP",
   879  					}: {
   880  						Address:   netutils.ParseIPSloppy("10.20.30.41"),
   881  						Protocol:  "SCTP",
   882  						Port:      uint16(80),
   883  						Scheduler: "rr",
   884  					},
   885  					{
   886  						IP:       "100.101.102.103",
   887  						Port:     3001,
   888  						Protocol: "SCTP",
   889  					}: {
   890  						Address:   netutils.ParseIPSloppy("100.101.102.103"),
   891  						Protocol:  "SCTP",
   892  						Port:      uint16(3001),
   893  						Scheduler: "rr",
   894  					},
   895  				},
   896  				Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{
   897  					{
   898  						IP:       "10.20.30.41",
   899  						Port:     80,
   900  						Protocol: "SCTP",
   901  					}: {
   902  						{
   903  							Address: netutils.ParseIPSloppy("10.180.0.1"),
   904  							Port:    uint16(80),
   905  							Weight:  1,
   906  						},
   907  						{
   908  							Address: netutils.ParseIPSloppy("10.180.1.1"),
   909  							Port:    uint16(80),
   910  							Weight:  1,
   911  						},
   912  					},
   913  					{
   914  						IP:       "100.101.102.103",
   915  						Port:     3001,
   916  						Protocol: "SCTP",
   917  					}: {
   918  						{
   919  							Address: netutils.ParseIPSloppy("10.180.0.1"),
   920  							Port:    uint16(80),
   921  							Weight:  1,
   922  						},
   923  					},
   924  				},
   925  			},
   926  			expectedIPSets: netlinktest.ExpectedIPSet{
   927  				kubeNodePortSetSCTP: {
   928  					{
   929  						IP:       "100.101.102.103",
   930  						Port:     3001,
   931  						Protocol: strings.ToLower(string(v1.ProtocolSCTP)),
   932  						SetType:  utilipset.HashIPPort,
   933  					},
   934  				},
   935  				kubeNodePortLocalSetSCTP: {
   936  					{
   937  						IP:       "100.101.102.103",
   938  						Port:     3001,
   939  						Protocol: strings.ToLower(string(v1.ProtocolSCTP)),
   940  						SetType:  utilipset.HashIPPort,
   941  					},
   942  				},
   943  			},
   944  			expectedIptablesChains: netlinktest.ExpectedIptablesChain{
   945  				string(kubeNodePortChain): {{
   946  					JumpChain: "RETURN", MatchSet: kubeNodePortLocalSetSCTP,
   947  				}, {
   948  					JumpChain: string(kubeMarkMasqChain), MatchSet: kubeNodePortSetSCTP,
   949  				}, {
   950  					JumpChain: "ACCEPT", MatchSet: kubeHealthCheckNodePortSet,
   951  				}},
   952  			},
   953  		},
   954  	}
   955  
   956  	for _, test := range tests {
   957  		t.Run(test.name, func(t *testing.T) {
   958  			ipt := iptablestest.NewFake()
   959  			ipvs := ipvstest.NewFake()
   960  			ipset := ipsettest.NewFake(testIPSetVersion)
   961  			fp := NewFakeProxier(ipt, ipvs, ipset, test.nodeIPs, nil, v1.IPv4Protocol)
   962  			fp.nodePortAddresses = proxyutil.NewNodePortAddresses(v1.IPv4Protocol, test.nodePortAddresses)
   963  
   964  			makeServiceMap(fp, test.services...)
   965  			populateEndpointSlices(fp, test.endpoints...)
   966  
   967  			fp.syncProxyRules()
   968  
   969  			if !reflect.DeepEqual(ipvs, test.expectedIPVS) {
   970  				t.Logf("actual ipvs state: %+v", ipvs)
   971  				t.Logf("expected ipvs state: %+v", test.expectedIPVS)
   972  				t.Errorf("unexpected IPVS state")
   973  			}
   974  
   975  			if test.expectedIPSets != nil {
   976  				checkIPSet(t, fp, test.expectedIPSets)
   977  			}
   978  
   979  			if test.expectedIptablesChains != nil {
   980  				checkIptables(t, ipt, test.expectedIptablesChains)
   981  			}
   982  		})
   983  	}
   984  }
   985  
   986  func TestNodePortIPv6(t *testing.T) {
   987  	tests := []struct {
   988  		name                   string
   989  		services               []*v1.Service
   990  		endpoints              []*discovery.EndpointSlice
   991  		nodeIPs                []string
   992  		nodePortAddresses      []string
   993  		expectedIPVS           *ipvstest.FakeIPVS
   994  		expectedIPSets         netlinktest.ExpectedIPSet
   995  		expectedIptablesChains netlinktest.ExpectedIptablesChain
   996  	}{
   997  		{
   998  			name: "1 service with node port, has 2 endpoints",
   999  			services: []*v1.Service{
  1000  				makeTestService("ns1", "svc1", func(svc *v1.Service) {
  1001  					svc.Spec.Type = "NodePort"
  1002  					svc.Spec.ClusterIP = "2020::1"
  1003  					svc.Spec.Ports = []v1.ServicePort{{
  1004  						Name:     "p80",
  1005  						Port:     int32(80),
  1006  						Protocol: v1.ProtocolTCP,
  1007  						NodePort: int32(3001),
  1008  					}}
  1009  				}),
  1010  			},
  1011  			endpoints: []*discovery.EndpointSlice{
  1012  				makeTestEndpointSlice("ns1", "svc1", 1, func(eps *discovery.EndpointSlice) {
  1013  					eps.AddressType = discovery.AddressTypeIPv4
  1014  					eps.Endpoints = []discovery.Endpoint{{
  1015  						Addresses: []string{"10.180.0.1"},
  1016  					}}
  1017  					eps.Ports = []discovery.EndpointPort{{
  1018  						Name:     ptr.To("p80"),
  1019  						Port:     ptr.To[int32](80),
  1020  						Protocol: ptr.To(v1.ProtocolTCP),
  1021  					}}
  1022  				}),
  1023  				makeTestEndpointSlice("ns1", "svc1", 2, func(eps *discovery.EndpointSlice) {
  1024  					eps.AddressType = discovery.AddressTypeIPv6
  1025  					eps.Endpoints = []discovery.Endpoint{{
  1026  						Addresses: []string{"1002:ab8::2:10"},
  1027  					}}
  1028  					eps.Ports = []discovery.EndpointPort{{
  1029  						Name:     ptr.To("p80"),
  1030  						Port:     ptr.To[int32](80),
  1031  						Protocol: ptr.To(v1.ProtocolTCP),
  1032  					}}
  1033  				}),
  1034  			},
  1035  			nodeIPs:           []string{"100.101.102.103", "2001:db8::1:1"},
  1036  			nodePortAddresses: []string{},
  1037  			expectedIPVS: &ipvstest.FakeIPVS{
  1038  				Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{
  1039  					{
  1040  						IP:       "2001:db8::1:1",
  1041  						Port:     3001,
  1042  						Protocol: "TCP",
  1043  					}: {
  1044  						Address:   netutils.ParseIPSloppy("2001:db8::1:1"),
  1045  						Protocol:  "TCP",
  1046  						Port:      uint16(3001),
  1047  						Scheduler: "rr",
  1048  					},
  1049  					{
  1050  						IP:       "2020::1",
  1051  						Port:     80,
  1052  						Protocol: "TCP",
  1053  					}: {
  1054  						Address:   netutils.ParseIPSloppy("2020::1"),
  1055  						Protocol:  "TCP",
  1056  						Port:      uint16(80),
  1057  						Scheduler: "rr",
  1058  					},
  1059  				},
  1060  				Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{
  1061  					{
  1062  						IP:       "2001:db8::1:1",
  1063  						Port:     3001,
  1064  						Protocol: "TCP",
  1065  					}: {
  1066  						{
  1067  							Address: netutils.ParseIPSloppy("1002:ab8::2:10"),
  1068  							Port:    uint16(80),
  1069  							Weight:  1,
  1070  						},
  1071  					},
  1072  
  1073  					{
  1074  						IP:       "2020::1",
  1075  						Port:     80,
  1076  						Protocol: "TCP",
  1077  					}: {
  1078  						{
  1079  							Address: netutils.ParseIPSloppy("1002:ab8::2:10"),
  1080  							Port:    uint16(80),
  1081  							Weight:  1,
  1082  						},
  1083  					},
  1084  				},
  1085  			},
  1086  		},
  1087  
  1088  		{
  1089  			name: "1 UDP service with node port, has endpoints (no action on IPv6 Proxier)",
  1090  			services: []*v1.Service{
  1091  				makeTestService("ns1", "svc1", func(svc *v1.Service) {
  1092  					svc.Spec.Type = "NodePort"
  1093  					svc.Spec.ClusterIP = "10.20.30.41"
  1094  					svc.Spec.Ports = []v1.ServicePort{{
  1095  						Name:     "p80",
  1096  						Port:     int32(80),
  1097  						Protocol: v1.ProtocolUDP,
  1098  						NodePort: int32(3001),
  1099  					}}
  1100  				}),
  1101  			},
  1102  			endpoints: []*discovery.EndpointSlice{
  1103  				makeTestEndpointSlice("ns1", "svc1", 1, func(eps *discovery.EndpointSlice) {
  1104  					eps.AddressType = discovery.AddressTypeIPv6
  1105  					eps.Endpoints = []discovery.Endpoint{{
  1106  						Addresses: []string{"10.180.0.1"},
  1107  					}}
  1108  					eps.Ports = []discovery.EndpointPort{{
  1109  						Name:     ptr.To("p80"),
  1110  						Port:     ptr.To[int32](80),
  1111  						Protocol: ptr.To(v1.ProtocolUDP),
  1112  					}}
  1113  				}),
  1114  			},
  1115  			nodeIPs:           []string{"100.101.102.103"},
  1116  			nodePortAddresses: []string{"0.0.0.0/0"},
  1117  			/*since this is a node with only IPv4, proxier should not do anything */
  1118  			expectedIPVS: &ipvstest.FakeIPVS{
  1119  				Services:     map[ipvstest.ServiceKey]*utilipvs.VirtualServer{},
  1120  				Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{},
  1121  			},
  1122  			expectedIPSets:         nil,
  1123  			expectedIptablesChains: nil,
  1124  		},
  1125  
  1126  		{
  1127  			name: "service has node port but no endpoints",
  1128  			services: []*v1.Service{
  1129  				makeTestService("ns1", "svc1", func(svc *v1.Service) {
  1130  					svc.Spec.Type = "NodePort"
  1131  					svc.Spec.ClusterIP = "2020::1"
  1132  					svc.Spec.Ports = []v1.ServicePort{{
  1133  						Name:     "p80",
  1134  						Port:     int32(80),
  1135  						Protocol: v1.ProtocolTCP,
  1136  						NodePort: int32(3001),
  1137  					}}
  1138  				}),
  1139  			},
  1140  			endpoints:         []*discovery.EndpointSlice{},
  1141  			nodeIPs:           []string{"100.101.102.103", "2001:db8::1:1"},
  1142  			nodePortAddresses: []string{},
  1143  			expectedIPVS: &ipvstest.FakeIPVS{
  1144  				Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{
  1145  					{
  1146  						IP:       "2001:db8::1:1",
  1147  						Port:     3001,
  1148  						Protocol: "TCP",
  1149  					}: {
  1150  						Address:   netutils.ParseIPSloppy("2001:db8::1:1"),
  1151  						Protocol:  "TCP",
  1152  						Port:      uint16(3001),
  1153  						Scheduler: "rr",
  1154  					},
  1155  					{
  1156  						IP:       "2020::1",
  1157  						Port:     80,
  1158  						Protocol: "TCP",
  1159  					}: {
  1160  						Address:   netutils.ParseIPSloppy("2020::1"),
  1161  						Protocol:  "TCP",
  1162  						Port:      uint16(80),
  1163  						Scheduler: "rr",
  1164  					},
  1165  				},
  1166  				Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{
  1167  					{
  1168  						IP:       "2020::1",
  1169  						Port:     80,
  1170  						Protocol: "TCP",
  1171  					}: {}, // no real servers corresponding to no endpoints
  1172  					{
  1173  						IP:       "2001:db8::1:1",
  1174  						Port:     3001,
  1175  						Protocol: "TCP",
  1176  					}: {}, // no real servers corresponding to no endpoints
  1177  				},
  1178  			},
  1179  		},
  1180  
  1181  		{
  1182  			name: "node port service with protocol sctp on a node with multiple nodeIPs",
  1183  			services: []*v1.Service{
  1184  				makeTestService("ns1", "svc1", func(svc *v1.Service) {
  1185  					svc.Spec.Type = "NodePort"
  1186  					svc.Spec.ClusterIP = "2020::1"
  1187  					svc.Spec.Ports = []v1.ServicePort{{
  1188  						Name:     "p80",
  1189  						Port:     int32(80),
  1190  						Protocol: v1.ProtocolSCTP,
  1191  						NodePort: int32(3001),
  1192  					}}
  1193  				}),
  1194  			},
  1195  			endpoints: []*discovery.EndpointSlice{
  1196  				makeTestEndpointSlice("ns1", "svc1", 1, func(eps *discovery.EndpointSlice) {
  1197  					eps.AddressType = discovery.AddressTypeIPv6
  1198  					eps.Endpoints = []discovery.Endpoint{{
  1199  						Addresses: []string{"2001::1"},
  1200  					}}
  1201  					eps.Ports = []discovery.EndpointPort{{
  1202  						Name:     ptr.To("p80"),
  1203  						Port:     ptr.To[int32](80),
  1204  						Protocol: ptr.To(v1.ProtocolSCTP),
  1205  					}}
  1206  				}),
  1207  			},
  1208  			nodeIPs:           []string{"2001:db8::1:1", "2001:db8::1:2"},
  1209  			nodePortAddresses: []string{},
  1210  			expectedIPVS: &ipvstest.FakeIPVS{
  1211  				Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{
  1212  					{
  1213  						IP:       "2001:db8::1:1",
  1214  						Port:     3001,
  1215  						Protocol: "SCTP",
  1216  					}: {
  1217  						Address:   netutils.ParseIPSloppy("2001:db8::1:1"),
  1218  						Protocol:  "SCTP",
  1219  						Port:      uint16(3001),
  1220  						Scheduler: "rr",
  1221  					},
  1222  					{
  1223  						IP:       "2001:db8::1:2",
  1224  						Port:     3001,
  1225  						Protocol: "SCTP",
  1226  					}: {
  1227  						Address:   netutils.ParseIPSloppy("2001:db8::1:2"),
  1228  						Protocol:  "SCTP",
  1229  						Port:      uint16(3001),
  1230  						Scheduler: "rr",
  1231  					},
  1232  					{
  1233  						IP:       "2020::1",
  1234  						Port:     80,
  1235  						Protocol: "SCTP",
  1236  					}: {
  1237  						Address:   netutils.ParseIPSloppy("2020::1"),
  1238  						Protocol:  "SCTP",
  1239  						Port:      uint16(80),
  1240  						Scheduler: "rr",
  1241  					},
  1242  				},
  1243  				Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{
  1244  					{
  1245  						IP:       "2001:db8::1:1",
  1246  						Port:     3001,
  1247  						Protocol: "SCTP",
  1248  					}: {
  1249  						{
  1250  							Address: netutils.ParseIPSloppy("2001::1"),
  1251  							Port:    uint16(80),
  1252  							Weight:  1,
  1253  						},
  1254  					},
  1255  					{
  1256  						IP:       "2001:db8::1:2",
  1257  						Port:     3001,
  1258  						Protocol: "SCTP",
  1259  					}: {
  1260  						{
  1261  							Address: netutils.ParseIPSloppy("2001::1"),
  1262  							Port:    uint16(80),
  1263  							Weight:  1,
  1264  						},
  1265  					},
  1266  					{
  1267  						IP:       "2020::1",
  1268  						Port:     80,
  1269  						Protocol: "SCTP",
  1270  					}: {
  1271  						{
  1272  							Address: netutils.ParseIPSloppy("2001::1"),
  1273  							Port:    uint16(80),
  1274  							Weight:  1,
  1275  						},
  1276  					},
  1277  				},
  1278  			},
  1279  			expectedIPSets: netlinktest.ExpectedIPSet{
  1280  				kubeNodePortSetSCTP: {
  1281  					{
  1282  						IP:       "2001:db8::1:1",
  1283  						Port:     3001,
  1284  						Protocol: strings.ToLower(string(v1.ProtocolSCTP)),
  1285  						SetType:  utilipset.HashIPPort,
  1286  					},
  1287  					{
  1288  						IP:       "2001:db8::1:2",
  1289  						Port:     3001,
  1290  						Protocol: strings.ToLower(string(v1.ProtocolSCTP)),
  1291  						SetType:  utilipset.HashIPPort,
  1292  					},
  1293  				},
  1294  			},
  1295  		},
  1296  	}
  1297  
  1298  	for _, test := range tests {
  1299  		t.Run(test.name, func(t *testing.T) {
  1300  			ipt := iptablestest.NewFake()
  1301  			ipvs := ipvstest.NewFake()
  1302  			ipset := ipsettest.NewFake(testIPSetVersion)
  1303  			fp := NewFakeProxier(ipt, ipvs, ipset, test.nodeIPs, nil, v1.IPv6Protocol)
  1304  			fp.nodePortAddresses = proxyutil.NewNodePortAddresses(v1.IPv6Protocol, test.nodePortAddresses)
  1305  
  1306  			makeServiceMap(fp, test.services...)
  1307  			populateEndpointSlices(fp, test.endpoints...)
  1308  
  1309  			fp.syncProxyRules()
  1310  
  1311  			if !reflect.DeepEqual(ipvs, test.expectedIPVS) {
  1312  				t.Logf("actual ipvs state: %+v", ipvs)
  1313  				t.Logf("expected ipvs state: %+v", test.expectedIPVS)
  1314  				t.Errorf("unexpected IPVS state")
  1315  			}
  1316  
  1317  			if test.expectedIPSets != nil {
  1318  				checkIPSet(t, fp, test.expectedIPSets)
  1319  			}
  1320  
  1321  			if test.expectedIptablesChains != nil {
  1322  				checkIptables(t, ipt, test.expectedIptablesChains)
  1323  			}
  1324  		})
  1325  	}
  1326  }
  1327  
  1328  func Test_syncEndpoint_updateWeightsOnRestart(t *testing.T) {
  1329  	ipt := iptablestest.NewFake()
  1330  	ipvs := ipvstest.NewFake()
  1331  	ipset := ipsettest.NewFake(testIPSetVersion)
  1332  	fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol)
  1333  
  1334  	svc1 := makeTestService("ns1", "svc1", func(svc *v1.Service) {
  1335  		svc.Spec.ClusterIP = "10.20.30.41"
  1336  		svc.Spec.Ports = []v1.ServicePort{{
  1337  			Name:     "p80",
  1338  			Port:     int32(80),
  1339  			Protocol: v1.ProtocolTCP,
  1340  		}}
  1341  	})
  1342  	epSlice1 := makeTestEndpointSlice("ns1", "svc1", 1, func(eps *discovery.EndpointSlice) {
  1343  		eps.AddressType = discovery.AddressTypeIPv4
  1344  		eps.Endpoints = []discovery.Endpoint{{
  1345  			Addresses: []string{"10.180.0.1"},
  1346  		}}
  1347  		eps.Ports = []discovery.EndpointPort{{
  1348  			Name:     ptr.To("p80"),
  1349  			Port:     ptr.To[int32](80),
  1350  			Protocol: ptr.To(v1.ProtocolTCP),
  1351  		}}
  1352  	})
  1353  
  1354  	// sync proxy rules to get to the desired initial state
  1355  	makeServiceMap(fp, svc1)
  1356  	makeEndpointSliceMap(fp, epSlice1)
  1357  	fp.syncProxyRules()
  1358  
  1359  	serv := &utilipvs.VirtualServer{
  1360  		Address:   netutils.ParseIPSloppy("10.20.30.41"),
  1361  		Port:      uint16(80),
  1362  		Protocol:  string(v1.ProtocolTCP),
  1363  		Scheduler: fp.ipvsScheduler,
  1364  	}
  1365  
  1366  	vs, err := fp.ipvs.GetVirtualServer(serv)
  1367  	if err != nil {
  1368  		t.Errorf("failed to get virtual server, err: %v", err)
  1369  	}
  1370  
  1371  	rss, err := fp.ipvs.GetRealServers(vs)
  1372  	if err != nil {
  1373  		t.Errorf("failed to get real servers, err: %v", err)
  1374  	}
  1375  	for _, rs := range rss {
  1376  		rs.Weight = 0
  1377  		if err = fp.ipvs.UpdateRealServer(vs, rs); err != nil {
  1378  			t.Errorf("failed to update real server: %v, err: %v", rs, err)
  1379  		}
  1380  	}
  1381  
  1382  	// simulate a restart by enabling initial sync logic.
  1383  	fp.initialSync = true
  1384  	err = fp.syncEndpoint(proxy.ServicePortName{
  1385  		NamespacedName: types.NamespacedName{
  1386  			Name:      "svc1",
  1387  			Namespace: "ns1",
  1388  		},
  1389  		Port:     "80",
  1390  		Protocol: v1.ProtocolTCP,
  1391  	}, true, vs)
  1392  	if err != nil {
  1393  		t.Errorf("failed to sync endpoint, err: %v", err)
  1394  	}
  1395  
  1396  	rss, err = fp.ipvs.GetRealServers(vs)
  1397  	if err != nil {
  1398  		t.Errorf("failed to get real server, err: %v", err)
  1399  	}
  1400  	for _, rs := range rss {
  1401  		if rs.Weight != 1 {
  1402  			t.Logf("unexpected realserver weight: %d, expected weight: 1", rs.Weight)
  1403  			t.Errorf("unexpected realserver state")
  1404  		}
  1405  	}
  1406  }
  1407  
  1408  func TestIPv4Proxier(t *testing.T) {
  1409  	tests := []struct {
  1410  		name         string
  1411  		services     []*v1.Service
  1412  		endpoints    []*discovery.EndpointSlice
  1413  		expectedIPVS *ipvstest.FakeIPVS
  1414  	}{
  1415  		{
  1416  			name: "2 services with Cluster IP, each with endpoints",
  1417  			services: []*v1.Service{
  1418  				makeTestService("ns1", "svc1", func(svc *v1.Service) {
  1419  					svc.Spec.ClusterIP = "10.20.30.41"
  1420  					svc.Spec.Ports = []v1.ServicePort{{
  1421  						Name:     "p80",
  1422  						Port:     int32(80),
  1423  						Protocol: v1.ProtocolTCP,
  1424  					}}
  1425  				}),
  1426  				makeTestService("ns2", "svc2", func(svc *v1.Service) {
  1427  					svc.Spec.ClusterIP = "1002:ab8::2:1"
  1428  					svc.Spec.Ports = []v1.ServicePort{{
  1429  						Name:     "p8080",
  1430  						Port:     int32(8080),
  1431  						Protocol: v1.ProtocolTCP,
  1432  					}}
  1433  				}),
  1434  			},
  1435  			endpoints: []*discovery.EndpointSlice{
  1436  				makeTestEndpointSlice("ns1", "svc1", 1, func(eps *discovery.EndpointSlice) {
  1437  					eps.AddressType = discovery.AddressTypeIPv4
  1438  					eps.Endpoints = []discovery.Endpoint{{
  1439  						Addresses: []string{"10.180.0.1"},
  1440  					}}
  1441  					eps.Ports = []discovery.EndpointPort{{
  1442  						Name:     ptr.To("p80"),
  1443  						Port:     ptr.To[int32](80),
  1444  						Protocol: ptr.To(v1.ProtocolTCP),
  1445  					}}
  1446  				}),
  1447  				makeTestEndpointSlice("ns2", "svc2", 1, func(eps *discovery.EndpointSlice) {
  1448  					eps.AddressType = discovery.AddressTypeIPv6
  1449  					eps.Endpoints = []discovery.Endpoint{{
  1450  						Addresses: []string{"1009:ab8::5:6"},
  1451  					}}
  1452  					eps.Ports = []discovery.EndpointPort{{
  1453  						Name:     ptr.To("p8080"),
  1454  						Port:     ptr.To[int32](8080),
  1455  						Protocol: ptr.To(v1.ProtocolTCP),
  1456  					}}
  1457  				}),
  1458  			},
  1459  			expectedIPVS: &ipvstest.FakeIPVS{
  1460  				Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{
  1461  					{
  1462  						IP:       "10.20.30.41",
  1463  						Port:     80,
  1464  						Protocol: "TCP",
  1465  					}: {
  1466  						Address:   netutils.ParseIPSloppy("10.20.30.41"),
  1467  						Protocol:  "TCP",
  1468  						Port:      uint16(80),
  1469  						Scheduler: "rr",
  1470  					},
  1471  				},
  1472  				Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{
  1473  					{
  1474  						IP:       "10.20.30.41",
  1475  						Port:     80,
  1476  						Protocol: "TCP",
  1477  					}: {
  1478  						{
  1479  							Address: netutils.ParseIPSloppy("10.180.0.1"),
  1480  							Port:    uint16(80),
  1481  							Weight:  1,
  1482  						},
  1483  					},
  1484  				},
  1485  			},
  1486  		},
  1487  		{
  1488  			name: "cluster IP service with no endpoints",
  1489  			services: []*v1.Service{
  1490  				makeTestService("ns1", "svc1", func(svc *v1.Service) {
  1491  					svc.Spec.ClusterIP = "10.20.30.41"
  1492  					svc.Spec.Ports = []v1.ServicePort{{
  1493  						Name:     "p80",
  1494  						Port:     int32(80),
  1495  						Protocol: v1.ProtocolTCP,
  1496  					}}
  1497  				}),
  1498  			},
  1499  			endpoints: []*discovery.EndpointSlice{},
  1500  			expectedIPVS: &ipvstest.FakeIPVS{
  1501  				Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{
  1502  					{
  1503  						IP:       "10.20.30.41",
  1504  						Port:     80,
  1505  						Protocol: "TCP",
  1506  					}: {
  1507  						Address:   netutils.ParseIPSloppy("10.20.30.41"),
  1508  						Protocol:  "TCP",
  1509  						Port:      uint16(80),
  1510  						Scheduler: "rr",
  1511  					},
  1512  				},
  1513  				Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{
  1514  					{
  1515  						IP:       "10.20.30.41",
  1516  						Port:     80,
  1517  						Protocol: "TCP",
  1518  					}: {},
  1519  				},
  1520  			},
  1521  		},
  1522  	}
  1523  
  1524  	for _, test := range tests {
  1525  		t.Run(test.name, func(t *testing.T) {
  1526  			ipt := iptablestest.NewFake()
  1527  			ipvs := ipvstest.NewFake()
  1528  			ipset := ipsettest.NewFake(testIPSetVersion)
  1529  			fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol)
  1530  
  1531  			makeServiceMap(fp, test.services...)
  1532  			populateEndpointSlices(fp, test.endpoints...)
  1533  
  1534  			fp.syncProxyRules()
  1535  
  1536  			if !reflect.DeepEqual(ipvs, test.expectedIPVS) {
  1537  				t.Logf("actual ipvs state: %v", ipvs)
  1538  				t.Logf("expected ipvs state: %v", test.expectedIPVS)
  1539  				t.Errorf("unexpected IPVS state")
  1540  			}
  1541  		})
  1542  	}
  1543  }
  1544  
  1545  func TestIPv6Proxier(t *testing.T) {
  1546  	tests := []struct {
  1547  		name         string
  1548  		services     []*v1.Service
  1549  		endpoints    []*discovery.EndpointSlice
  1550  		expectedIPVS *ipvstest.FakeIPVS
  1551  	}{
  1552  		{
  1553  			name: "2 services with Cluster IP, each with endpoints",
  1554  			services: []*v1.Service{
  1555  				makeTestService("ns1", "svc1", func(svc *v1.Service) {
  1556  					svc.Spec.ClusterIP = "10.20.30.41"
  1557  					svc.Spec.Ports = []v1.ServicePort{{
  1558  						Name:     "p80",
  1559  						Port:     int32(80),
  1560  						Protocol: v1.ProtocolTCP,
  1561  					}}
  1562  				}),
  1563  				makeTestService("ns2", "svc2", func(svc *v1.Service) {
  1564  					svc.Spec.ClusterIP = "1002:ab8::2:1"
  1565  					svc.Spec.Ports = []v1.ServicePort{{
  1566  						Name:     "p8080",
  1567  						Port:     int32(8080),
  1568  						Protocol: v1.ProtocolTCP,
  1569  					}}
  1570  				}),
  1571  			},
  1572  			endpoints: []*discovery.EndpointSlice{
  1573  				makeTestEndpointSlice("ns1", "svc1", 1, func(eps *discovery.EndpointSlice) {
  1574  					eps.AddressType = discovery.AddressTypeIPv4
  1575  					eps.Endpoints = []discovery.Endpoint{{
  1576  						Addresses: []string{"10.180.0.1"},
  1577  					}}
  1578  					eps.Ports = []discovery.EndpointPort{{
  1579  						Name:     ptr.To("p80"),
  1580  						Port:     ptr.To[int32](80),
  1581  						Protocol: ptr.To(v1.ProtocolTCP),
  1582  					}}
  1583  				}),
  1584  				makeTestEndpointSlice("ns2", "svc2", 1, func(eps *discovery.EndpointSlice) {
  1585  					eps.AddressType = discovery.AddressTypeIPv6
  1586  					eps.Endpoints = []discovery.Endpoint{{
  1587  						Addresses: []string{"1009:ab8::5:6"},
  1588  					}}
  1589  					eps.Ports = []discovery.EndpointPort{{
  1590  						Name:     ptr.To("p8080"),
  1591  						Port:     ptr.To[int32](8080),
  1592  						Protocol: ptr.To(v1.ProtocolTCP),
  1593  					}}
  1594  				}),
  1595  			},
  1596  			expectedIPVS: &ipvstest.FakeIPVS{
  1597  				Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{
  1598  					{
  1599  						IP:       "1002:ab8::2:1",
  1600  						Port:     8080,
  1601  						Protocol: "TCP",
  1602  					}: {
  1603  						Address:   netutils.ParseIPSloppy("1002:ab8::2:1"),
  1604  						Protocol:  "TCP",
  1605  						Port:      uint16(8080),
  1606  						Scheduler: "rr",
  1607  					},
  1608  				},
  1609  				Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{
  1610  					{
  1611  						IP:       "1002:ab8::2:1",
  1612  						Port:     8080,
  1613  						Protocol: "TCP",
  1614  					}: {
  1615  						{
  1616  							Address: netutils.ParseIPSloppy("1009:ab8::5:6"),
  1617  							Port:    uint16(8080),
  1618  							Weight:  1,
  1619  						},
  1620  					},
  1621  				},
  1622  			},
  1623  		},
  1624  		{
  1625  			name: "cluster IP service with no endpoints",
  1626  			services: []*v1.Service{
  1627  				makeTestService("ns1", "svc1", func(svc *v1.Service) {
  1628  					svc.Spec.ClusterIP = "2001::1"
  1629  					svc.Spec.Ports = []v1.ServicePort{{
  1630  						Name:     "p80",
  1631  						Port:     int32(80),
  1632  						Protocol: v1.ProtocolTCP,
  1633  					}}
  1634  				}),
  1635  			},
  1636  			endpoints: []*discovery.EndpointSlice{},
  1637  			expectedIPVS: &ipvstest.FakeIPVS{
  1638  				Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{
  1639  					{
  1640  						IP:       "2001::1",
  1641  						Port:     80,
  1642  						Protocol: "TCP",
  1643  					}: {
  1644  						Address:   netutils.ParseIPSloppy("2001::1"),
  1645  						Protocol:  "TCP",
  1646  						Port:      uint16(80),
  1647  						Scheduler: "rr",
  1648  					},
  1649  				},
  1650  				Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{
  1651  					{
  1652  						IP:       "2001::1",
  1653  						Port:     80,
  1654  						Protocol: "TCP",
  1655  					}: {},
  1656  				},
  1657  			},
  1658  		},
  1659  	}
  1660  
  1661  	for _, test := range tests {
  1662  		t.Run(test.name, func(t *testing.T) {
  1663  			ipt := iptablestest.NewFake()
  1664  			ipvs := ipvstest.NewFake()
  1665  			ipset := ipsettest.NewFake(testIPSetVersion)
  1666  			fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv6Protocol)
  1667  
  1668  			makeServiceMap(fp, test.services...)
  1669  			populateEndpointSlices(fp, test.endpoints...)
  1670  
  1671  			fp.syncProxyRules()
  1672  
  1673  			if !reflect.DeepEqual(ipvs, test.expectedIPVS) {
  1674  				t.Logf("actual ipvs state: %v", ipvs)
  1675  				t.Logf("expected ipvs state: %v", test.expectedIPVS)
  1676  				t.Errorf("unexpected IPVS state")
  1677  			}
  1678  		})
  1679  	}
  1680  }
  1681  
  1682  func TestMasqueradeRule(t *testing.T) {
  1683  	for _, testcase := range []bool{false, true} {
  1684  		ipt := iptablestest.NewFake().SetHasRandomFully(testcase)
  1685  		ipvs := ipvstest.NewFake()
  1686  		ipset := ipsettest.NewFake(testIPSetVersion)
  1687  		fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol)
  1688  		makeServiceMap(fp)
  1689  		fp.syncProxyRules()
  1690  
  1691  		buf := bytes.NewBuffer(nil)
  1692  		_ = ipt.SaveInto(utiliptables.TableNAT, buf)
  1693  		natRules := strings.Split(buf.String(), "\n")
  1694  		var hasMasqueradeJump, hasMasqRandomFully bool
  1695  		for _, line := range natRules {
  1696  			rule, _ := iptablestest.ParseRule(line, false)
  1697  			if rule != nil && rule.Chain == kubePostroutingChain && rule.Jump != nil && rule.Jump.Value == "MASQUERADE" {
  1698  				hasMasqueradeJump = true
  1699  				if rule.RandomFully != nil {
  1700  					hasMasqRandomFully = true
  1701  				}
  1702  				break
  1703  			}
  1704  		}
  1705  
  1706  		if !hasMasqueradeJump {
  1707  			t.Errorf("Failed to find -j MASQUERADE in %s chain", kubePostroutingChain)
  1708  		}
  1709  		if hasMasqRandomFully != testcase {
  1710  			probs := map[bool]string{false: "found", true: "did not find"}
  1711  			t.Errorf("%s --random-fully in -j MASQUERADE rule in %s chain for HasRandomFully()=%v", probs[testcase], kubePostroutingChain, testcase)
  1712  		}
  1713  	}
  1714  }
  1715  
  1716  func TestExternalIPsNoEndpoint(t *testing.T) {
  1717  	ipt := iptablestest.NewFake()
  1718  	ipvs := ipvstest.NewFake()
  1719  	ipset := ipsettest.NewFake(testIPSetVersion)
  1720  	fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol)
  1721  	svcIP := "10.20.30.41"
  1722  	svcPort := 80
  1723  	svcExternalIPs := "50.60.70.81"
  1724  	svcPortName := proxy.ServicePortName{
  1725  		NamespacedName: makeNSN("ns1", "svc1"),
  1726  		Port:           "p80",
  1727  	}
  1728  
  1729  	makeServiceMap(fp,
  1730  		makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) {
  1731  			svc.Spec.Type = "ClusterIP"
  1732  			svc.Spec.ClusterIP = svcIP
  1733  			svc.Spec.ExternalIPs = []string{svcExternalIPs}
  1734  			svc.Spec.Ports = []v1.ServicePort{{
  1735  				Name:       svcPortName.Port,
  1736  				Port:       int32(svcPort),
  1737  				Protocol:   v1.ProtocolTCP,
  1738  				TargetPort: intstr.FromInt32(int32(svcPort)),
  1739  			}}
  1740  		}),
  1741  	)
  1742  	fp.syncProxyRules()
  1743  
  1744  	// check ipvs service and destinations
  1745  	services, err := ipvs.GetVirtualServers()
  1746  	if err != nil {
  1747  		t.Errorf("Failed to get ipvs services, err: %v", err)
  1748  	}
  1749  	if len(services) != 2 {
  1750  		t.Errorf("Expect 2 ipvs services, got %d", len(services))
  1751  	}
  1752  	found := false
  1753  	for _, svc := range services {
  1754  		if svc.Address.String() == svcExternalIPs && svc.Port == uint16(svcPort) && svc.Protocol == string(v1.ProtocolTCP) {
  1755  			found = true
  1756  			destinations, _ := ipvs.GetRealServers(svc)
  1757  			if len(destinations) != 0 {
  1758  				t.Errorf("Unexpected %d destinations, expect 0 destinations", len(destinations))
  1759  			}
  1760  			break
  1761  		}
  1762  	}
  1763  	if !found {
  1764  		t.Errorf("Expect external ip type service, got none")
  1765  	}
  1766  }
  1767  
  1768  func TestExternalIPs(t *testing.T) {
  1769  	ipt := iptablestest.NewFake()
  1770  	ipvs := ipvstest.NewFake()
  1771  	ipset := ipsettest.NewFake(testIPSetVersion)
  1772  	fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol)
  1773  	svcIP := "10.20.30.41"
  1774  	svcPort := 80
  1775  	svcExternalIPs := sets.New[string]("50.60.70.81", "2012::51", "127.0.0.1")
  1776  	svcPortName := proxy.ServicePortName{
  1777  		NamespacedName: makeNSN("ns1", "svc1"),
  1778  		Port:           "p80",
  1779  	}
  1780  
  1781  	makeServiceMap(fp,
  1782  		makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) {
  1783  			svc.Spec.Type = "ClusterIP"
  1784  			svc.Spec.ClusterIP = svcIP
  1785  			svc.Spec.ExternalIPs = svcExternalIPs.UnsortedList()
  1786  			svc.Spec.Ports = []v1.ServicePort{{
  1787  				Name:       svcPortName.Port,
  1788  				Port:       int32(svcPort),
  1789  				Protocol:   v1.ProtocolTCP,
  1790  				TargetPort: intstr.FromInt32(int32(svcPort)),
  1791  			}}
  1792  		}),
  1793  	)
  1794  
  1795  	epIP := "10.180.0.1"
  1796  	populateEndpointSlices(fp,
  1797  		makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) {
  1798  			eps.AddressType = discovery.AddressTypeIPv4
  1799  			eps.Endpoints = []discovery.Endpoint{{
  1800  				Addresses: []string{epIP},
  1801  			}}
  1802  			eps.Ports = []discovery.EndpointPort{{
  1803  				Name:     ptr.To(svcPortName.Port),
  1804  				Port:     ptr.To(int32(svcPort)),
  1805  				Protocol: ptr.To(v1.ProtocolUDP),
  1806  			}}
  1807  		}),
  1808  	)
  1809  
  1810  	fp.syncProxyRules()
  1811  
  1812  	// check ipvs service and destinations
  1813  	services, err := ipvs.GetVirtualServers()
  1814  	if err != nil {
  1815  		t.Errorf("Failed to get ipvs services, err: %v", err)
  1816  	}
  1817  	if len(services) != 3 { // ipvs filters out by ipfamily
  1818  		t.Errorf("Expect 3 ipvs services, got %d", len(services))
  1819  	}
  1820  	found := false
  1821  	for _, svc := range services {
  1822  		if svcExternalIPs.Has(svc.Address.String()) && svc.Port == uint16(svcPort) && svc.Protocol == string(v1.ProtocolTCP) {
  1823  			found = true
  1824  			destinations, _ := ipvs.GetRealServers(svc)
  1825  			for _, dest := range destinations {
  1826  				if dest.Address.String() != epIP || dest.Port != uint16(svcPort) {
  1827  					t.Errorf("service Endpoint mismatch ipvs service destination")
  1828  				}
  1829  			}
  1830  			break
  1831  		}
  1832  	}
  1833  	if !found {
  1834  		t.Errorf("Expect external ip type service, got none")
  1835  	}
  1836  }
  1837  
  1838  func TestOnlyLocalExternalIPs(t *testing.T) {
  1839  	ipt := iptablestest.NewFake()
  1840  	ipvs := ipvstest.NewFake()
  1841  	ipset := ipsettest.NewFake(testIPSetVersion)
  1842  	fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol)
  1843  	svcIP := "10.20.30.41"
  1844  	svcPort := 80
  1845  	svcExternalIPs := sets.New[string]("50.60.70.81", "2012::51", "127.0.0.1")
  1846  	svcPortName := proxy.ServicePortName{
  1847  		NamespacedName: makeNSN("ns1", "svc1"),
  1848  		Port:           "p80",
  1849  	}
  1850  
  1851  	makeServiceMap(fp,
  1852  		makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) {
  1853  			svc.Spec.Type = "NodePort"
  1854  			svc.Spec.ClusterIP = svcIP
  1855  			svc.Spec.ExternalIPs = svcExternalIPs.UnsortedList()
  1856  			svc.Spec.Ports = []v1.ServicePort{{
  1857  				Name:       svcPortName.Port,
  1858  				Port:       int32(svcPort),
  1859  				Protocol:   v1.ProtocolTCP,
  1860  				TargetPort: intstr.FromInt32(int32(svcPort)),
  1861  			}}
  1862  			svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyLocal
  1863  		}),
  1864  	)
  1865  	epIP := "10.180.0.1"
  1866  	epIP1 := "10.180.1.1"
  1867  	thisHostname := testHostname
  1868  	otherHostname := "other-hostname"
  1869  	populateEndpointSlices(fp,
  1870  		makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) {
  1871  			eps.AddressType = discovery.AddressTypeIPv4
  1872  			eps.Endpoints = []discovery.Endpoint{{
  1873  				Addresses: []string{epIP},
  1874  				NodeName:  ptr.To(thisHostname),
  1875  			},
  1876  				{
  1877  					Addresses: []string{epIP1},
  1878  					NodeName:  ptr.To(otherHostname),
  1879  				}}
  1880  			eps.Ports = []discovery.EndpointPort{{
  1881  				Name:     ptr.To(svcPortName.Port),
  1882  				Port:     ptr.To(int32(svcPort)),
  1883  				Protocol: ptr.To(v1.ProtocolTCP),
  1884  			}}
  1885  		}),
  1886  	)
  1887  
  1888  	fp.syncProxyRules()
  1889  
  1890  	// check ipvs service and destinations
  1891  	services, err := ipvs.GetVirtualServers()
  1892  	if err != nil {
  1893  		t.Errorf("Failed to get ipvs services, err: %v", err)
  1894  	}
  1895  	if len(services) != 3 { // ipvs filters out by IPFamily
  1896  		t.Errorf("Expect 3 ipvs services, got %d", len(services))
  1897  	}
  1898  	found := false
  1899  	for _, svc := range services {
  1900  		if svcExternalIPs.Has(svc.Address.String()) && svc.Port == uint16(svcPort) && svc.Protocol == string(v1.ProtocolTCP) {
  1901  			found = true
  1902  			destinations, _ := ipvs.GetRealServers(svc)
  1903  			if len(destinations) != 1 {
  1904  				t.Errorf("Expect only 1 local endpoint. but got %v", len(destinations))
  1905  			}
  1906  			for _, dest := range destinations {
  1907  				if dest.Address.String() != epIP || dest.Port != uint16(svcPort) {
  1908  					t.Errorf("service Endpoint mismatch ipvs service destination")
  1909  				}
  1910  			}
  1911  			break
  1912  		}
  1913  	}
  1914  	if !found {
  1915  		t.Errorf("Expect external ip type service, got none")
  1916  	}
  1917  }
  1918  
  1919  func TestLoadBalancer(t *testing.T) {
  1920  	ipt, fp := buildFakeProxier()
  1921  	svcIP := "10.20.30.41"
  1922  	svcPort := 80
  1923  	svcNodePort := 3001
  1924  	svcLBIP := "1.2.3.4"
  1925  	svcPortName := proxy.ServicePortName{
  1926  		NamespacedName: makeNSN("ns1", "svc1"),
  1927  		Port:           "p80",
  1928  	}
  1929  
  1930  	makeServiceMap(fp,
  1931  		makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) {
  1932  			svc.Spec.Type = "LoadBalancer"
  1933  			svc.Spec.ClusterIP = svcIP
  1934  			svc.Spec.Ports = []v1.ServicePort{{
  1935  				Name:     svcPortName.Port,
  1936  				Port:     int32(svcPort),
  1937  				Protocol: v1.ProtocolTCP,
  1938  				NodePort: int32(svcNodePort),
  1939  			}}
  1940  			svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
  1941  				IP: svcLBIP,
  1942  			}}
  1943  		}),
  1944  	)
  1945  
  1946  	epIP := "10.180.0.1"
  1947  	populateEndpointSlices(fp,
  1948  		makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) {
  1949  			eps.AddressType = discovery.AddressTypeIPv4
  1950  			eps.Endpoints = []discovery.Endpoint{{
  1951  				Addresses: []string{epIP},
  1952  			}}
  1953  			eps.Ports = []discovery.EndpointPort{{
  1954  				Name:     ptr.To(svcPortName.Port),
  1955  				Port:     ptr.To(int32(svcPort)),
  1956  				Protocol: ptr.To(v1.ProtocolUDP),
  1957  			}}
  1958  		}),
  1959  	)
  1960  
  1961  	fp.syncProxyRules()
  1962  
  1963  	// Expect 2 services and 1 destination
  1964  	epVS := &netlinktest.ExpectedVirtualServer{
  1965  		VSNum: 2, IP: svcLBIP, Port: uint16(svcNodePort), Protocol: string(v1.ProtocolTCP),
  1966  		RS: []netlinktest.ExpectedRealServer{{
  1967  			IP: epIP, Port: uint16(svcPort),
  1968  		}}}
  1969  	checkIPVS(t, fp, epVS)
  1970  
  1971  	// check ipSet rules
  1972  	epIPSet := netlinktest.ExpectedIPSet{
  1973  		kubeLoadBalancerSet: {{
  1974  			IP:       svcLBIP,
  1975  			Port:     svcPort,
  1976  			Protocol: strings.ToLower(string(v1.ProtocolTCP)),
  1977  			SetType:  utilipset.HashIPPort,
  1978  		}},
  1979  	}
  1980  	checkIPSet(t, fp, epIPSet)
  1981  
  1982  	// Check iptables chain and rules
  1983  	epIpt := netlinktest.ExpectedIptablesChain{
  1984  		string(kubeServicesChain): {{
  1985  			JumpChain: "RETURN", SourceAddress: "127.0.0.0/8",
  1986  		}, {
  1987  			JumpChain: string(kubeLoadBalancerChain), MatchSet: kubeLoadBalancerSet,
  1988  		}, {
  1989  			JumpChain: string(kubeMarkMasqChain), MatchSet: kubeClusterIPSet,
  1990  		}, {
  1991  			JumpChain: string(kubeNodePortChain), MatchSet: "",
  1992  		}, {
  1993  			JumpChain: "ACCEPT", MatchSet: kubeClusterIPSet,
  1994  		}, {
  1995  			JumpChain: "ACCEPT", MatchSet: kubeLoadBalancerSet,
  1996  		}},
  1997  		string(kubeLoadBalancerSet): {{
  1998  			JumpChain: string(kubeMarkMasqChain), MatchSet: "",
  1999  		}},
  2000  	}
  2001  	checkIptables(t, ipt, epIpt)
  2002  }
  2003  
  2004  func TestOnlyLocalNodePorts(t *testing.T) {
  2005  	nodeIP := netutils.ParseIPSloppy("100.101.102.103")
  2006  	ipt, fp := buildFakeProxier()
  2007  
  2008  	svcIP := "10.20.30.41"
  2009  	svcPort := 80
  2010  	svcNodePort := 3001
  2011  	svcPortName := proxy.ServicePortName{
  2012  		NamespacedName: makeNSN("ns1", "svc1"),
  2013  		Port:           "p80",
  2014  	}
  2015  
  2016  	makeServiceMap(fp,
  2017  		makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) {
  2018  			svc.Spec.Type = "NodePort"
  2019  			svc.Spec.ClusterIP = svcIP
  2020  			svc.Spec.Ports = []v1.ServicePort{{
  2021  				Name:     svcPortName.Port,
  2022  				Port:     int32(svcPort),
  2023  				Protocol: v1.ProtocolTCP,
  2024  				NodePort: int32(svcNodePort),
  2025  			}}
  2026  			svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyLocal
  2027  		}),
  2028  	)
  2029  
  2030  	epIP := "10.180.0.1"
  2031  	epIP1 := "10.180.1.1"
  2032  
  2033  	populateEndpointSlices(fp,
  2034  		makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) {
  2035  			eps.AddressType = discovery.AddressTypeIPv4
  2036  			eps.Endpoints = []discovery.Endpoint{{
  2037  				Addresses: []string{epIP},
  2038  				NodeName:  ptr.To(testHostname),
  2039  			}, {
  2040  				Addresses: []string{epIP1},
  2041  				NodeName:  ptr.To("other-hostname"),
  2042  			}}
  2043  			eps.Ports = []discovery.EndpointPort{{
  2044  				Name:     ptr.To(svcPortName.Port),
  2045  				Port:     ptr.To(int32(svcPort)),
  2046  				Protocol: ptr.To(v1.ProtocolTCP),
  2047  			}}
  2048  		}),
  2049  	)
  2050  
  2051  	itf := net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}
  2052  	addrs := []net.Addr{&net.IPNet{IP: netutils.ParseIPSloppy("100.101.102.103"), Mask: net.CIDRMask(24, 32)}}
  2053  	itf1 := net.Interface{Index: 1, MTU: 0, Name: "eth1", HardwareAddr: nil, Flags: 0}
  2054  	addrs1 := []net.Addr{&net.IPNet{IP: netutils.ParseIPSloppy("2001:db8::"), Mask: net.CIDRMask(64, 128)}}
  2055  	fp.networkInterfacer.(*proxyutiltest.FakeNetwork).AddInterfaceAddr(&itf, addrs)
  2056  	fp.networkInterfacer.(*proxyutiltest.FakeNetwork).AddInterfaceAddr(&itf1, addrs1)
  2057  	fp.nodePortAddresses = proxyutil.NewNodePortAddresses(v1.IPv4Protocol, []string{"100.101.102.0/24"})
  2058  
  2059  	fp.syncProxyRules()
  2060  
  2061  	// Expect 2 services and 1 destination
  2062  	epVS := &netlinktest.ExpectedVirtualServer{
  2063  		VSNum: 2, IP: nodeIP.String(), Port: uint16(svcNodePort), Protocol: string(v1.ProtocolTCP),
  2064  		RS: []netlinktest.ExpectedRealServer{{
  2065  			IP: epIP, Port: uint16(svcPort),
  2066  		}}}
  2067  	checkIPVS(t, fp, epVS)
  2068  
  2069  	// check ipSet rules
  2070  	epEntry := &utilipset.Entry{
  2071  		Port:     svcNodePort,
  2072  		Protocol: strings.ToLower(string(v1.ProtocolTCP)),
  2073  		SetType:  utilipset.BitmapPort,
  2074  	}
  2075  	epIPSet := netlinktest.ExpectedIPSet{
  2076  		kubeNodePortSetTCP:      {epEntry},
  2077  		kubeNodePortLocalSetTCP: {epEntry},
  2078  	}
  2079  	checkIPSet(t, fp, epIPSet)
  2080  
  2081  	// Check iptables chain and rules
  2082  	epIpt := netlinktest.ExpectedIptablesChain{
  2083  		string(kubeServicesChain): {{
  2084  			JumpChain: "RETURN", SourceAddress: "127.0.0.0/8",
  2085  		}, {
  2086  			JumpChain: string(kubeMarkMasqChain), MatchSet: kubeClusterIPSet,
  2087  		}, {
  2088  			JumpChain: string(kubeNodePortChain), MatchSet: "",
  2089  		}, {
  2090  			JumpChain: "ACCEPT", MatchSet: kubeClusterIPSet,
  2091  		}},
  2092  		string(kubeNodePortChain): {{
  2093  			JumpChain: "RETURN", MatchSet: kubeNodePortLocalSetTCP,
  2094  		}, {
  2095  			JumpChain: string(kubeMarkMasqChain), MatchSet: kubeNodePortSetTCP,
  2096  		}, {
  2097  			JumpChain: "ACCEPT", MatchSet: kubeHealthCheckNodePortSet,
  2098  		}},
  2099  	}
  2100  	checkIptables(t, ipt, epIpt)
  2101  }
  2102  
  2103  func TestHealthCheckNodePort(t *testing.T) {
  2104  	ipt, fp := buildFakeProxier()
  2105  
  2106  	svcIP := "10.20.30.41"
  2107  	svcPort := 80
  2108  	svcNodePort := 3000
  2109  	svcPortName := proxy.ServicePortName{
  2110  		NamespacedName: makeNSN("ns1", "svc1"),
  2111  		Port:           "p80",
  2112  	}
  2113  
  2114  	sampleSvc := makeTestService(svcPortName.Namespace, "", func(svc *v1.Service) {
  2115  		svc.Spec.Type = "LoadBalancer"
  2116  		svc.Spec.ClusterIP = svcIP
  2117  		svc.Spec.Ports = []v1.ServicePort{{
  2118  			Name:     svcPortName.Port,
  2119  			Port:     int32(svcPort),
  2120  			Protocol: v1.ProtocolTCP,
  2121  			NodePort: int32(svcNodePort),
  2122  		}}
  2123  		svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyLocal
  2124  	})
  2125  
  2126  	svc1, svc2, invalidSvc3 := *sampleSvc, *sampleSvc, *sampleSvc
  2127  	svc1.Name, svc1.Spec.HealthCheckNodePort = "valid-svc1", 30000
  2128  	svc2.Name, svc2.Spec.HealthCheckNodePort = "valid-svc2", 30001
  2129  	// make svc3 invalid by setting external traffic policy to cluster
  2130  	invalidSvc3.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyCluster
  2131  	invalidSvc3.Name, invalidSvc3.Spec.HealthCheckNodePort = "invalid-svc3", 30002
  2132  
  2133  	makeServiceMap(fp,
  2134  		&svc1,
  2135  		&svc2,
  2136  		&invalidSvc3,
  2137  	)
  2138  
  2139  	itf := net.Interface{Index: 0, MTU: 0, Name: "eth0", HardwareAddr: nil, Flags: 0}
  2140  	addrs := []net.Addr{&net.IPNet{IP: netutils.ParseIPSloppy("100.101.102.103"), Mask: net.CIDRMask(24, 32)}}
  2141  	itf1 := net.Interface{Index: 1, MTU: 0, Name: "eth1", HardwareAddr: nil, Flags: 0}
  2142  	addrs1 := []net.Addr{&net.IPNet{IP: netutils.ParseIPSloppy("2001:db8::"), Mask: net.CIDRMask(64, 128)}}
  2143  	fp.networkInterfacer.(*proxyutiltest.FakeNetwork).AddInterfaceAddr(&itf, addrs)
  2144  	fp.networkInterfacer.(*proxyutiltest.FakeNetwork).AddInterfaceAddr(&itf1, addrs1)
  2145  	fp.nodePortAddresses = proxyutil.NewNodePortAddresses(v1.IPv4Protocol, []string{"100.101.102.0/24"})
  2146  
  2147  	fp.syncProxyRules()
  2148  
  2149  	// check ipSet rules
  2150  	makeTCPEntry := func(port int) *utilipset.Entry {
  2151  		return &utilipset.Entry{
  2152  			Port:     port,
  2153  			Protocol: strings.ToLower(string(v1.ProtocolTCP)),
  2154  			SetType:  utilipset.BitmapPort,
  2155  		}
  2156  	}
  2157  	epIPSet := netlinktest.ExpectedIPSet{
  2158  		// healthcheck node port set should only contain valid HC node ports
  2159  		kubeHealthCheckNodePortSet: {makeTCPEntry(30000), makeTCPEntry(30001)},
  2160  	}
  2161  	checkIPSet(t, fp, epIPSet)
  2162  
  2163  	// Check iptables chain and rules
  2164  	epIpt := netlinktest.ExpectedIptablesChain{
  2165  		string(kubeNodePortChain): {{
  2166  			JumpChain: "RETURN", MatchSet: kubeNodePortLocalSetTCP,
  2167  		}, {
  2168  			JumpChain: string(kubeMarkMasqChain), MatchSet: kubeNodePortSetTCP,
  2169  		}, {
  2170  			JumpChain: "ACCEPT", MatchSet: kubeHealthCheckNodePortSet,
  2171  		}},
  2172  	}
  2173  	checkIptables(t, ipt, epIpt)
  2174  }
  2175  
  2176  func TestLoadBalancerSourceRanges(t *testing.T) {
  2177  	ipt, fp := buildFakeProxier()
  2178  
  2179  	svcIP := "10.20.30.41"
  2180  	svcPort := 80
  2181  	svcLBIP := "1.2.3.4"
  2182  	svcLBSource := "10.0.0.0/8"
  2183  	svcPortName := proxy.ServicePortName{
  2184  		NamespacedName: makeNSN("ns1", "svc1"),
  2185  		Port:           "p80",
  2186  	}
  2187  	epIP := "10.180.0.1"
  2188  
  2189  	makeServiceMap(fp,
  2190  		makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) {
  2191  			svc.Spec.Type = "LoadBalancer"
  2192  			svc.Spec.ClusterIP = svcIP
  2193  			svc.Spec.Ports = []v1.ServicePort{{
  2194  				Name:     svcPortName.Port,
  2195  				Port:     int32(svcPort),
  2196  				Protocol: v1.ProtocolTCP,
  2197  			}}
  2198  			svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
  2199  				IP: svcLBIP,
  2200  			}}
  2201  			svc.Spec.LoadBalancerSourceRanges = []string{
  2202  				svcLBSource,
  2203  			}
  2204  		}),
  2205  	)
  2206  	populateEndpointSlices(fp,
  2207  		makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) {
  2208  			eps.AddressType = discovery.AddressTypeIPv4
  2209  			eps.Endpoints = []discovery.Endpoint{{
  2210  				Addresses: []string{epIP},
  2211  			}}
  2212  			eps.Ports = []discovery.EndpointPort{{
  2213  				Name:     ptr.To(svcPortName.Port),
  2214  				Port:     ptr.To(int32(svcPort)),
  2215  				Protocol: ptr.To(v1.ProtocolTCP),
  2216  			}}
  2217  		}),
  2218  	)
  2219  
  2220  	fp.syncProxyRules()
  2221  
  2222  	// Check ipvs service and destinations
  2223  	epVS := &netlinktest.ExpectedVirtualServer{
  2224  		VSNum: 2, IP: svcLBIP, Port: uint16(svcPort), Protocol: string(v1.ProtocolTCP),
  2225  		RS: []netlinktest.ExpectedRealServer{{
  2226  			IP: epIP, Port: uint16(svcPort),
  2227  		}}}
  2228  	checkIPVS(t, fp, epVS)
  2229  
  2230  	// Check ipset entry
  2231  	epIPSet := netlinktest.ExpectedIPSet{
  2232  		kubeLoadBalancerSet: {{
  2233  			IP:       svcLBIP,
  2234  			Port:     svcPort,
  2235  			Protocol: strings.ToLower(string(v1.ProtocolTCP)),
  2236  			SetType:  utilipset.HashIPPort,
  2237  		}},
  2238  		kubeLoadBalancerFWSet: {{
  2239  			IP:       svcLBIP,
  2240  			Port:     svcPort,
  2241  			Protocol: strings.ToLower(string(v1.ProtocolTCP)),
  2242  			SetType:  utilipset.HashIPPort,
  2243  		}},
  2244  		kubeLoadBalancerSourceCIDRSet: {{
  2245  			IP:       svcLBIP,
  2246  			Port:     svcPort,
  2247  			Protocol: strings.ToLower(string(v1.ProtocolTCP)),
  2248  			Net:      svcLBSource,
  2249  			SetType:  utilipset.HashIPPortNet,
  2250  		}},
  2251  	}
  2252  	checkIPSet(t, fp, epIPSet)
  2253  
  2254  	// Check iptables chain and rules
  2255  	epIpt := netlinktest.ExpectedIptablesChain{
  2256  		string(kubeServicesChain): {{
  2257  			JumpChain: "RETURN", SourceAddress: "127.0.0.0/8",
  2258  		}, {
  2259  			JumpChain: string(kubeLoadBalancerChain), MatchSet: kubeLoadBalancerSet,
  2260  		}, {
  2261  			JumpChain: string(kubeMarkMasqChain), MatchSet: kubeClusterIPSet,
  2262  		}, {
  2263  			JumpChain: string(kubeNodePortChain), MatchSet: "",
  2264  		}, {
  2265  			JumpChain: "ACCEPT", MatchSet: kubeClusterIPSet,
  2266  		}, {
  2267  			JumpChain: "ACCEPT", MatchSet: kubeLoadBalancerSet,
  2268  		}},
  2269  		string(kubeProxyFirewallChain): {{
  2270  			JumpChain: string(kubeSourceRangesFirewallChain), MatchSet: kubeLoadBalancerFWSet,
  2271  		}},
  2272  		string(kubeSourceRangesFirewallChain): {{
  2273  			JumpChain: "RETURN", MatchSet: kubeLoadBalancerSourceCIDRSet,
  2274  		}, {
  2275  			JumpChain: "DROP", MatchSet: "",
  2276  		}},
  2277  	}
  2278  	checkIptables(t, ipt, epIpt)
  2279  }
  2280  
  2281  func TestAcceptIPVSTraffic(t *testing.T) {
  2282  	ipt, fp := buildFakeProxier()
  2283  
  2284  	ingressIP := "1.2.3.4"
  2285  	externalIP := []string{"5.6.7.8"}
  2286  	svcInfos := []struct {
  2287  		svcType v1.ServiceType
  2288  		svcIP   string
  2289  		svcName string
  2290  		epIP    string
  2291  	}{
  2292  		{v1.ServiceTypeClusterIP, "10.20.30.40", "svc1", "10.180.0.1"},
  2293  		{v1.ServiceTypeLoadBalancer, "10.20.30.41", "svc2", "10.180.0.2"},
  2294  		{v1.ServiceTypeNodePort, "10.20.30.42", "svc3", "10.180.0.3"},
  2295  	}
  2296  
  2297  	for _, svcInfo := range svcInfos {
  2298  		makeServiceMap(fp,
  2299  			makeTestService("ns1", svcInfo.svcName, func(svc *v1.Service) {
  2300  				svc.Spec.Type = svcInfo.svcType
  2301  				svc.Spec.ClusterIP = svcInfo.svcIP
  2302  				svc.Spec.Ports = []v1.ServicePort{{
  2303  					Name:     "p80",
  2304  					Port:     80,
  2305  					Protocol: v1.ProtocolTCP,
  2306  					NodePort: 80,
  2307  				}}
  2308  				if svcInfo.svcType == v1.ServiceTypeLoadBalancer {
  2309  					svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
  2310  						IP: ingressIP,
  2311  					}}
  2312  				}
  2313  				if svcInfo.svcType == v1.ServiceTypeClusterIP {
  2314  					svc.Spec.ExternalIPs = externalIP
  2315  				}
  2316  			}),
  2317  		)
  2318  
  2319  		populateEndpointSlices(fp,
  2320  			makeTestEndpointSlice("ns1", "p80", 1, func(eps *discovery.EndpointSlice) {
  2321  				eps.Endpoints = []discovery.Endpoint{{
  2322  					Addresses: []string{svcInfo.epIP},
  2323  				}}
  2324  				eps.Ports = []discovery.EndpointPort{{
  2325  					Name:     ptr.To("p80"),
  2326  					Port:     ptr.To[int32](80),
  2327  					Protocol: ptr.To(v1.ProtocolUDP),
  2328  				}}
  2329  			}),
  2330  		)
  2331  	}
  2332  	fp.syncProxyRules()
  2333  
  2334  	// Check iptables chain and rules
  2335  	epIpt := netlinktest.ExpectedIptablesChain{
  2336  		string(kubeServicesChain): {
  2337  			{JumpChain: "RETURN", SourceAddress: "127.0.0.0/8"},
  2338  			{JumpChain: string(kubeLoadBalancerChain), MatchSet: kubeLoadBalancerSet},
  2339  			{JumpChain: string(kubeMarkMasqChain), MatchSet: kubeClusterIPSet},
  2340  			{JumpChain: string(kubeMarkMasqChain), MatchSet: kubeExternalIPSet},
  2341  			{JumpChain: "ACCEPT", MatchSet: kubeExternalIPSet}, // With externalTrafficOnlyArgs
  2342  			{JumpChain: "ACCEPT", MatchSet: kubeExternalIPSet}, // With dstLocalOnlyArgs
  2343  			{JumpChain: string(kubeNodePortChain), MatchSet: ""},
  2344  			{JumpChain: "ACCEPT", MatchSet: kubeClusterIPSet},
  2345  			{JumpChain: "ACCEPT", MatchSet: kubeLoadBalancerSet},
  2346  		},
  2347  	}
  2348  	checkIptables(t, ipt, epIpt)
  2349  }
  2350  
  2351  func TestOnlyLocalLoadBalancing(t *testing.T) {
  2352  	ipt, fp := buildFakeProxier()
  2353  
  2354  	svcIP := "10.20.30.41"
  2355  	svcPort := 80
  2356  	svcNodePort := 3001
  2357  	svcLBIP := "1.2.3.4"
  2358  	svcPortName := proxy.ServicePortName{
  2359  		NamespacedName: makeNSN("ns1", "svc1"),
  2360  		Port:           "p80",
  2361  	}
  2362  
  2363  	makeServiceMap(fp,
  2364  		makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) {
  2365  			svc.Spec.Type = "LoadBalancer"
  2366  			svc.Spec.ClusterIP = svcIP
  2367  			svc.Spec.Ports = []v1.ServicePort{{
  2368  				Name:     svcPortName.Port,
  2369  				Port:     int32(svcPort),
  2370  				Protocol: v1.ProtocolTCP,
  2371  				NodePort: int32(svcNodePort),
  2372  			}}
  2373  			svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
  2374  				IP: svcLBIP,
  2375  			}}
  2376  			svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyLocal
  2377  		}),
  2378  	)
  2379  
  2380  	epIP := "10.180.0.1"
  2381  	epIP1 := "10.180.1.1"
  2382  
  2383  	populateEndpointSlices(fp,
  2384  		makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) {
  2385  			eps.AddressType = discovery.AddressTypeIPv4
  2386  			eps.Endpoints = []discovery.Endpoint{
  2387  				{ // **local** endpoint address, should be added as RS
  2388  					Addresses: []string{epIP},
  2389  					NodeName:  ptr.To(testHostname),
  2390  				},
  2391  				{ // **remote** endpoint address, should not be added as RS
  2392  					Addresses: []string{epIP1},
  2393  					NodeName:  ptr.To("other-hostname"),
  2394  				}}
  2395  			eps.Ports = []discovery.EndpointPort{{
  2396  				Name:     ptr.To(svcPortName.Port),
  2397  				Port:     ptr.To(int32(svcPort)),
  2398  				Protocol: ptr.To(v1.ProtocolTCP),
  2399  			}}
  2400  		}),
  2401  	)
  2402  
  2403  	fp.syncProxyRules()
  2404  
  2405  	// Expect 2 services and 1 destination
  2406  	epVS := &netlinktest.ExpectedVirtualServer{
  2407  		VSNum: 2, IP: svcLBIP, Port: uint16(svcPort), Protocol: string(v1.ProtocolTCP),
  2408  		RS: []netlinktest.ExpectedRealServer{{
  2409  			IP: epIP, Port: uint16(svcPort),
  2410  		}}}
  2411  	checkIPVS(t, fp, epVS)
  2412  
  2413  	// check ipSet rules
  2414  	epIPSet := netlinktest.ExpectedIPSet{
  2415  		kubeLoadBalancerSet: {{
  2416  			IP:       svcLBIP,
  2417  			Port:     svcPort,
  2418  			Protocol: strings.ToLower(string(v1.ProtocolTCP)),
  2419  			SetType:  utilipset.HashIPPort,
  2420  		}},
  2421  		kubeLoadBalancerLocalSet: {{
  2422  			IP:       svcLBIP,
  2423  			Port:     svcPort,
  2424  			Protocol: strings.ToLower(string(v1.ProtocolTCP)),
  2425  			SetType:  utilipset.HashIPPort,
  2426  		}},
  2427  	}
  2428  	checkIPSet(t, fp, epIPSet)
  2429  
  2430  	// Check iptables chain and rules
  2431  	epIpt := netlinktest.ExpectedIptablesChain{
  2432  		string(kubeServicesChain): {{
  2433  			JumpChain: "RETURN", SourceAddress: "127.0.0.0/8",
  2434  		}, {
  2435  			JumpChain: string(kubeLoadBalancerChain), MatchSet: kubeLoadBalancerSet,
  2436  		}, {
  2437  			JumpChain: string(kubeMarkMasqChain), MatchSet: kubeClusterIPSet,
  2438  		}, {
  2439  			JumpChain: string(kubeNodePortChain), MatchSet: "",
  2440  		}, {
  2441  			JumpChain: "ACCEPT", MatchSet: kubeClusterIPSet,
  2442  		}, {
  2443  			JumpChain: "ACCEPT", MatchSet: kubeLoadBalancerSet,
  2444  		}},
  2445  		string(kubeLoadBalancerChain): {{
  2446  			JumpChain: "RETURN", MatchSet: kubeLoadBalancerLocalSet,
  2447  		}, {
  2448  			JumpChain: string(kubeMarkMasqChain), MatchSet: "",
  2449  		}},
  2450  	}
  2451  	checkIptables(t, ipt, epIpt)
  2452  }
  2453  
  2454  func addTestPort(array []v1.ServicePort, name string, protocol v1.Protocol, port, nodeport int32, targetPort int) []v1.ServicePort {
  2455  	svcPort := v1.ServicePort{
  2456  		Name:       name,
  2457  		Protocol:   protocol,
  2458  		Port:       port,
  2459  		NodePort:   nodeport,
  2460  		TargetPort: intstr.FromInt32(int32(targetPort)),
  2461  	}
  2462  	return append(array, svcPort)
  2463  }
  2464  
  2465  func TestBuildServiceMapAddRemove(t *testing.T) {
  2466  	ipt := iptablestest.NewFake()
  2467  	ipvs := ipvstest.NewFake()
  2468  	ipset := ipsettest.NewFake(testIPSetVersion)
  2469  	fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol)
  2470  
  2471  	services := []*v1.Service{
  2472  		makeTestService("somewhere-else", "cluster-ip", func(svc *v1.Service) {
  2473  			svc.Spec.Type = v1.ServiceTypeClusterIP
  2474  			svc.Spec.ClusterIP = "172.16.55.4"
  2475  			svc.Spec.Ports = addTestPort(svc.Spec.Ports, "something", "UDP", 1234, 4321, 0)
  2476  			svc.Spec.Ports = addTestPort(svc.Spec.Ports, "somethingelse", "UDP", 1235, 5321, 0)
  2477  			svc.Spec.Ports = addTestPort(svc.Spec.Ports, "somesctp", "SCTP", 1236, 6321, 0)
  2478  		}),
  2479  		makeTestService("somewhere-else", "node-port", func(svc *v1.Service) {
  2480  			svc.Spec.Type = v1.ServiceTypeNodePort
  2481  			svc.Spec.ClusterIP = "172.16.55.10"
  2482  			svc.Spec.Ports = addTestPort(svc.Spec.Ports, "blahblah", "UDP", 345, 678, 0)
  2483  			svc.Spec.Ports = addTestPort(svc.Spec.Ports, "moreblahblah", "TCP", 344, 677, 0)
  2484  			svc.Spec.Ports = addTestPort(svc.Spec.Ports, "sctpblah", "SCTP", 343, 676, 0)
  2485  		}),
  2486  		makeTestService("somewhere", "load-balancer", func(svc *v1.Service) {
  2487  			svc.Spec.Type = v1.ServiceTypeLoadBalancer
  2488  			svc.Spec.ClusterIP = "172.16.55.11"
  2489  			svc.Spec.LoadBalancerIP = "5.6.7.8"
  2490  			svc.Spec.Ports = addTestPort(svc.Spec.Ports, "foobar", "UDP", 8675, 30061, 7000)
  2491  			svc.Spec.Ports = addTestPort(svc.Spec.Ports, "baz", "UDP", 8676, 30062, 7001)
  2492  			svc.Spec.Ports = addTestPort(svc.Spec.Ports, "sctpfoo", "SCTP", 8677, 30063, 7002)
  2493  			svc.Status.LoadBalancer = v1.LoadBalancerStatus{
  2494  				Ingress: []v1.LoadBalancerIngress{
  2495  					{IP: "10.1.2.4"},
  2496  				},
  2497  			}
  2498  		}),
  2499  		makeTestService("somewhere", "only-local-load-balancer", func(svc *v1.Service) {
  2500  			svc.Spec.Type = v1.ServiceTypeLoadBalancer
  2501  			svc.Spec.ClusterIP = "172.16.55.12"
  2502  			svc.Spec.LoadBalancerIP = "5.6.7.8"
  2503  			svc.Spec.Ports = addTestPort(svc.Spec.Ports, "foobar2", "UDP", 8677, 30063, 7002)
  2504  			svc.Spec.Ports = addTestPort(svc.Spec.Ports, "baz", "UDP", 8678, 30064, 7003)
  2505  			svc.Spec.Ports = addTestPort(svc.Spec.Ports, "sctpbaz", "SCTP", 8679, 30065, 7004)
  2506  			svc.Status.LoadBalancer = v1.LoadBalancerStatus{
  2507  				Ingress: []v1.LoadBalancerIngress{
  2508  					{IP: "10.1.2.3"},
  2509  				},
  2510  			}
  2511  			svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyLocal
  2512  			svc.Spec.HealthCheckNodePort = 345
  2513  		}),
  2514  	}
  2515  
  2516  	for i := range services {
  2517  		fp.OnServiceAdd(services[i])
  2518  	}
  2519  	result := fp.svcPortMap.Update(fp.serviceChanges)
  2520  	if len(fp.svcPortMap) != 12 {
  2521  		t.Errorf("expected service map length 12, got %v", fp.svcPortMap)
  2522  	}
  2523  
  2524  	if len(result.DeletedUDPClusterIPs) != 0 {
  2525  		// Services only added, so nothing stale yet
  2526  		t.Errorf("expected stale UDP services length 0, got %d", len(result.DeletedUDPClusterIPs))
  2527  	}
  2528  
  2529  	// The only-local-loadbalancer ones get added
  2530  	healthCheckNodePorts := fp.svcPortMap.HealthCheckNodePorts()
  2531  	if len(healthCheckNodePorts) != 1 {
  2532  		t.Errorf("expected 1 healthcheck port, got %v", healthCheckNodePorts)
  2533  	} else {
  2534  		nsn := makeNSN("somewhere", "only-local-load-balancer")
  2535  		if port, found := healthCheckNodePorts[nsn]; !found || port != 345 {
  2536  			t.Errorf("expected healthcheck port [%q]=345: got %v", nsn, healthCheckNodePorts)
  2537  		}
  2538  	}
  2539  
  2540  	// Remove some stuff
  2541  	// oneService is a modification of services[0] with removed first port.
  2542  	oneService := makeTestService("somewhere-else", "cluster-ip", func(svc *v1.Service) {
  2543  		svc.Spec.Type = v1.ServiceTypeClusterIP
  2544  		svc.Spec.ClusterIP = "172.16.55.4"
  2545  		svc.Spec.Ports = addTestPort(svc.Spec.Ports, "somethingelse", "UDP", 1235, 5321, 0)
  2546  	})
  2547  
  2548  	fp.OnServiceUpdate(services[0], oneService)
  2549  	fp.OnServiceDelete(services[1])
  2550  	fp.OnServiceDelete(services[2])
  2551  	fp.OnServiceDelete(services[3])
  2552  
  2553  	result = fp.svcPortMap.Update(fp.serviceChanges)
  2554  	if len(fp.svcPortMap) != 1 {
  2555  		t.Errorf("expected service map length 1, got %v", fp.svcPortMap)
  2556  	}
  2557  
  2558  	// All services but one were deleted. While you'd expect only the ClusterIPs
  2559  	// from the three deleted services here, we still have the ClusterIP for
  2560  	// the not-deleted service, because one of it's ServicePorts was deleted.
  2561  	expectedStaleUDPServices := []string{"172.16.55.10", "172.16.55.4", "172.16.55.11", "172.16.55.12"}
  2562  	if len(result.DeletedUDPClusterIPs) != len(expectedStaleUDPServices) {
  2563  		t.Errorf("expected stale UDP services length %d, got %v", len(expectedStaleUDPServices), result.DeletedUDPClusterIPs.UnsortedList())
  2564  	}
  2565  	for _, ip := range expectedStaleUDPServices {
  2566  		if !result.DeletedUDPClusterIPs.Has(ip) {
  2567  			t.Errorf("expected stale UDP service service %s", ip)
  2568  		}
  2569  	}
  2570  
  2571  	healthCheckNodePorts = fp.svcPortMap.HealthCheckNodePorts()
  2572  	if len(healthCheckNodePorts) != 0 {
  2573  		t.Errorf("expected 0 healthcheck ports, got %v", healthCheckNodePorts)
  2574  	}
  2575  }
  2576  
  2577  func TestBuildServiceMapServiceHeadless(t *testing.T) {
  2578  	ipt := iptablestest.NewFake()
  2579  	ipvs := ipvstest.NewFake()
  2580  	ipset := ipsettest.NewFake(testIPSetVersion)
  2581  	fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol)
  2582  
  2583  	makeServiceMap(fp,
  2584  		makeTestService("somewhere-else", "headless", func(svc *v1.Service) {
  2585  			svc.Spec.Type = v1.ServiceTypeClusterIP
  2586  			svc.Spec.ClusterIP = v1.ClusterIPNone
  2587  			svc.Spec.Ports = addTestPort(svc.Spec.Ports, "rpc", "UDP", 1234, 0, 0)
  2588  		}),
  2589  		makeTestService("somewhere-else", "headless-without-port", func(svc *v1.Service) {
  2590  			svc.Spec.Type = v1.ServiceTypeClusterIP
  2591  			svc.Spec.ClusterIP = v1.ClusterIPNone
  2592  		}),
  2593  		makeTestService("somewhere-else", "headless-sctp", func(svc *v1.Service) {
  2594  			svc.Spec.Type = v1.ServiceTypeClusterIP
  2595  			svc.Spec.ClusterIP = v1.ClusterIPNone
  2596  			svc.Spec.Ports = addTestPort(svc.Spec.Ports, "sip", "SCTP", 1235, 0, 0)
  2597  		}),
  2598  	)
  2599  
  2600  	// Headless service should be ignored
  2601  	result := fp.svcPortMap.Update(fp.serviceChanges)
  2602  	if len(fp.svcPortMap) != 0 {
  2603  		t.Errorf("expected service map length 0, got %d", len(fp.svcPortMap))
  2604  	}
  2605  
  2606  	if len(result.DeletedUDPClusterIPs) != 0 {
  2607  		t.Errorf("expected stale UDP services length 0, got %d", len(result.DeletedUDPClusterIPs))
  2608  	}
  2609  
  2610  	// No proxied services, so no healthchecks
  2611  	healthCheckNodePorts := fp.svcPortMap.HealthCheckNodePorts()
  2612  	if len(healthCheckNodePorts) != 0 {
  2613  		t.Errorf("expected healthcheck ports length 0, got %d", len(healthCheckNodePorts))
  2614  	}
  2615  }
  2616  
  2617  func TestBuildServiceMapServiceTypeExternalName(t *testing.T) {
  2618  	ipt := iptablestest.NewFake()
  2619  	ipvs := ipvstest.NewFake()
  2620  	ipset := ipsettest.NewFake(testIPSetVersion)
  2621  	fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol)
  2622  
  2623  	makeServiceMap(fp,
  2624  		makeTestService("somewhere-else", "external-name", func(svc *v1.Service) {
  2625  			svc.Spec.Type = v1.ServiceTypeExternalName
  2626  			svc.Spec.ClusterIP = "172.16.55.4" // Should be ignored
  2627  			svc.Spec.ExternalName = "foo2.bar.com"
  2628  			svc.Spec.Ports = addTestPort(svc.Spec.Ports, "blah", "UDP", 1235, 5321, 0)
  2629  		}),
  2630  	)
  2631  
  2632  	result := fp.svcPortMap.Update(fp.serviceChanges)
  2633  	if len(fp.svcPortMap) != 0 {
  2634  		t.Errorf("expected service map length 0, got %v", fp.svcPortMap)
  2635  	}
  2636  	if len(result.DeletedUDPClusterIPs) != 0 {
  2637  		t.Errorf("expected stale UDP services length 0, got %v", result.DeletedUDPClusterIPs)
  2638  	}
  2639  
  2640  	// No proxied services, so no healthchecks
  2641  	healthCheckNodePorts := fp.svcPortMap.HealthCheckNodePorts()
  2642  	if len(healthCheckNodePorts) != 0 {
  2643  		t.Errorf("expected healthcheck ports length 0, got %v", healthCheckNodePorts)
  2644  	}
  2645  }
  2646  
  2647  func TestBuildServiceMapServiceUpdate(t *testing.T) {
  2648  	ipt := iptablestest.NewFake()
  2649  	ipvs := ipvstest.NewFake()
  2650  	ipset := ipsettest.NewFake(testIPSetVersion)
  2651  	fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol)
  2652  
  2653  	servicev1 := makeTestService("somewhere", "some-service", func(svc *v1.Service) {
  2654  		svc.Spec.Type = v1.ServiceTypeClusterIP
  2655  		svc.Spec.ClusterIP = "172.16.55.4"
  2656  		svc.Spec.Ports = addTestPort(svc.Spec.Ports, "something", "UDP", 1234, 4321, 0)
  2657  		svc.Spec.Ports = addTestPort(svc.Spec.Ports, "somethingelse", "TCP", 1235, 5321, 0)
  2658  	})
  2659  	servicev2 := makeTestService("somewhere", "some-service", func(svc *v1.Service) {
  2660  		svc.Spec.Type = v1.ServiceTypeLoadBalancer
  2661  		svc.Spec.ClusterIP = "172.16.55.4"
  2662  		svc.Spec.LoadBalancerIP = "5.6.7.8"
  2663  		svc.Spec.Ports = addTestPort(svc.Spec.Ports, "something", "UDP", 1234, 4321, 7002)
  2664  		svc.Spec.Ports = addTestPort(svc.Spec.Ports, "somethingelse", "TCP", 1235, 5321, 7003)
  2665  		svc.Status.LoadBalancer = v1.LoadBalancerStatus{
  2666  			Ingress: []v1.LoadBalancerIngress{
  2667  				{IP: "10.1.2.3"},
  2668  			},
  2669  		}
  2670  		svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyLocal
  2671  		svc.Spec.HealthCheckNodePort = 345
  2672  	})
  2673  
  2674  	fp.OnServiceAdd(servicev1)
  2675  
  2676  	result := fp.svcPortMap.Update(fp.serviceChanges)
  2677  	if len(fp.svcPortMap) != 2 {
  2678  		t.Errorf("expected service map length 2, got %v", fp.svcPortMap)
  2679  	}
  2680  	if len(result.DeletedUDPClusterIPs) != 0 {
  2681  		// Services only added, so nothing stale yet
  2682  		t.Errorf("expected stale UDP services length 0, got %d", len(result.DeletedUDPClusterIPs))
  2683  	}
  2684  
  2685  	healthCheckNodePorts := fp.svcPortMap.HealthCheckNodePorts()
  2686  	if len(healthCheckNodePorts) != 0 {
  2687  		t.Errorf("expected healthcheck ports length 0, got %v", healthCheckNodePorts)
  2688  	}
  2689  
  2690  	// Change service to load-balancer
  2691  	fp.OnServiceUpdate(servicev1, servicev2)
  2692  	result = fp.svcPortMap.Update(fp.serviceChanges)
  2693  	if len(fp.svcPortMap) != 2 {
  2694  		t.Errorf("expected service map length 2, got %v", fp.svcPortMap)
  2695  	}
  2696  	if len(result.DeletedUDPClusterIPs) != 0 {
  2697  		t.Errorf("expected stale UDP services length 0, got %v", result.DeletedUDPClusterIPs.UnsortedList())
  2698  	}
  2699  
  2700  	healthCheckNodePorts = fp.svcPortMap.HealthCheckNodePorts()
  2701  	if len(healthCheckNodePorts) != 1 {
  2702  		t.Errorf("expected healthcheck ports length 1, got %v", healthCheckNodePorts)
  2703  	}
  2704  
  2705  	// No change; make sure the service map stays the same and there are
  2706  	// no health-check changes
  2707  	fp.OnServiceUpdate(servicev2, servicev2)
  2708  	result = fp.svcPortMap.Update(fp.serviceChanges)
  2709  	if len(fp.svcPortMap) != 2 {
  2710  		t.Errorf("expected service map length 2, got %v", fp.svcPortMap)
  2711  	}
  2712  	if len(result.DeletedUDPClusterIPs) != 0 {
  2713  		t.Errorf("expected stale UDP services length 0, got %v", result.DeletedUDPClusterIPs.UnsortedList())
  2714  	}
  2715  
  2716  	healthCheckNodePorts = fp.svcPortMap.HealthCheckNodePorts()
  2717  	if len(healthCheckNodePorts) != 1 {
  2718  		t.Errorf("expected healthcheck ports length 1, got %v", healthCheckNodePorts)
  2719  	}
  2720  
  2721  	// And back to ClusterIP
  2722  	fp.OnServiceUpdate(servicev2, servicev1)
  2723  	result = fp.svcPortMap.Update(fp.serviceChanges)
  2724  	if len(fp.svcPortMap) != 2 {
  2725  		t.Errorf("expected service map length 2, got %v", fp.svcPortMap)
  2726  	}
  2727  	if len(result.DeletedUDPClusterIPs) != 0 {
  2728  		// Services only added, so nothing stale yet
  2729  		t.Errorf("expected stale UDP services length 0, got %d", len(result.DeletedUDPClusterIPs))
  2730  	}
  2731  
  2732  	healthCheckNodePorts = fp.svcPortMap.HealthCheckNodePorts()
  2733  	if len(healthCheckNodePorts) != 0 {
  2734  		t.Errorf("expected healthcheck ports length 0, got %v", healthCheckNodePorts)
  2735  	}
  2736  }
  2737  
  2738  func TestSessionAffinity(t *testing.T) {
  2739  	ipt := iptablestest.NewFake()
  2740  	ipvs := ipvstest.NewFake()
  2741  	ipset := ipsettest.NewFake(testIPSetVersion)
  2742  	nodeIP := "100.101.102.103"
  2743  	fp := NewFakeProxier(ipt, ipvs, ipset, []string{nodeIP}, nil, v1.IPv4Protocol)
  2744  	svcIP := "10.20.30.41"
  2745  	svcPort := 80
  2746  	svcNodePort := 3001
  2747  	svcExternalIPs := "50.60.70.81"
  2748  	svcPortName := proxy.ServicePortName{
  2749  		NamespacedName: makeNSN("ns1", "svc1"),
  2750  		Port:           "p80",
  2751  	}
  2752  
  2753  	makeServiceMap(fp,
  2754  		makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) {
  2755  			svc.Spec.Type = "NodePort"
  2756  			svc.Spec.ClusterIP = svcIP
  2757  			svc.Spec.ExternalIPs = []string{svcExternalIPs}
  2758  			svc.Spec.SessionAffinity = v1.ServiceAffinityClientIP
  2759  			svc.Spec.SessionAffinityConfig = &v1.SessionAffinityConfig{
  2760  				ClientIP: &v1.ClientIPConfig{
  2761  					TimeoutSeconds: ptr.To[int32](v1.DefaultClientIPServiceAffinitySeconds),
  2762  				},
  2763  			}
  2764  			svc.Spec.Ports = []v1.ServicePort{{
  2765  				Name:     svcPortName.Port,
  2766  				Port:     int32(svcPort),
  2767  				Protocol: v1.ProtocolTCP,
  2768  				NodePort: int32(svcNodePort),
  2769  			}}
  2770  		}),
  2771  	)
  2772  	fp.syncProxyRules()
  2773  
  2774  	// check ipvs service and destinations
  2775  	services, err := ipvs.GetVirtualServers()
  2776  	if err != nil {
  2777  		t.Errorf("Failed to get ipvs services, err: %v", err)
  2778  	}
  2779  	for _, svc := range services {
  2780  		if svc.Timeout != uint32(v1.DefaultClientIPServiceAffinitySeconds) {
  2781  			t.Errorf("Unexpected mismatch ipvs service session affinity timeout: %d, expected: %d", svc.Timeout, v1.DefaultClientIPServiceAffinitySeconds)
  2782  		}
  2783  	}
  2784  }
  2785  
  2786  func makeServicePortName(ns, name, port string, protocol v1.Protocol) proxy.ServicePortName {
  2787  	return proxy.ServicePortName{
  2788  		NamespacedName: makeNSN(ns, name),
  2789  		Port:           port,
  2790  		Protocol:       protocol,
  2791  	}
  2792  }
  2793  
  2794  func Test_updateEndpointsMap(t *testing.T) {
  2795  	emptyEndpointSlices := []*discovery.EndpointSlice{
  2796  		makeTestEndpointSlice("ns1", "ep1", 1, func(*discovery.EndpointSlice) {}),
  2797  	}
  2798  	subset1 := func(eps *discovery.EndpointSlice) {
  2799  		eps.AddressType = discovery.AddressTypeIPv4
  2800  		eps.Endpoints = []discovery.Endpoint{{
  2801  			Addresses: []string{"1.1.1.1"},
  2802  		}}
  2803  		eps.Ports = []discovery.EndpointPort{{
  2804  			Name:     ptr.To("p11"),
  2805  			Port:     ptr.To[int32](11),
  2806  			Protocol: ptr.To(v1.ProtocolUDP),
  2807  		}}
  2808  	}
  2809  	subset2 := func(eps *discovery.EndpointSlice) {
  2810  		eps.AddressType = discovery.AddressTypeIPv4
  2811  		eps.Endpoints = []discovery.Endpoint{{
  2812  			Addresses: []string{"1.1.1.2"},
  2813  		}}
  2814  		eps.Ports = []discovery.EndpointPort{{
  2815  			Name:     ptr.To("p12"),
  2816  			Port:     ptr.To[int32](12),
  2817  			Protocol: ptr.To(v1.ProtocolUDP),
  2818  		}}
  2819  	}
  2820  	namedPortLocal := []*discovery.EndpointSlice{
  2821  		makeTestEndpointSlice("ns1", "ep1", 1,
  2822  			func(eps *discovery.EndpointSlice) {
  2823  				eps.AddressType = discovery.AddressTypeIPv4
  2824  				eps.Endpoints = []discovery.Endpoint{{
  2825  					Addresses: []string{"1.1.1.1"},
  2826  					NodeName:  ptr.To(testHostname),
  2827  				}}
  2828  				eps.Ports = []discovery.EndpointPort{{
  2829  					Name:     ptr.To("p11"),
  2830  					Port:     ptr.To[int32](11),
  2831  					Protocol: ptr.To(v1.ProtocolUDP),
  2832  				}}
  2833  			}),
  2834  	}
  2835  	namedPort := []*discovery.EndpointSlice{
  2836  		makeTestEndpointSlice("ns1", "ep1", 1, subset1),
  2837  	}
  2838  	namedPortRenamed := []*discovery.EndpointSlice{
  2839  		makeTestEndpointSlice("ns1", "ep1", 1,
  2840  			func(eps *discovery.EndpointSlice) {
  2841  				eps.AddressType = discovery.AddressTypeIPv4
  2842  				eps.Endpoints = []discovery.Endpoint{{
  2843  					Addresses: []string{"1.1.1.1"},
  2844  				}}
  2845  				eps.Ports = []discovery.EndpointPort{{
  2846  					Name:     ptr.To("p11-2"),
  2847  					Port:     ptr.To[int32](11),
  2848  					Protocol: ptr.To(v1.ProtocolUDP),
  2849  				}}
  2850  			}),
  2851  	}
  2852  	namedPortRenumbered := []*discovery.EndpointSlice{
  2853  		makeTestEndpointSlice("ns1", "ep1", 1,
  2854  			func(eps *discovery.EndpointSlice) {
  2855  				eps.AddressType = discovery.AddressTypeIPv4
  2856  				eps.Endpoints = []discovery.Endpoint{{
  2857  					Addresses: []string{"1.1.1.1"},
  2858  				}}
  2859  				eps.Ports = []discovery.EndpointPort{{
  2860  					Name:     ptr.To("p11"),
  2861  					Port:     ptr.To[int32](22),
  2862  					Protocol: ptr.To(v1.ProtocolUDP),
  2863  				}}
  2864  			}),
  2865  	}
  2866  	namedPortsLocalNoLocal := []*discovery.EndpointSlice{
  2867  		makeTestEndpointSlice("ns1", "ep1", 1,
  2868  			func(eps *discovery.EndpointSlice) {
  2869  				eps.AddressType = discovery.AddressTypeIPv4
  2870  				eps.Endpoints = []discovery.Endpoint{{
  2871  					Addresses: []string{"1.1.1.1"},
  2872  				}, {
  2873  					Addresses: []string{"1.1.1.2"},
  2874  					NodeName:  ptr.To(testHostname),
  2875  				}}
  2876  				eps.Ports = []discovery.EndpointPort{{
  2877  					Name:     ptr.To("p11"),
  2878  					Port:     ptr.To[int32](11),
  2879  					Protocol: ptr.To(v1.ProtocolUDP),
  2880  				}, {
  2881  					Name:     ptr.To("p12"),
  2882  					Port:     ptr.To[int32](12),
  2883  					Protocol: ptr.To(v1.ProtocolUDP),
  2884  				}}
  2885  			}),
  2886  	}
  2887  	multipleSubsets := []*discovery.EndpointSlice{
  2888  		makeTestEndpointSlice("ns1", "ep1", 1, subset1),
  2889  		makeTestEndpointSlice("ns1", "ep1", 2, subset2),
  2890  	}
  2891  	subsetLocal := func(eps *discovery.EndpointSlice) {
  2892  		eps.AddressType = discovery.AddressTypeIPv4
  2893  		eps.Endpoints = []discovery.Endpoint{{
  2894  			Addresses: []string{"1.1.1.2"},
  2895  			NodeName:  ptr.To(testHostname),
  2896  		}}
  2897  		eps.Ports = []discovery.EndpointPort{{
  2898  			Name:     ptr.To("p12"),
  2899  			Port:     ptr.To[int32](12),
  2900  			Protocol: ptr.To(v1.ProtocolUDP),
  2901  		}}
  2902  	}
  2903  	multipleSubsetsWithLocal := []*discovery.EndpointSlice{
  2904  		makeTestEndpointSlice("ns1", "ep1", 1, subset1),
  2905  		makeTestEndpointSlice("ns1", "ep1", 2, subsetLocal),
  2906  	}
  2907  	subsetMultiplePortsLocal := func(eps *discovery.EndpointSlice) {
  2908  		eps.AddressType = discovery.AddressTypeIPv4
  2909  		eps.Endpoints = []discovery.Endpoint{{
  2910  			Addresses: []string{"1.1.1.1"},
  2911  			NodeName:  ptr.To(testHostname),
  2912  		}}
  2913  		eps.Ports = []discovery.EndpointPort{{
  2914  			Name:     ptr.To("p11"),
  2915  			Port:     ptr.To[int32](11),
  2916  			Protocol: ptr.To(v1.ProtocolUDP),
  2917  		}, {
  2918  			Name:     ptr.To("p12"),
  2919  			Port:     ptr.To[int32](12),
  2920  			Protocol: ptr.To(v1.ProtocolUDP),
  2921  		}}
  2922  	}
  2923  	subset3 := func(eps *discovery.EndpointSlice) {
  2924  		eps.AddressType = discovery.AddressTypeIPv4
  2925  		eps.Endpoints = []discovery.Endpoint{{
  2926  			Addresses: []string{"1.1.1.3"},
  2927  		}}
  2928  		eps.Ports = []discovery.EndpointPort{{
  2929  			Name:     ptr.To("p13"),
  2930  			Port:     ptr.To[int32](13),
  2931  			Protocol: ptr.To(v1.ProtocolUDP),
  2932  		}}
  2933  	}
  2934  	multipleSubsetsMultiplePortsLocal := []*discovery.EndpointSlice{
  2935  		makeTestEndpointSlice("ns1", "ep1", 1, subsetMultiplePortsLocal),
  2936  		makeTestEndpointSlice("ns1", "ep1", 2, subset3),
  2937  	}
  2938  	subsetMultipleIPsPorts1 := func(eps *discovery.EndpointSlice) {
  2939  		eps.AddressType = discovery.AddressTypeIPv4
  2940  		eps.Endpoints = []discovery.Endpoint{{
  2941  			Addresses: []string{"1.1.1.1"},
  2942  		}, {
  2943  			Addresses: []string{"1.1.1.2"},
  2944  			NodeName:  ptr.To(testHostname),
  2945  		}}
  2946  		eps.Ports = []discovery.EndpointPort{{
  2947  			Name:     ptr.To("p11"),
  2948  			Port:     ptr.To[int32](11),
  2949  			Protocol: ptr.To(v1.ProtocolUDP),
  2950  		}, {
  2951  			Name:     ptr.To("p12"),
  2952  			Port:     ptr.To[int32](12),
  2953  			Protocol: ptr.To(v1.ProtocolUDP),
  2954  		}}
  2955  	}
  2956  	subsetMultipleIPsPorts2 := func(eps *discovery.EndpointSlice) {
  2957  		eps.AddressType = discovery.AddressTypeIPv4
  2958  		eps.Endpoints = []discovery.Endpoint{{
  2959  			Addresses: []string{"1.1.1.3"},
  2960  		}, {
  2961  			Addresses: []string{"1.1.1.4"},
  2962  			NodeName:  ptr.To(testHostname),
  2963  		}}
  2964  		eps.Ports = []discovery.EndpointPort{{
  2965  			Name:     ptr.To("p13"),
  2966  			Port:     ptr.To[int32](13),
  2967  			Protocol: ptr.To(v1.ProtocolUDP),
  2968  		}, {
  2969  			Name:     ptr.To("p14"),
  2970  			Port:     ptr.To[int32](14),
  2971  			Protocol: ptr.To(v1.ProtocolUDP),
  2972  		}}
  2973  	}
  2974  	subsetMultipleIPsPorts3 := func(eps *discovery.EndpointSlice) {
  2975  		eps.AddressType = discovery.AddressTypeIPv4
  2976  		eps.Endpoints = []discovery.Endpoint{{
  2977  			Addresses: []string{"2.2.2.1"},
  2978  		}, {
  2979  			Addresses: []string{"2.2.2.2"},
  2980  			NodeName:  ptr.To(testHostname),
  2981  		}}
  2982  		eps.Ports = []discovery.EndpointPort{{
  2983  			Name:     ptr.To("p21"),
  2984  			Port:     ptr.To[int32](21),
  2985  			Protocol: ptr.To(v1.ProtocolUDP),
  2986  		}, {
  2987  			Name:     ptr.To("p22"),
  2988  			Port:     ptr.To[int32](22),
  2989  			Protocol: ptr.To(v1.ProtocolUDP),
  2990  		}}
  2991  	}
  2992  	multipleSubsetsIPsPorts := []*discovery.EndpointSlice{
  2993  		makeTestEndpointSlice("ns1", "ep1", 1, subsetMultipleIPsPorts1),
  2994  		makeTestEndpointSlice("ns1", "ep1", 2, subsetMultipleIPsPorts2),
  2995  		makeTestEndpointSlice("ns2", "ep2", 1, subsetMultipleIPsPorts3),
  2996  	}
  2997  	complexSubset1 := func(eps *discovery.EndpointSlice) {
  2998  		eps.AddressType = discovery.AddressTypeIPv4
  2999  		eps.Endpoints = []discovery.Endpoint{{
  3000  			Addresses: []string{"2.2.2.2"},
  3001  			NodeName:  ptr.To(testHostname),
  3002  		}, {
  3003  			Addresses: []string{"2.2.2.22"},
  3004  			NodeName:  ptr.To(testHostname),
  3005  		}}
  3006  		eps.Ports = []discovery.EndpointPort{{
  3007  			Name:     ptr.To("p22"),
  3008  			Port:     ptr.To[int32](22),
  3009  			Protocol: ptr.To(v1.ProtocolUDP),
  3010  		}}
  3011  	}
  3012  	complexSubset2 := func(eps *discovery.EndpointSlice) {
  3013  		eps.AddressType = discovery.AddressTypeIPv4
  3014  		eps.Endpoints = []discovery.Endpoint{{
  3015  			Addresses: []string{"2.2.2.3"},
  3016  			NodeName:  ptr.To(testHostname),
  3017  		}}
  3018  		eps.Ports = []discovery.EndpointPort{{
  3019  			Name:     ptr.To("p23"),
  3020  			Port:     ptr.To[int32](23),
  3021  			Protocol: ptr.To(v1.ProtocolUDP),
  3022  		}}
  3023  	}
  3024  	complexSubset3 := func(eps *discovery.EndpointSlice) {
  3025  		eps.AddressType = discovery.AddressTypeIPv4
  3026  		eps.Endpoints = []discovery.Endpoint{{
  3027  			Addresses: []string{"4.4.4.4"},
  3028  			NodeName:  ptr.To(testHostname),
  3029  		}, {
  3030  			Addresses: []string{"4.4.4.5"},
  3031  			NodeName:  ptr.To(testHostname),
  3032  		}}
  3033  		eps.Ports = []discovery.EndpointPort{{
  3034  			Name:     ptr.To("p44"),
  3035  			Port:     ptr.To[int32](44),
  3036  			Protocol: ptr.To(v1.ProtocolUDP),
  3037  		}}
  3038  	}
  3039  	complexSubset4 := func(eps *discovery.EndpointSlice) {
  3040  		eps.AddressType = discovery.AddressTypeIPv4
  3041  		eps.Endpoints = []discovery.Endpoint{{
  3042  			Addresses: []string{"4.4.4.6"},
  3043  			NodeName:  ptr.To(testHostname),
  3044  		}}
  3045  		eps.Ports = []discovery.EndpointPort{{
  3046  			Name:     ptr.To("p45"),
  3047  			Port:     ptr.To[int32](45),
  3048  			Protocol: ptr.To(v1.ProtocolUDP),
  3049  		}}
  3050  	}
  3051  	complexSubset5 := func(eps *discovery.EndpointSlice) {
  3052  		eps.AddressType = discovery.AddressTypeIPv4
  3053  		eps.Endpoints = []discovery.Endpoint{{
  3054  			Addresses: []string{"1.1.1.1"},
  3055  		}, {
  3056  			Addresses: []string{"1.1.1.11"},
  3057  		}}
  3058  		eps.Ports = []discovery.EndpointPort{{
  3059  			Name:     ptr.To("p11"),
  3060  			Port:     ptr.To[int32](11),
  3061  			Protocol: ptr.To(v1.ProtocolUDP),
  3062  		}}
  3063  	}
  3064  	complexSubset6 := func(eps *discovery.EndpointSlice) {
  3065  		eps.AddressType = discovery.AddressTypeIPv4
  3066  		eps.Endpoints = []discovery.Endpoint{{
  3067  			Addresses: []string{"1.1.1.2"},
  3068  		}}
  3069  		eps.Ports = []discovery.EndpointPort{{
  3070  			Name:     ptr.To("p12"),
  3071  			Port:     ptr.To[int32](12),
  3072  			Protocol: ptr.To(v1.ProtocolUDP),
  3073  		}, {
  3074  			Name:     ptr.To("p122"),
  3075  			Port:     ptr.To[int32](122),
  3076  			Protocol: ptr.To(v1.ProtocolUDP),
  3077  		}}
  3078  	}
  3079  	complexSubset7 := func(eps *discovery.EndpointSlice) {
  3080  		eps.AddressType = discovery.AddressTypeIPv4
  3081  		eps.Endpoints = []discovery.Endpoint{{
  3082  			Addresses: []string{"3.3.3.3"},
  3083  		}}
  3084  		eps.Ports = []discovery.EndpointPort{{
  3085  			Name:     ptr.To("p33"),
  3086  			Port:     ptr.To[int32](33),
  3087  			Protocol: ptr.To(v1.ProtocolUDP),
  3088  		}}
  3089  	}
  3090  	complexSubset8 := func(eps *discovery.EndpointSlice) {
  3091  		eps.AddressType = discovery.AddressTypeIPv4
  3092  		eps.Endpoints = []discovery.Endpoint{{
  3093  			Addresses: []string{"4.4.4.4"},
  3094  			NodeName:  ptr.To(testHostname),
  3095  		}}
  3096  		eps.Ports = []discovery.EndpointPort{{
  3097  			Name:     ptr.To("p44"),
  3098  			Port:     ptr.To[int32](44),
  3099  			Protocol: ptr.To(v1.ProtocolUDP),
  3100  		}}
  3101  	}
  3102  	complexBefore := []*discovery.EndpointSlice{
  3103  		makeTestEndpointSlice("ns1", "ep1", 1, subset1),
  3104  		nil,
  3105  		makeTestEndpointSlice("ns2", "ep2", 1, complexSubset1),
  3106  		makeTestEndpointSlice("ns2", "ep2", 2, complexSubset2),
  3107  		nil,
  3108  		makeTestEndpointSlice("ns4", "ep4", 1, complexSubset3),
  3109  		makeTestEndpointSlice("ns4", "ep4", 2, complexSubset4),
  3110  	}
  3111  	complexAfter := []*discovery.EndpointSlice{
  3112  		makeTestEndpointSlice("ns1", "ep1", 1, complexSubset5),
  3113  		makeTestEndpointSlice("ns1", "ep1", 2, complexSubset6),
  3114  		nil,
  3115  		nil,
  3116  		makeTestEndpointSlice("ns3", "ep3", 1, complexSubset7),
  3117  		makeTestEndpointSlice("ns4", "ep4", 1, complexSubset8),
  3118  		nil,
  3119  	}
  3120  
  3121  	testCases := []struct {
  3122  		// previousEndpoints and currentEndpoints are used to call appropriate
  3123  		// handlers OnEndpoints* (based on whether corresponding values are nil
  3124  		// or non-nil) and must be of equal length.
  3125  		name                           string
  3126  		previousEndpoints              []*discovery.EndpointSlice
  3127  		currentEndpoints               []*discovery.EndpointSlice
  3128  		oldEndpoints                   map[proxy.ServicePortName][]endpointExpectation
  3129  		expectedResult                 map[proxy.ServicePortName][]endpointExpectation
  3130  		expectedDeletedUDPEndpoints    []proxy.ServiceEndpoint
  3131  		expectedNewlyActiveUDPServices map[proxy.ServicePortName]bool
  3132  		expectedReadyEndpoints         map[types.NamespacedName]int
  3133  	}{{
  3134  		// Case[0]: nothing
  3135  		name:                           "nothing",
  3136  		oldEndpoints:                   map[proxy.ServicePortName][]endpointExpectation{},
  3137  		expectedResult:                 map[proxy.ServicePortName][]endpointExpectation{},
  3138  		expectedDeletedUDPEndpoints:    []proxy.ServiceEndpoint{},
  3139  		expectedNewlyActiveUDPServices: map[proxy.ServicePortName]bool{},
  3140  		expectedReadyEndpoints:         map[types.NamespacedName]int{},
  3141  	}, {
  3142  		// Case[1]: no change, named port, local
  3143  		name:              "no change, named port, local",
  3144  		previousEndpoints: namedPortLocal,
  3145  		currentEndpoints:  namedPortLocal,
  3146  		oldEndpoints: map[proxy.ServicePortName][]endpointExpectation{
  3147  			makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): {
  3148  				{endpoint: "1.1.1.1:11", isLocal: true},
  3149  			},
  3150  		},
  3151  		expectedResult: map[proxy.ServicePortName][]endpointExpectation{
  3152  			makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): {
  3153  				{endpoint: "1.1.1.1:11", isLocal: true},
  3154  			},
  3155  		},
  3156  		expectedDeletedUDPEndpoints:    []proxy.ServiceEndpoint{},
  3157  		expectedNewlyActiveUDPServices: map[proxy.ServicePortName]bool{},
  3158  		expectedReadyEndpoints: map[types.NamespacedName]int{
  3159  			makeNSN("ns1", "ep1"): 1,
  3160  		},
  3161  	}, {
  3162  		// Case[2]: no change, multiple subsets
  3163  		name:              "no change, multiple subsets",
  3164  		previousEndpoints: multipleSubsets,
  3165  		currentEndpoints:  multipleSubsets,
  3166  		oldEndpoints: map[proxy.ServicePortName][]endpointExpectation{
  3167  			makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): {
  3168  				{endpoint: "1.1.1.1:11", isLocal: false},
  3169  			},
  3170  			makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP): {
  3171  				{endpoint: "1.1.1.2:12", isLocal: false},
  3172  			},
  3173  		},
  3174  		expectedResult: map[proxy.ServicePortName][]endpointExpectation{
  3175  			makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): {
  3176  				{endpoint: "1.1.1.1:11", isLocal: false},
  3177  			},
  3178  			makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP): {
  3179  				{endpoint: "1.1.1.2:12", isLocal: false},
  3180  			},
  3181  		},
  3182  		expectedDeletedUDPEndpoints:    []proxy.ServiceEndpoint{},
  3183  		expectedNewlyActiveUDPServices: map[proxy.ServicePortName]bool{},
  3184  		expectedReadyEndpoints:         map[types.NamespacedName]int{},
  3185  	}, {
  3186  		// Case[3]: no change, multiple subsets, multiple ports, local
  3187  		name:              "no change, multiple subsets, multiple ports, local",
  3188  		previousEndpoints: multipleSubsetsMultiplePortsLocal,
  3189  		currentEndpoints:  multipleSubsetsMultiplePortsLocal,
  3190  		oldEndpoints: map[proxy.ServicePortName][]endpointExpectation{
  3191  			makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): {
  3192  				{endpoint: "1.1.1.1:11", isLocal: true},
  3193  			},
  3194  			makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP): {
  3195  				{endpoint: "1.1.1.1:12", isLocal: true},
  3196  			},
  3197  			makeServicePortName("ns1", "ep1", "p13", v1.ProtocolUDP): {
  3198  				{endpoint: "1.1.1.3:13", isLocal: false},
  3199  			},
  3200  		},
  3201  		expectedResult: map[proxy.ServicePortName][]endpointExpectation{
  3202  			makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): {
  3203  				{endpoint: "1.1.1.1:11", isLocal: true},
  3204  			},
  3205  			makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP): {
  3206  				{endpoint: "1.1.1.1:12", isLocal: true},
  3207  			},
  3208  			makeServicePortName("ns1", "ep1", "p13", v1.ProtocolUDP): {
  3209  				{endpoint: "1.1.1.3:13", isLocal: false},
  3210  			},
  3211  		},
  3212  		expectedDeletedUDPEndpoints:    []proxy.ServiceEndpoint{},
  3213  		expectedNewlyActiveUDPServices: map[proxy.ServicePortName]bool{},
  3214  		expectedReadyEndpoints: map[types.NamespacedName]int{
  3215  			makeNSN("ns1", "ep1"): 1,
  3216  		},
  3217  	}, {
  3218  		// Case[4]: no change, multiple endpoints, subsets, IPs, and ports
  3219  		name:              "no change, multiple endpoints, subsets, IPs, and ports",
  3220  		previousEndpoints: multipleSubsetsIPsPorts,
  3221  		currentEndpoints:  multipleSubsetsIPsPorts,
  3222  		oldEndpoints: map[proxy.ServicePortName][]endpointExpectation{
  3223  			makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): {
  3224  				{endpoint: "1.1.1.1:11", isLocal: false},
  3225  				{endpoint: "1.1.1.2:11", isLocal: true},
  3226  			},
  3227  			makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP): {
  3228  				{endpoint: "1.1.1.1:12", isLocal: false},
  3229  				{endpoint: "1.1.1.2:12", isLocal: true},
  3230  			},
  3231  			makeServicePortName("ns1", "ep1", "p13", v1.ProtocolUDP): {
  3232  				{endpoint: "1.1.1.3:13", isLocal: false},
  3233  				{endpoint: "1.1.1.4:13", isLocal: true},
  3234  			},
  3235  			makeServicePortName("ns1", "ep1", "p14", v1.ProtocolUDP): {
  3236  				{endpoint: "1.1.1.3:14", isLocal: false},
  3237  				{endpoint: "1.1.1.4:14", isLocal: true},
  3238  			},
  3239  			makeServicePortName("ns2", "ep2", "p21", v1.ProtocolUDP): {
  3240  				{endpoint: "2.2.2.1:21", isLocal: false},
  3241  				{endpoint: "2.2.2.2:21", isLocal: true},
  3242  			},
  3243  			makeServicePortName("ns2", "ep2", "p22", v1.ProtocolUDP): {
  3244  				{endpoint: "2.2.2.1:22", isLocal: false},
  3245  				{endpoint: "2.2.2.2:22", isLocal: true},
  3246  			},
  3247  		},
  3248  		expectedResult: map[proxy.ServicePortName][]endpointExpectation{
  3249  			makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): {
  3250  				{endpoint: "1.1.1.1:11", isLocal: false},
  3251  				{endpoint: "1.1.1.2:11", isLocal: true},
  3252  			},
  3253  			makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP): {
  3254  				{endpoint: "1.1.1.1:12", isLocal: false},
  3255  				{endpoint: "1.1.1.2:12", isLocal: true},
  3256  			},
  3257  			makeServicePortName("ns1", "ep1", "p13", v1.ProtocolUDP): {
  3258  				{endpoint: "1.1.1.3:13", isLocal: false},
  3259  				{endpoint: "1.1.1.4:13", isLocal: true},
  3260  			},
  3261  			makeServicePortName("ns1", "ep1", "p14", v1.ProtocolUDP): {
  3262  				{endpoint: "1.1.1.3:14", isLocal: false},
  3263  				{endpoint: "1.1.1.4:14", isLocal: true},
  3264  			},
  3265  			makeServicePortName("ns2", "ep2", "p21", v1.ProtocolUDP): {
  3266  				{endpoint: "2.2.2.1:21", isLocal: false},
  3267  				{endpoint: "2.2.2.2:21", isLocal: true},
  3268  			},
  3269  			makeServicePortName("ns2", "ep2", "p22", v1.ProtocolUDP): {
  3270  				{endpoint: "2.2.2.1:22", isLocal: false},
  3271  				{endpoint: "2.2.2.2:22", isLocal: true},
  3272  			},
  3273  		},
  3274  		expectedDeletedUDPEndpoints:    []proxy.ServiceEndpoint{},
  3275  		expectedNewlyActiveUDPServices: map[proxy.ServicePortName]bool{},
  3276  		expectedReadyEndpoints: map[types.NamespacedName]int{
  3277  			makeNSN("ns1", "ep1"): 2,
  3278  			makeNSN("ns2", "ep2"): 1,
  3279  		},
  3280  	}, {
  3281  		// Case[5]: add an Endpoints
  3282  		name:              "add an Endpoints",
  3283  		previousEndpoints: []*discovery.EndpointSlice{nil},
  3284  		currentEndpoints:  namedPortLocal,
  3285  		oldEndpoints:      map[proxy.ServicePortName][]endpointExpectation{},
  3286  		expectedResult: map[proxy.ServicePortName][]endpointExpectation{
  3287  			makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): {
  3288  				{endpoint: "1.1.1.1:11", isLocal: true},
  3289  			},
  3290  		},
  3291  		expectedDeletedUDPEndpoints: []proxy.ServiceEndpoint{},
  3292  		expectedNewlyActiveUDPServices: map[proxy.ServicePortName]bool{
  3293  			makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): true,
  3294  		},
  3295  		expectedReadyEndpoints: map[types.NamespacedName]int{
  3296  			makeNSN("ns1", "ep1"): 1,
  3297  		},
  3298  	}, {
  3299  		// Case[6]: remove an Endpoints
  3300  		name:              "remove an Endpoints",
  3301  		previousEndpoints: namedPortLocal,
  3302  		currentEndpoints:  []*discovery.EndpointSlice{nil},
  3303  		oldEndpoints: map[proxy.ServicePortName][]endpointExpectation{
  3304  			makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): {
  3305  				{endpoint: "1.1.1.1:11", isLocal: true},
  3306  			},
  3307  		},
  3308  		expectedResult: map[proxy.ServicePortName][]endpointExpectation{},
  3309  		expectedDeletedUDPEndpoints: []proxy.ServiceEndpoint{{
  3310  			Endpoint:        "1.1.1.1:11",
  3311  			ServicePortName: makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP),
  3312  		}},
  3313  		expectedNewlyActiveUDPServices: map[proxy.ServicePortName]bool{},
  3314  		expectedReadyEndpoints:         map[types.NamespacedName]int{},
  3315  	}, {
  3316  		// Case[7]: add an IP and port
  3317  		name:              "add an IP and port",
  3318  		previousEndpoints: namedPort,
  3319  		currentEndpoints:  namedPortsLocalNoLocal,
  3320  		oldEndpoints: map[proxy.ServicePortName][]endpointExpectation{
  3321  			makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): {
  3322  				{endpoint: "1.1.1.1:11", isLocal: false},
  3323  			},
  3324  		},
  3325  		expectedResult: map[proxy.ServicePortName][]endpointExpectation{
  3326  			makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): {
  3327  				{endpoint: "1.1.1.1:11", isLocal: false},
  3328  				{endpoint: "1.1.1.2:11", isLocal: true},
  3329  			},
  3330  			makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP): {
  3331  				{endpoint: "1.1.1.1:12", isLocal: false},
  3332  				{endpoint: "1.1.1.2:12", isLocal: true},
  3333  			},
  3334  		},
  3335  		expectedDeletedUDPEndpoints: []proxy.ServiceEndpoint{},
  3336  		expectedNewlyActiveUDPServices: map[proxy.ServicePortName]bool{
  3337  			makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP): true,
  3338  		},
  3339  		expectedReadyEndpoints: map[types.NamespacedName]int{
  3340  			makeNSN("ns1", "ep1"): 1,
  3341  		},
  3342  	}, {
  3343  		// Case[8]: remove an IP and port
  3344  		name:              "remove an IP and port",
  3345  		previousEndpoints: namedPortsLocalNoLocal,
  3346  		currentEndpoints:  namedPort,
  3347  		oldEndpoints: map[proxy.ServicePortName][]endpointExpectation{
  3348  			makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): {
  3349  				{endpoint: "1.1.1.1:11", isLocal: false},
  3350  				{endpoint: "1.1.1.2:11", isLocal: true},
  3351  			},
  3352  			makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP): {
  3353  				{endpoint: "1.1.1.1:12", isLocal: false},
  3354  				{endpoint: "1.1.1.2:12", isLocal: true},
  3355  			},
  3356  		},
  3357  		expectedResult: map[proxy.ServicePortName][]endpointExpectation{
  3358  			makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): {
  3359  				{endpoint: "1.1.1.1:11", isLocal: false},
  3360  			},
  3361  		},
  3362  		expectedDeletedUDPEndpoints: []proxy.ServiceEndpoint{{
  3363  			Endpoint:        "1.1.1.2:11",
  3364  			ServicePortName: makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP),
  3365  		}, {
  3366  			Endpoint:        "1.1.1.1:12",
  3367  			ServicePortName: makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP),
  3368  		}, {
  3369  			Endpoint:        "1.1.1.2:12",
  3370  			ServicePortName: makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP),
  3371  		}},
  3372  		expectedNewlyActiveUDPServices: map[proxy.ServicePortName]bool{},
  3373  		expectedReadyEndpoints:         map[types.NamespacedName]int{},
  3374  	}, {
  3375  		// Case[9]: add a subset
  3376  		name:              "add a subset",
  3377  		previousEndpoints: []*discovery.EndpointSlice{namedPort[0], nil},
  3378  		currentEndpoints:  multipleSubsetsWithLocal,
  3379  		oldEndpoints: map[proxy.ServicePortName][]endpointExpectation{
  3380  			makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): {
  3381  				{endpoint: "1.1.1.1:11", isLocal: false},
  3382  			},
  3383  		},
  3384  		expectedResult: map[proxy.ServicePortName][]endpointExpectation{
  3385  			makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): {
  3386  				{endpoint: "1.1.1.1:11", isLocal: false},
  3387  			},
  3388  			makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP): {
  3389  				{endpoint: "1.1.1.2:12", isLocal: true},
  3390  			},
  3391  		},
  3392  		expectedDeletedUDPEndpoints: []proxy.ServiceEndpoint{},
  3393  		expectedNewlyActiveUDPServices: map[proxy.ServicePortName]bool{
  3394  			makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP): true,
  3395  		},
  3396  		expectedReadyEndpoints: map[types.NamespacedName]int{
  3397  			makeNSN("ns1", "ep1"): 1,
  3398  		},
  3399  	}, {
  3400  		// Case[10]: remove a subset
  3401  		name:              "remove a subset",
  3402  		previousEndpoints: multipleSubsets,
  3403  		currentEndpoints:  []*discovery.EndpointSlice{namedPort[0], nil},
  3404  		oldEndpoints: map[proxy.ServicePortName][]endpointExpectation{
  3405  			makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): {
  3406  				{endpoint: "1.1.1.1:11", isLocal: false},
  3407  			},
  3408  			makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP): {
  3409  				{endpoint: "1.1.1.2:12", isLocal: false},
  3410  			},
  3411  		},
  3412  		expectedResult: map[proxy.ServicePortName][]endpointExpectation{
  3413  			makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): {
  3414  				{endpoint: "1.1.1.1:11", isLocal: false},
  3415  			},
  3416  		},
  3417  		expectedDeletedUDPEndpoints: []proxy.ServiceEndpoint{{
  3418  			Endpoint:        "1.1.1.2:12",
  3419  			ServicePortName: makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP),
  3420  		}},
  3421  		expectedNewlyActiveUDPServices: map[proxy.ServicePortName]bool{},
  3422  		expectedReadyEndpoints:         map[types.NamespacedName]int{},
  3423  	}, {
  3424  		// Case[11]: rename a port
  3425  		name:              "rename a port",
  3426  		previousEndpoints: namedPort,
  3427  		currentEndpoints:  namedPortRenamed,
  3428  		oldEndpoints: map[proxy.ServicePortName][]endpointExpectation{
  3429  			makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): {
  3430  				{endpoint: "1.1.1.1:11", isLocal: false},
  3431  			},
  3432  		},
  3433  		expectedResult: map[proxy.ServicePortName][]endpointExpectation{
  3434  			makeServicePortName("ns1", "ep1", "p11-2", v1.ProtocolUDP): {
  3435  				{endpoint: "1.1.1.1:11", isLocal: false},
  3436  			},
  3437  		},
  3438  		expectedDeletedUDPEndpoints: []proxy.ServiceEndpoint{{
  3439  			Endpoint:        "1.1.1.1:11",
  3440  			ServicePortName: makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP),
  3441  		}},
  3442  		expectedNewlyActiveUDPServices: map[proxy.ServicePortName]bool{
  3443  			makeServicePortName("ns1", "ep1", "p11-2", v1.ProtocolUDP): true,
  3444  		},
  3445  		expectedReadyEndpoints: map[types.NamespacedName]int{},
  3446  	}, {
  3447  		// Case[12]: renumber a port
  3448  		name:              "renumber a port",
  3449  		previousEndpoints: namedPort,
  3450  		currentEndpoints:  namedPortRenumbered,
  3451  		oldEndpoints: map[proxy.ServicePortName][]endpointExpectation{
  3452  			makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): {
  3453  				{endpoint: "1.1.1.1:11", isLocal: false},
  3454  			},
  3455  		},
  3456  		expectedResult: map[proxy.ServicePortName][]endpointExpectation{
  3457  			makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): {
  3458  				{endpoint: "1.1.1.1:22", isLocal: false},
  3459  			},
  3460  		},
  3461  		expectedDeletedUDPEndpoints: []proxy.ServiceEndpoint{{
  3462  			Endpoint:        "1.1.1.1:11",
  3463  			ServicePortName: makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP),
  3464  		}},
  3465  		expectedNewlyActiveUDPServices: map[proxy.ServicePortName]bool{},
  3466  		expectedReadyEndpoints:         map[types.NamespacedName]int{},
  3467  	}, {
  3468  		// Case[13]: complex add and remove
  3469  		name:              "complex add and remove",
  3470  		previousEndpoints: complexBefore,
  3471  		currentEndpoints:  complexAfter,
  3472  		oldEndpoints: map[proxy.ServicePortName][]endpointExpectation{
  3473  			makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): {
  3474  				{endpoint: "1.1.1.1:11", isLocal: false},
  3475  			},
  3476  			makeServicePortName("ns2", "ep2", "p22", v1.ProtocolUDP): {
  3477  				{endpoint: "2.2.2.22:22", isLocal: true},
  3478  				{endpoint: "2.2.2.2:22", isLocal: true},
  3479  			},
  3480  			makeServicePortName("ns2", "ep2", "p23", v1.ProtocolUDP): {
  3481  				{endpoint: "2.2.2.3:23", isLocal: true},
  3482  			},
  3483  			makeServicePortName("ns4", "ep4", "p44", v1.ProtocolUDP): {
  3484  				{endpoint: "4.4.4.4:44", isLocal: true},
  3485  				{endpoint: "4.4.4.5:44", isLocal: true},
  3486  			},
  3487  			makeServicePortName("ns4", "ep4", "p45", v1.ProtocolUDP): {
  3488  				{endpoint: "4.4.4.6:45", isLocal: true},
  3489  			},
  3490  		},
  3491  		expectedResult: map[proxy.ServicePortName][]endpointExpectation{
  3492  			makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): {
  3493  				{endpoint: "1.1.1.11:11", isLocal: false},
  3494  				{endpoint: "1.1.1.1:11", isLocal: false},
  3495  			},
  3496  			makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP): {
  3497  				{endpoint: "1.1.1.2:12", isLocal: false},
  3498  			},
  3499  			makeServicePortName("ns1", "ep1", "p122", v1.ProtocolUDP): {
  3500  				{endpoint: "1.1.1.2:122", isLocal: false},
  3501  			},
  3502  			makeServicePortName("ns3", "ep3", "p33", v1.ProtocolUDP): {
  3503  				{endpoint: "3.3.3.3:33", isLocal: false},
  3504  			},
  3505  			makeServicePortName("ns4", "ep4", "p44", v1.ProtocolUDP): {
  3506  				{endpoint: "4.4.4.4:44", isLocal: true},
  3507  			},
  3508  		},
  3509  		expectedDeletedUDPEndpoints: []proxy.ServiceEndpoint{{
  3510  			Endpoint:        "2.2.2.2:22",
  3511  			ServicePortName: makeServicePortName("ns2", "ep2", "p22", v1.ProtocolUDP),
  3512  		}, {
  3513  			Endpoint:        "2.2.2.22:22",
  3514  			ServicePortName: makeServicePortName("ns2", "ep2", "p22", v1.ProtocolUDP),
  3515  		}, {
  3516  			Endpoint:        "2.2.2.3:23",
  3517  			ServicePortName: makeServicePortName("ns2", "ep2", "p23", v1.ProtocolUDP),
  3518  		}, {
  3519  			Endpoint:        "4.4.4.5:44",
  3520  			ServicePortName: makeServicePortName("ns4", "ep4", "p44", v1.ProtocolUDP),
  3521  		}, {
  3522  			Endpoint:        "4.4.4.6:45",
  3523  			ServicePortName: makeServicePortName("ns4", "ep4", "p45", v1.ProtocolUDP),
  3524  		}},
  3525  		expectedNewlyActiveUDPServices: map[proxy.ServicePortName]bool{
  3526  			makeServicePortName("ns1", "ep1", "p12", v1.ProtocolUDP):  true,
  3527  			makeServicePortName("ns1", "ep1", "p122", v1.ProtocolUDP): true,
  3528  			makeServicePortName("ns3", "ep3", "p33", v1.ProtocolUDP):  true,
  3529  		},
  3530  		expectedReadyEndpoints: map[types.NamespacedName]int{
  3531  			makeNSN("ns4", "ep4"): 1,
  3532  		},
  3533  	}, {
  3534  		// Case[14]: change from 0 endpoint address to 1 named port
  3535  		name:              "change from 0 endpoint address to 1 named port",
  3536  		previousEndpoints: emptyEndpointSlices,
  3537  		currentEndpoints:  namedPort,
  3538  		oldEndpoints:      map[proxy.ServicePortName][]endpointExpectation{},
  3539  		expectedResult: map[proxy.ServicePortName][]endpointExpectation{
  3540  			makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): {
  3541  				{endpoint: "1.1.1.1:11", isLocal: false},
  3542  			},
  3543  		},
  3544  		expectedDeletedUDPEndpoints: []proxy.ServiceEndpoint{},
  3545  		expectedNewlyActiveUDPServices: map[proxy.ServicePortName]bool{
  3546  			makeServicePortName("ns1", "ep1", "p11", v1.ProtocolUDP): true,
  3547  		},
  3548  		expectedReadyEndpoints: map[types.NamespacedName]int{},
  3549  	},
  3550  	}
  3551  
  3552  	for tci, tc := range testCases {
  3553  		t.Run(tc.name, func(t *testing.T) {
  3554  			ipt := iptablestest.NewFake()
  3555  			ipvs := ipvstest.NewFake()
  3556  			ipset := ipsettest.NewFake(testIPSetVersion)
  3557  			fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol)
  3558  			fp.hostname = testHostname
  3559  
  3560  			// First check that after adding all previous versions of endpoints,
  3561  			// the fp.oldEndpoints is as we expect.
  3562  			for i := range tc.previousEndpoints {
  3563  				if tc.previousEndpoints[i] != nil {
  3564  					fp.OnEndpointSliceAdd(tc.previousEndpoints[i])
  3565  				}
  3566  			}
  3567  			fp.endpointsMap.Update(fp.endpointsChanges)
  3568  			checkEndpointExpectations(t, tci, fp.endpointsMap, tc.oldEndpoints)
  3569  
  3570  			// Now let's call appropriate handlers to get to state we want to be.
  3571  			if len(tc.previousEndpoints) != len(tc.currentEndpoints) {
  3572  				t.Fatalf("[%d] different lengths of previous and current endpoints", tci)
  3573  			}
  3574  
  3575  			for i := range tc.previousEndpoints {
  3576  				prev, curr := tc.previousEndpoints[i], tc.currentEndpoints[i]
  3577  				switch {
  3578  				case prev == nil:
  3579  					fp.OnEndpointSliceAdd(curr)
  3580  				case curr == nil:
  3581  					fp.OnEndpointSliceDelete(prev)
  3582  				default:
  3583  					fp.OnEndpointSliceUpdate(prev, curr)
  3584  				}
  3585  			}
  3586  			result := fp.endpointsMap.Update(fp.endpointsChanges)
  3587  			newMap := fp.endpointsMap
  3588  			checkEndpointExpectations(t, tci, newMap, tc.expectedResult)
  3589  			if len(result.DeletedUDPEndpoints) != len(tc.expectedDeletedUDPEndpoints) {
  3590  				t.Errorf("[%d] expected %d staleEndpoints, got %d: %v", tci, len(tc.expectedDeletedUDPEndpoints), len(result.DeletedUDPEndpoints), result.DeletedUDPEndpoints)
  3591  			}
  3592  			for _, x := range tc.expectedDeletedUDPEndpoints {
  3593  				found := false
  3594  				for _, stale := range result.DeletedUDPEndpoints {
  3595  					if stale == x {
  3596  						found = true
  3597  						break
  3598  					}
  3599  				}
  3600  				if !found {
  3601  					t.Errorf("[%d] expected staleEndpoints[%v], but didn't find it: %v", tci, x, result.DeletedUDPEndpoints)
  3602  				}
  3603  			}
  3604  			if len(result.NewlyActiveUDPServices) != len(tc.expectedNewlyActiveUDPServices) {
  3605  				t.Errorf("[%d] expected %d staleServiceNames, got %d: %v", tci, len(tc.expectedNewlyActiveUDPServices), len(result.NewlyActiveUDPServices), result.NewlyActiveUDPServices)
  3606  			}
  3607  			for svcName := range tc.expectedNewlyActiveUDPServices {
  3608  				found := false
  3609  				for _, stale := range result.NewlyActiveUDPServices {
  3610  					if stale == svcName {
  3611  						found = true
  3612  						break
  3613  					}
  3614  				}
  3615  				if !found {
  3616  					t.Errorf("[%d] expected staleServiceNames[%v], but didn't find it: %v", tci, svcName, result.NewlyActiveUDPServices)
  3617  				}
  3618  			}
  3619  			localReadyEndpoints := fp.endpointsMap.LocalReadyEndpoints()
  3620  			if !reflect.DeepEqual(localReadyEndpoints, tc.expectedReadyEndpoints) {
  3621  				t.Errorf("[%d] expected healthchecks %v, got %v", tci, tc.expectedReadyEndpoints, localReadyEndpoints)
  3622  			}
  3623  		})
  3624  	}
  3625  }
  3626  
  3627  type endpointExpectation struct {
  3628  	endpoint string
  3629  	isLocal  bool
  3630  }
  3631  
  3632  func checkEndpointExpectations(t *testing.T, tci int, newMap proxy.EndpointsMap, expected map[proxy.ServicePortName][]endpointExpectation) {
  3633  	if len(newMap) != len(expected) {
  3634  		t.Errorf("[%d] expected %d results, got %d: %v", tci, len(expected), len(newMap), newMap)
  3635  	}
  3636  	for x := range expected {
  3637  		if len(newMap[x]) != len(expected[x]) {
  3638  			t.Errorf("[%d] expected %d endpoints for %v, got %d", tci, len(expected[x]), x, len(newMap[x]))
  3639  		} else {
  3640  			for i := range expected[x] {
  3641  				newEp := newMap[x][i]
  3642  				if newEp.String() != expected[x][i].endpoint ||
  3643  					newEp.IsLocal() != expected[x][i].isLocal {
  3644  					t.Errorf("[%d] expected new[%v][%d] to be %v, got %v", tci, x, i, expected[x][i], newEp)
  3645  				}
  3646  			}
  3647  		}
  3648  	}
  3649  }
  3650  
  3651  func Test_syncService(t *testing.T) {
  3652  	testCases := []struct {
  3653  		oldVirtualServer  *utilipvs.VirtualServer
  3654  		svcName           string
  3655  		newVirtualServer  *utilipvs.VirtualServer
  3656  		bindAddr          bool
  3657  		alreadyBoundAddrs sets.Set[string]
  3658  	}{
  3659  		{
  3660  			// case 0, old virtual server is same as new virtual server
  3661  			oldVirtualServer: &utilipvs.VirtualServer{
  3662  				Address:   netutils.ParseIPSloppy("1.2.3.4"),
  3663  				Protocol:  string(v1.ProtocolTCP),
  3664  				Port:      80,
  3665  				Scheduler: "rr",
  3666  				Flags:     utilipvs.FlagHashed,
  3667  			},
  3668  			svcName: "foo",
  3669  			newVirtualServer: &utilipvs.VirtualServer{
  3670  				Address:   netutils.ParseIPSloppy("1.2.3.4"),
  3671  				Protocol:  string(v1.ProtocolTCP),
  3672  				Port:      80,
  3673  				Scheduler: "rr",
  3674  				Flags:     utilipvs.FlagHashed,
  3675  			},
  3676  			bindAddr:          false,
  3677  			alreadyBoundAddrs: nil,
  3678  		},
  3679  		{
  3680  			// case 1, old virtual server is different from new virtual server
  3681  			oldVirtualServer: &utilipvs.VirtualServer{
  3682  				Address:   netutils.ParseIPSloppy("1.2.3.4"),
  3683  				Protocol:  string(v1.ProtocolTCP),
  3684  				Port:      8080,
  3685  				Scheduler: "rr",
  3686  				Flags:     utilipvs.FlagHashed,
  3687  			},
  3688  			svcName: "bar",
  3689  			newVirtualServer: &utilipvs.VirtualServer{
  3690  				Address:   netutils.ParseIPSloppy("1.2.3.4"),
  3691  				Protocol:  string(v1.ProtocolTCP),
  3692  				Port:      8080,
  3693  				Scheduler: "rr",
  3694  				Flags:     utilipvs.FlagPersistent,
  3695  			},
  3696  			bindAddr:          false,
  3697  			alreadyBoundAddrs: nil,
  3698  		},
  3699  		{
  3700  			// case 2, old virtual server is different from new virtual server
  3701  			oldVirtualServer: &utilipvs.VirtualServer{
  3702  				Address:   netutils.ParseIPSloppy("1.2.3.4"),
  3703  				Protocol:  string(v1.ProtocolTCP),
  3704  				Port:      8080,
  3705  				Scheduler: "rr",
  3706  				Flags:     utilipvs.FlagHashed,
  3707  			},
  3708  			svcName: "bar",
  3709  			newVirtualServer: &utilipvs.VirtualServer{
  3710  				Address:   netutils.ParseIPSloppy("1.2.3.4"),
  3711  				Protocol:  string(v1.ProtocolTCP),
  3712  				Port:      8080,
  3713  				Scheduler: "wlc",
  3714  				Flags:     utilipvs.FlagHashed,
  3715  			},
  3716  			bindAddr:          false,
  3717  			alreadyBoundAddrs: nil,
  3718  		},
  3719  		{
  3720  			// case 3, old virtual server is nil, and create new virtual server
  3721  			oldVirtualServer: nil,
  3722  			svcName:          "baz",
  3723  			newVirtualServer: &utilipvs.VirtualServer{
  3724  				Address:   netutils.ParseIPSloppy("1.2.3.4"),
  3725  				Protocol:  string(v1.ProtocolUDP),
  3726  				Port:      53,
  3727  				Scheduler: "rr",
  3728  				Flags:     utilipvs.FlagHashed,
  3729  			},
  3730  			bindAddr:          true,
  3731  			alreadyBoundAddrs: nil,
  3732  		},
  3733  		{
  3734  			// case 4, SCTP, old virtual server is same as new virtual server
  3735  			oldVirtualServer: &utilipvs.VirtualServer{
  3736  				Address:   netutils.ParseIPSloppy("1.2.3.4"),
  3737  				Protocol:  string(v1.ProtocolSCTP),
  3738  				Port:      80,
  3739  				Scheduler: "rr",
  3740  				Flags:     utilipvs.FlagHashed,
  3741  			},
  3742  			svcName: "foo",
  3743  			newVirtualServer: &utilipvs.VirtualServer{
  3744  				Address:   netutils.ParseIPSloppy("1.2.3.4"),
  3745  				Protocol:  string(v1.ProtocolSCTP),
  3746  				Port:      80,
  3747  				Scheduler: "rr",
  3748  				Flags:     utilipvs.FlagHashed,
  3749  			},
  3750  			bindAddr:          false,
  3751  			alreadyBoundAddrs: nil,
  3752  		},
  3753  		{
  3754  			// case 5, old virtual server is different from new virtual server
  3755  			oldVirtualServer: &utilipvs.VirtualServer{
  3756  				Address:   netutils.ParseIPSloppy("1.2.3.4"),
  3757  				Protocol:  string(v1.ProtocolSCTP),
  3758  				Port:      8080,
  3759  				Scheduler: "rr",
  3760  				Flags:     utilipvs.FlagHashed,
  3761  			},
  3762  			svcName: "bar",
  3763  			newVirtualServer: &utilipvs.VirtualServer{
  3764  				Address:   netutils.ParseIPSloppy("1.2.3.4"),
  3765  				Protocol:  string(v1.ProtocolSCTP),
  3766  				Port:      8080,
  3767  				Scheduler: "rr",
  3768  				Flags:     utilipvs.FlagPersistent,
  3769  			},
  3770  			bindAddr:          false,
  3771  			alreadyBoundAddrs: nil,
  3772  		},
  3773  		{
  3774  			// case 6, old virtual server is different from new virtual server
  3775  			oldVirtualServer: &utilipvs.VirtualServer{
  3776  				Address:   netutils.ParseIPSloppy("1.2.3.4"),
  3777  				Protocol:  string(v1.ProtocolSCTP),
  3778  				Port:      8080,
  3779  				Scheduler: "rr",
  3780  				Flags:     utilipvs.FlagHashed,
  3781  			},
  3782  			svcName: "bar",
  3783  			newVirtualServer: &utilipvs.VirtualServer{
  3784  				Address:   netutils.ParseIPSloppy("1.2.3.4"),
  3785  				Protocol:  string(v1.ProtocolSCTP),
  3786  				Port:      8080,
  3787  				Scheduler: "wlc",
  3788  				Flags:     utilipvs.FlagHashed,
  3789  			},
  3790  			bindAddr:          false,
  3791  			alreadyBoundAddrs: nil,
  3792  		},
  3793  		{
  3794  			// case 7, old virtual server is nil, and create new virtual server
  3795  			oldVirtualServer: nil,
  3796  			svcName:          "baz",
  3797  			newVirtualServer: &utilipvs.VirtualServer{
  3798  				Address:   netutils.ParseIPSloppy("1.2.3.4"),
  3799  				Protocol:  string(v1.ProtocolSCTP),
  3800  				Port:      53,
  3801  				Scheduler: "rr",
  3802  				Flags:     utilipvs.FlagHashed,
  3803  			},
  3804  			bindAddr:          true,
  3805  			alreadyBoundAddrs: sets.New[string](),
  3806  		},
  3807  		{
  3808  			// case 8, virtual server address already binded, skip sync
  3809  			oldVirtualServer: &utilipvs.VirtualServer{
  3810  				Address:   netutils.ParseIPSloppy("1.2.3.4"),
  3811  				Protocol:  string(v1.ProtocolSCTP),
  3812  				Port:      53,
  3813  				Scheduler: "rr",
  3814  				Flags:     utilipvs.FlagHashed,
  3815  			},
  3816  			svcName: "baz",
  3817  			newVirtualServer: &utilipvs.VirtualServer{
  3818  				Address:   netutils.ParseIPSloppy("1.2.3.4"),
  3819  				Protocol:  string(v1.ProtocolSCTP),
  3820  				Port:      53,
  3821  				Scheduler: "rr",
  3822  				Flags:     utilipvs.FlagHashed,
  3823  			},
  3824  			bindAddr:          true,
  3825  			alreadyBoundAddrs: sets.New("1.2.3.4"),
  3826  		},
  3827  	}
  3828  
  3829  	for i := range testCases {
  3830  		ipt := iptablestest.NewFake()
  3831  		ipvs := ipvstest.NewFake()
  3832  		ipset := ipsettest.NewFake(testIPSetVersion)
  3833  		proxier := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol)
  3834  
  3835  		proxier.netlinkHandle.EnsureDummyDevice(defaultDummyDevice)
  3836  		if testCases[i].oldVirtualServer != nil {
  3837  			if err := proxier.ipvs.AddVirtualServer(testCases[i].oldVirtualServer); err != nil {
  3838  				t.Errorf("Case [%d], unexpected add IPVS virtual server error: %v", i, err)
  3839  			}
  3840  		}
  3841  		if err := proxier.syncService(testCases[i].svcName, testCases[i].newVirtualServer, testCases[i].bindAddr, testCases[i].alreadyBoundAddrs); err != nil {
  3842  			t.Errorf("Case [%d], unexpected sync IPVS virtual server error: %v", i, err)
  3843  		}
  3844  		// check
  3845  		list, err := proxier.ipvs.GetVirtualServers()
  3846  		if err != nil {
  3847  			t.Errorf("Case [%d], unexpected list IPVS virtual server error: %v", i, err)
  3848  		}
  3849  		if len(list) != 1 {
  3850  			t.Errorf("Case [%d], expect %d virtual servers, got %d", i, 1, len(list))
  3851  			continue
  3852  		}
  3853  		if !list[0].Equal(testCases[i].newVirtualServer) {
  3854  			t.Errorf("Case [%d], unexpected mismatch, expect: %#v, got: %#v", i, testCases[i].newVirtualServer, list[0])
  3855  		}
  3856  	}
  3857  }
  3858  
  3859  func buildFakeProxier() (*iptablestest.FakeIPTables, *Proxier) {
  3860  	ipt := iptablestest.NewFake()
  3861  	ipvs := ipvstest.NewFake()
  3862  	ipset := ipsettest.NewFake(testIPSetVersion)
  3863  	return ipt, NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol)
  3864  }
  3865  
  3866  func getRules(ipt *iptablestest.FakeIPTables, chain utiliptables.Chain) []*iptablestest.Rule {
  3867  	var rules []*iptablestest.Rule
  3868  
  3869  	buf := bytes.NewBuffer(nil)
  3870  	_ = ipt.SaveInto(utiliptables.TableNAT, buf)
  3871  	_ = ipt.SaveInto(utiliptables.TableFilter, buf)
  3872  	lines := strings.Split(buf.String(), "\n")
  3873  	for _, l := range lines {
  3874  		if !strings.HasPrefix(l, "-A ") {
  3875  			continue
  3876  		}
  3877  		rule, _ := iptablestest.ParseRule(l, false)
  3878  		if rule != nil && rule.Chain == chain {
  3879  			rules = append(rules, rule)
  3880  		}
  3881  	}
  3882  	return rules
  3883  }
  3884  
  3885  // checkIptables to check expected iptables chain and rules. The got rules must have same number and order as the
  3886  // expected rules.
  3887  func checkIptables(t *testing.T, ipt *iptablestest.FakeIPTables, epIpt netlinktest.ExpectedIptablesChain) {
  3888  	for epChain, epRules := range epIpt {
  3889  		rules := getRules(ipt, utiliptables.Chain(epChain))
  3890  		if len(rules) != len(epRules) {
  3891  			t.Errorf("Expected %d iptables rule in chain %s, got %d", len(epRules), epChain, len(rules))
  3892  			continue
  3893  		}
  3894  		for i, epRule := range epRules {
  3895  			rule := rules[i]
  3896  			if rule.Jump == nil || rule.Jump.Value != epRule.JumpChain {
  3897  				t.Errorf("Expected MatchSet=%s JumpChain=%s, got %s", epRule.MatchSet, epRule.JumpChain, rule.Raw)
  3898  			}
  3899  			if (epRule.MatchSet == "" && rule.MatchSet != nil) || (epRule.MatchSet != "" && (rule.MatchSet == nil || rule.MatchSet.Value != epRule.MatchSet)) {
  3900  				t.Errorf("Expected MatchSet=%s JumpChain=%s, got %s", epRule.MatchSet, epRule.JumpChain, rule.Raw)
  3901  			}
  3902  		}
  3903  	}
  3904  }
  3905  
  3906  // checkIPSet to check expected ipset and entries
  3907  func checkIPSet(t *testing.T, fp *Proxier, ipSet netlinktest.ExpectedIPSet) {
  3908  	for set, entries := range ipSet {
  3909  		ents, err := fp.ipset.ListEntries(set)
  3910  		if err != nil || len(ents) != len(entries) {
  3911  			t.Errorf("Check ipset entries failed for ipset: %q, expect %d, got %d", set, len(entries), len(ents))
  3912  			continue
  3913  		}
  3914  		expectedEntries := []string{}
  3915  		for _, entry := range entries {
  3916  			expectedEntries = append(expectedEntries, entry.String())
  3917  		}
  3918  		sort.Strings(ents)
  3919  		sort.Strings(expectedEntries)
  3920  		if !reflect.DeepEqual(ents, expectedEntries) {
  3921  			t.Errorf("Check ipset entries failed for ipset: %q", set)
  3922  		}
  3923  	}
  3924  }
  3925  
  3926  // checkIPVS to check expected ipvs service and destination
  3927  func checkIPVS(t *testing.T, fp *Proxier, vs *netlinktest.ExpectedVirtualServer) {
  3928  	t.Helper()
  3929  	services, err := fp.ipvs.GetVirtualServers()
  3930  	if err != nil {
  3931  		t.Errorf("Failed to get ipvs services, err: %v", err)
  3932  	}
  3933  	if len(services) != vs.VSNum {
  3934  		t.Errorf("Expect %d ipvs services, got %d", vs.VSNum, len(services))
  3935  	}
  3936  	for _, svc := range services {
  3937  		if svc.Address.String() == vs.IP && svc.Port == vs.Port && svc.Protocol == vs.Protocol {
  3938  			destinations, _ := fp.ipvs.GetRealServers(svc)
  3939  			if len(destinations) != len(vs.RS) {
  3940  				t.Errorf("Expected %d destinations, got %d destinations", len(vs.RS), len(destinations))
  3941  			}
  3942  			if len(vs.RS) == 1 {
  3943  				if destinations[0].Address.String() != vs.RS[0].IP || destinations[0].Port != vs.RS[0].Port {
  3944  					t.Errorf("Unexpected mismatch destinations")
  3945  				}
  3946  			}
  3947  		}
  3948  	}
  3949  }
  3950  
  3951  func TestCleanLegacyService(t *testing.T) {
  3952  	ipt := iptablestest.NewFake()
  3953  	ipvs := ipvstest.NewFake()
  3954  	ipset := ipsettest.NewFake(testIPSetVersion)
  3955  	excludeCIDRs, _ := netutils.ParseCIDRs([]string{"3.3.3.0/24", "4.4.4.0/24"})
  3956  	fp := NewFakeProxier(ipt, ipvs, ipset, nil, excludeCIDRs, v1.IPv4Protocol)
  3957  
  3958  	// All ipvs services that were processed in the latest sync loop.
  3959  	activeServices := sets.New("ipvs0", "ipvs1")
  3960  	// All ipvs services in the system.
  3961  	currentServices := map[string]*utilipvs.VirtualServer{
  3962  		// Created by kube-proxy.
  3963  		"ipvs0": {
  3964  			Address:   netutils.ParseIPSloppy("1.1.1.1"),
  3965  			Protocol:  string(v1.ProtocolUDP),
  3966  			Port:      53,
  3967  			Scheduler: "rr",
  3968  			Flags:     utilipvs.FlagHashed,
  3969  		},
  3970  		// Created by kube-proxy.
  3971  		"ipvs1": {
  3972  			Address:   netutils.ParseIPSloppy("2.2.2.2"),
  3973  			Protocol:  string(v1.ProtocolUDP),
  3974  			Port:      54,
  3975  			Scheduler: "rr",
  3976  			Flags:     utilipvs.FlagHashed,
  3977  		},
  3978  		// Created by an external party.
  3979  		"ipvs2": {
  3980  			Address:   netutils.ParseIPSloppy("3.3.3.3"),
  3981  			Protocol:  string(v1.ProtocolUDP),
  3982  			Port:      55,
  3983  			Scheduler: "rr",
  3984  			Flags:     utilipvs.FlagHashed,
  3985  		},
  3986  		// Created by an external party.
  3987  		"ipvs3": {
  3988  			Address:   netutils.ParseIPSloppy("4.4.4.4"),
  3989  			Protocol:  string(v1.ProtocolUDP),
  3990  			Port:      56,
  3991  			Scheduler: "rr",
  3992  			Flags:     utilipvs.FlagHashed,
  3993  		},
  3994  		// Created by an external party.
  3995  		"ipvs4": {
  3996  			Address:   netutils.ParseIPSloppy("5.5.5.5"),
  3997  			Protocol:  string(v1.ProtocolUDP),
  3998  			Port:      57,
  3999  			Scheduler: "rr",
  4000  			Flags:     utilipvs.FlagHashed,
  4001  		},
  4002  		// Created by kube-proxy, but now stale.
  4003  		"ipvs5": {
  4004  			Address:   netutils.ParseIPSloppy("6.6.6.6"),
  4005  			Protocol:  string(v1.ProtocolUDP),
  4006  			Port:      58,
  4007  			Scheduler: "rr",
  4008  			Flags:     utilipvs.FlagHashed,
  4009  		},
  4010  	}
  4011  	for v := range currentServices {
  4012  		fp.ipvs.AddVirtualServer(currentServices[v])
  4013  	}
  4014  
  4015  	fp.cleanLegacyService(activeServices, currentServices)
  4016  	// ipvs4 and ipvs5 should have been cleaned.
  4017  	remainingVirtualServers, _ := fp.ipvs.GetVirtualServers()
  4018  	if len(remainingVirtualServers) != 4 {
  4019  		t.Errorf("Expected number of remaining IPVS services after cleanup to be %v. Got %v", 4, len(remainingVirtualServers))
  4020  	}
  4021  	for _, vs := range remainingVirtualServers {
  4022  		// Checking that ipvs4 and ipvs5 were removed.
  4023  		if vs.Port == 57 {
  4024  			t.Errorf("Expected ipvs4 to be removed after cleanup. It still remains")
  4025  		}
  4026  		if vs.Port == 58 {
  4027  			t.Errorf("Expected ipvs5 to be removed after cleanup. It still remains")
  4028  		}
  4029  	}
  4030  }
  4031  
  4032  func TestCleanLegacyServiceWithRealServers(t *testing.T) {
  4033  	ipt := iptablestest.NewFake()
  4034  	ipvs := ipvstest.NewFake()
  4035  	ipset := ipsettest.NewFake(testIPSetVersion)
  4036  	fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol)
  4037  
  4038  	// all deleted expect ipvs2
  4039  	activeServices := sets.New("ipvs2")
  4040  	// All ipvs services in the system.
  4041  	currentServices := map[string]*utilipvs.VirtualServer{
  4042  		"ipvs0": { // deleted with real servers
  4043  			Address:   netutils.ParseIPSloppy("1.1.1.1"),
  4044  			Protocol:  string(v1.ProtocolUDP),
  4045  			Port:      53,
  4046  			Scheduler: "rr",
  4047  			Flags:     utilipvs.FlagHashed,
  4048  		},
  4049  		"ipvs1": { // deleted no real server
  4050  			Address:   netutils.ParseIPSloppy("2.2.2.2"),
  4051  			Protocol:  string(v1.ProtocolUDP),
  4052  			Port:      54,
  4053  			Scheduler: "rr",
  4054  			Flags:     utilipvs.FlagHashed,
  4055  		},
  4056  		"ipvs2": { // not deleted
  4057  			Address:   netutils.ParseIPSloppy("3.3.3.3"),
  4058  			Protocol:  string(v1.ProtocolUDP),
  4059  			Port:      54,
  4060  			Scheduler: "rr",
  4061  			Flags:     utilipvs.FlagHashed,
  4062  		},
  4063  	}
  4064  
  4065  	// "ipvs0" has a real server, but it should still be deleted since the Service is deleted
  4066  	realServers := map[*utilipvs.VirtualServer]*utilipvs.RealServer{
  4067  		{
  4068  			Address:   netutils.ParseIPSloppy("1.1.1.1"),
  4069  			Protocol:  string(v1.ProtocolUDP),
  4070  			Port:      53,
  4071  			Scheduler: "rr",
  4072  			Flags:     utilipvs.FlagHashed,
  4073  		}: {
  4074  			Address: netutils.ParseIPSloppy("10.180.0.1"),
  4075  			Port:    uint16(53),
  4076  			Weight:  1,
  4077  		},
  4078  	}
  4079  
  4080  	for v := range currentServices {
  4081  		fp.ipvs.AddVirtualServer(currentServices[v])
  4082  	}
  4083  
  4084  	for v, r := range realServers {
  4085  		fp.ipvs.AddRealServer(v, r)
  4086  	}
  4087  
  4088  	fp.cleanLegacyService(activeServices, currentServices)
  4089  	remainingVirtualServers, _ := fp.ipvs.GetVirtualServers()
  4090  	if len(remainingVirtualServers) != 1 {
  4091  		t.Errorf("Expected number of remaining IPVS services after cleanup to be %v. Got %v", 1, len(remainingVirtualServers))
  4092  	}
  4093  
  4094  	if remainingVirtualServers[0] != currentServices["ipvs2"] {
  4095  		t.Logf("actual virtual server: %v", remainingVirtualServers[0])
  4096  		t.Logf("expected virtual server: %v", currentServices["ipvs0"])
  4097  		t.Errorf("unexpected IPVS service")
  4098  	}
  4099  }
  4100  
  4101  func TestCleanLegacyRealServersExcludeCIDRs(t *testing.T) {
  4102  	ipt := iptablestest.NewFake()
  4103  	ipvs := ipvstest.NewFake()
  4104  	ipset := ipsettest.NewFake(testIPSetVersion)
  4105  	gtm := NewGracefulTerminationManager(ipvs)
  4106  	excludeCIDRs, _ := netutils.ParseCIDRs([]string{"4.4.4.4/32"})
  4107  	fp := NewFakeProxier(ipt, ipvs, ipset, nil, excludeCIDRs, v1.IPv4Protocol)
  4108  	fp.gracefuldeleteManager = gtm
  4109  
  4110  	vs := &utilipvs.VirtualServer{
  4111  		Address:   netutils.ParseIPSloppy("4.4.4.4"),
  4112  		Protocol:  string(v1.ProtocolUDP),
  4113  		Port:      56,
  4114  		Scheduler: "rr",
  4115  		Flags:     utilipvs.FlagHashed,
  4116  	}
  4117  
  4118  	fp.ipvs.AddVirtualServer(vs)
  4119  
  4120  	rss := []*utilipvs.RealServer{
  4121  		{
  4122  			Address:      netutils.ParseIPSloppy("10.10.10.10"),
  4123  			Port:         56,
  4124  			ActiveConn:   0,
  4125  			InactiveConn: 0,
  4126  		},
  4127  		{
  4128  			Address:      netutils.ParseIPSloppy("11.11.11.11"),
  4129  			Port:         56,
  4130  			ActiveConn:   0,
  4131  			InactiveConn: 0,
  4132  		},
  4133  	}
  4134  	for _, rs := range rss {
  4135  		fp.ipvs.AddRealServer(vs, rs)
  4136  	}
  4137  
  4138  	fp.netlinkHandle.EnsureDummyDevice(defaultDummyDevice)
  4139  
  4140  	fp.netlinkHandle.EnsureAddressBind("4.4.4.4", defaultDummyDevice)
  4141  
  4142  	fp.cleanLegacyService(nil, map[string]*utilipvs.VirtualServer{"ipvs0": vs})
  4143  
  4144  	fp.gracefuldeleteManager.tryDeleteRs()
  4145  
  4146  	remainingRealServers, _ := fp.ipvs.GetRealServers(vs)
  4147  
  4148  	if len(remainingRealServers) != 2 {
  4149  		t.Errorf("Expected number of remaining IPVS real servers after cleanup should be %v. Got %v", 2, len(remainingRealServers))
  4150  	}
  4151  }
  4152  
  4153  func TestCleanLegacyService6(t *testing.T) {
  4154  	ipt := iptablestest.NewFake()
  4155  	ipvs := ipvstest.NewFake()
  4156  	ipset := ipsettest.NewFake(testIPSetVersion)
  4157  	excludeCIDRs, _ := netutils.ParseCIDRs([]string{"3000::/64", "4000::/64"})
  4158  	fp := NewFakeProxier(ipt, ipvs, ipset, nil, excludeCIDRs, v1.IPv6Protocol)
  4159  	fp.nodeIP = netutils.ParseIPSloppy("::1")
  4160  
  4161  	// All ipvs services that were processed in the latest sync loop.
  4162  	activeServices := sets.New("ipvs0", "ipvs1")
  4163  	// All ipvs services in the system.
  4164  	currentServices := map[string]*utilipvs.VirtualServer{
  4165  		// Created by kube-proxy.
  4166  		"ipvs0": {
  4167  			Address:   netutils.ParseIPSloppy("1000::1"),
  4168  			Protocol:  string(v1.ProtocolUDP),
  4169  			Port:      53,
  4170  			Scheduler: "rr",
  4171  			Flags:     utilipvs.FlagHashed,
  4172  		},
  4173  		// Created by kube-proxy.
  4174  		"ipvs1": {
  4175  			Address:   netutils.ParseIPSloppy("1000::2"),
  4176  			Protocol:  string(v1.ProtocolUDP),
  4177  			Port:      54,
  4178  			Scheduler: "rr",
  4179  			Flags:     utilipvs.FlagHashed,
  4180  		},
  4181  		// Created by an external party.
  4182  		"ipvs2": {
  4183  			Address:   netutils.ParseIPSloppy("3000::1"),
  4184  			Protocol:  string(v1.ProtocolUDP),
  4185  			Port:      55,
  4186  			Scheduler: "rr",
  4187  			Flags:     utilipvs.FlagHashed,
  4188  		},
  4189  		// Created by an external party.
  4190  		"ipvs3": {
  4191  			Address:   netutils.ParseIPSloppy("4000::1"),
  4192  			Protocol:  string(v1.ProtocolUDP),
  4193  			Port:      56,
  4194  			Scheduler: "rr",
  4195  			Flags:     utilipvs.FlagHashed,
  4196  		},
  4197  		// Created by an external party.
  4198  		"ipvs4": {
  4199  			Address:   netutils.ParseIPSloppy("5000::1"),
  4200  			Protocol:  string(v1.ProtocolUDP),
  4201  			Port:      57,
  4202  			Scheduler: "rr",
  4203  			Flags:     utilipvs.FlagHashed,
  4204  		},
  4205  		// Created by kube-proxy, but now stale.
  4206  		"ipvs5": {
  4207  			Address:   netutils.ParseIPSloppy("1000::6"),
  4208  			Protocol:  string(v1.ProtocolUDP),
  4209  			Port:      58,
  4210  			Scheduler: "rr",
  4211  			Flags:     utilipvs.FlagHashed,
  4212  		},
  4213  	}
  4214  	for v := range currentServices {
  4215  		fp.ipvs.AddVirtualServer(currentServices[v])
  4216  	}
  4217  
  4218  	fp.cleanLegacyService(activeServices, currentServices)
  4219  	// ipvs4 and ipvs5 should have been cleaned.
  4220  	remainingVirtualServers, _ := fp.ipvs.GetVirtualServers()
  4221  	if len(remainingVirtualServers) != 4 {
  4222  		t.Errorf("Expected number of remaining IPVS services after cleanup to be %v. Got %v", 4, len(remainingVirtualServers))
  4223  	}
  4224  	for _, vs := range remainingVirtualServers {
  4225  		// Checking that ipvs4 and ipvs5 were removed.
  4226  		if vs.Port == 57 {
  4227  			t.Errorf("Expected ipvs4 to be removed after cleanup. It still remains")
  4228  		}
  4229  		if vs.Port == 58 {
  4230  			t.Errorf("Expected ipvs5 to be removed after cleanup. It still remains")
  4231  		}
  4232  	}
  4233  }
  4234  
  4235  func TestMultiPortServiceBindAddr(t *testing.T) {
  4236  	ipt := iptablestest.NewFake()
  4237  	ipvs := ipvstest.NewFake()
  4238  	ipset := ipsettest.NewFake(testIPSetVersion)
  4239  	fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol)
  4240  
  4241  	service1 := makeTestService("ns1", "svc1", func(svc *v1.Service) {
  4242  		svc.Spec.Type = v1.ServiceTypeClusterIP
  4243  		svc.Spec.ClusterIP = "172.16.55.4"
  4244  		svc.Spec.Ports = addTestPort(svc.Spec.Ports, "port1", "TCP", 1234, 0, 0)
  4245  		svc.Spec.Ports = addTestPort(svc.Spec.Ports, "port2", "TCP", 1235, 0, 0)
  4246  	})
  4247  	service2 := makeTestService("ns1", "svc1", func(svc *v1.Service) {
  4248  		svc.Spec.Type = v1.ServiceTypeClusterIP
  4249  		svc.Spec.ClusterIP = "172.16.55.4"
  4250  		svc.Spec.Ports = addTestPort(svc.Spec.Ports, "port1", "TCP", 1234, 0, 0)
  4251  	})
  4252  	service3 := makeTestService("ns1", "svc1", func(svc *v1.Service) {
  4253  		svc.Spec.Type = v1.ServiceTypeClusterIP
  4254  		svc.Spec.ClusterIP = "172.16.55.4"
  4255  		svc.Spec.Ports = addTestPort(svc.Spec.Ports, "port1", "TCP", 1234, 0, 0)
  4256  		svc.Spec.Ports = addTestPort(svc.Spec.Ports, "port2", "TCP", 1235, 0, 0)
  4257  		svc.Spec.Ports = addTestPort(svc.Spec.Ports, "port3", "UDP", 1236, 0, 0)
  4258  	})
  4259  
  4260  	fp.servicesSynced = true
  4261  
  4262  	// first, add multi-port service1
  4263  	fp.OnServiceAdd(service1)
  4264  	fp.syncProxyRules()
  4265  	remainingAddrs, _ := fp.netlinkHandle.ListBindAddress(defaultDummyDevice)
  4266  	// should only remain address "172.16.55.4"
  4267  	if len(remainingAddrs) != 1 {
  4268  		t.Errorf("Expected number of remaining bound addrs after cleanup to be %v. Got %v", 1, len(remainingAddrs))
  4269  	}
  4270  	if remainingAddrs[0] != "172.16.55.4" {
  4271  		t.Errorf("Expected remaining address should be %s, got %s", "172.16.55.4", remainingAddrs[0])
  4272  	}
  4273  
  4274  	// update multi-port service1 to single-port service2
  4275  	fp.OnServiceUpdate(service1, service2)
  4276  	fp.syncProxyRules()
  4277  	remainingAddrs, _ = fp.netlinkHandle.ListBindAddress(defaultDummyDevice)
  4278  	// should still only remain address "172.16.55.4"
  4279  	if len(remainingAddrs) != 1 {
  4280  		t.Errorf("Expected number of remaining bound addrs after cleanup to be %v. Got %v", 1, len(remainingAddrs))
  4281  	} else if remainingAddrs[0] != "172.16.55.4" {
  4282  		t.Errorf("Expected remaining address should be %s, got %s", "172.16.55.4", remainingAddrs[0])
  4283  	}
  4284  
  4285  	// update single-port service2 to multi-port service3
  4286  	fp.OnServiceUpdate(service2, service3)
  4287  	fp.syncProxyRules()
  4288  	remainingAddrs, _ = fp.netlinkHandle.ListBindAddress(defaultDummyDevice)
  4289  	// should still only remain address "172.16.55.4"
  4290  	if len(remainingAddrs) != 1 {
  4291  		t.Errorf("Expected number of remaining bound addrs after cleanup to be %v. Got %v", 1, len(remainingAddrs))
  4292  	} else if remainingAddrs[0] != "172.16.55.4" {
  4293  		t.Errorf("Expected remaining address should be %s, got %s", "172.16.55.4", remainingAddrs[0])
  4294  	}
  4295  
  4296  	// delete multi-port service3
  4297  	fp.OnServiceDelete(service3)
  4298  	fp.syncProxyRules()
  4299  	remainingAddrs, _ = fp.netlinkHandle.ListBindAddress(defaultDummyDevice)
  4300  	// all addresses should be unbound
  4301  	if len(remainingAddrs) != 0 {
  4302  		t.Errorf("Expected number of remaining bound addrs after cleanup to be %v. Got %v", 0, len(remainingAddrs))
  4303  	}
  4304  }
  4305  
  4306  func Test_getFirstColumn(t *testing.T) {
  4307  	testCases := []struct {
  4308  		name        string
  4309  		fileContent string
  4310  		want        []string
  4311  		wantErr     bool
  4312  	}{
  4313  		{
  4314  			name: "valid content",
  4315  			fileContent: `libiscsi_tcp 28672 1 iscsi_tcp, Live 0xffffffffc07ae000
  4316  libiscsi 57344 3 ib_iser,iscsi_tcp,libiscsi_tcp, Live 0xffffffffc079a000
  4317  raid10 57344 0 - Live 0xffffffffc0597000`,
  4318  			want:    []string{"libiscsi_tcp", "libiscsi", "raid10"},
  4319  			wantErr: false,
  4320  		},
  4321  	}
  4322  	for _, test := range testCases {
  4323  		t.Run(test.name, func(t *testing.T) {
  4324  			got, err := getFirstColumn(strings.NewReader(test.fileContent))
  4325  			if (err != nil) != test.wantErr {
  4326  				t.Errorf("getFirstColumn() error = %v, wantErr %v", err, test.wantErr)
  4327  				return
  4328  			}
  4329  			if !reflect.DeepEqual(got, test.want) {
  4330  				t.Errorf("getFirstColumn() = %v, want %v", got, test.want)
  4331  			}
  4332  		})
  4333  	}
  4334  }
  4335  
  4336  // The majority of EndpointSlice specific tests are not ipvs specific and focus on
  4337  // the shared EndpointsChangeTracker and EndpointSliceCache. This test ensures that the
  4338  // ipvs proxier supports translating EndpointSlices to ipvs output.
  4339  func TestEndpointSliceE2E(t *testing.T) {
  4340  	ipt := iptablestest.NewFake()
  4341  	ipvs := ipvstest.NewFake()
  4342  	ipset := ipsettest.NewFake(testIPSetVersion)
  4343  	fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol)
  4344  	fp.servicesSynced = true
  4345  	fp.endpointSlicesSynced = true
  4346  
  4347  	// Add initial service
  4348  	serviceName := "svc1"
  4349  	namespaceName := "ns1"
  4350  	fp.OnServiceAdd(&v1.Service{
  4351  		ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: namespaceName},
  4352  		Spec: v1.ServiceSpec{
  4353  			ClusterIP: "172.20.1.1",
  4354  			Selector:  map[string]string{"foo": "bar"},
  4355  			Ports:     []v1.ServicePort{{Name: "", TargetPort: intstr.FromInt32(80), Protocol: v1.ProtocolTCP}},
  4356  		},
  4357  	})
  4358  
  4359  	// Add initial endpoint slice
  4360  	endpointSlice := &discovery.EndpointSlice{
  4361  		ObjectMeta: metav1.ObjectMeta{
  4362  			Name:      fmt.Sprintf("%s-1", serviceName),
  4363  			Namespace: namespaceName,
  4364  			Labels:    map[string]string{discovery.LabelServiceName: serviceName},
  4365  		},
  4366  		Ports: []discovery.EndpointPort{{
  4367  			Name:     ptr.To(""),
  4368  			Port:     ptr.To[int32](80),
  4369  			Protocol: ptr.To(v1.ProtocolTCP),
  4370  		}},
  4371  		AddressType: discovery.AddressTypeIPv4,
  4372  		Endpoints: []discovery.Endpoint{{
  4373  			Addresses:  []string{"10.0.1.1"},
  4374  			Conditions: discovery.EndpointConditions{Ready: ptr.To(true)},
  4375  			NodeName:   ptr.To(testHostname),
  4376  		}, {
  4377  			Addresses:  []string{"10.0.1.2"},
  4378  			Conditions: discovery.EndpointConditions{Ready: ptr.To(true)},
  4379  			NodeName:   ptr.To("node2"),
  4380  		}, {
  4381  			Addresses:  []string{"10.0.1.3"},
  4382  			Conditions: discovery.EndpointConditions{Ready: ptr.To(true)},
  4383  			NodeName:   ptr.To("node3"),
  4384  		}, { // not ready endpoints should be ignored
  4385  			Addresses:  []string{"10.0.1.4"},
  4386  			Conditions: discovery.EndpointConditions{Ready: ptr.To(false)},
  4387  			NodeName:   ptr.To("node3"),
  4388  		}},
  4389  	}
  4390  
  4391  	fp.OnEndpointSliceAdd(endpointSlice)
  4392  	fp.syncProxyRules()
  4393  
  4394  	// Ensure that Proxier updates ipvs appropriately after EndpointSlice update
  4395  	assert.NotNil(t, fp.ipsetList["KUBE-LOOP-BACK"])
  4396  	activeEntries1 := fp.ipsetList["KUBE-LOOP-BACK"].activeEntries
  4397  	assert.Equal(t, 1, activeEntries1.Len(), "Expected 1 active entry in KUBE-LOOP-BACK")
  4398  	assert.Equal(t, true, activeEntries1.Has("10.0.1.1,tcp:80,10.0.1.1"), "Expected activeEntries to reference first (local) pod")
  4399  	virtualServers1, vsErr1 := ipvs.GetVirtualServers()
  4400  	assert.Nil(t, vsErr1, "Expected no error getting virtual servers")
  4401  	assert.Len(t, virtualServers1, 1, "Expected 1 virtual server")
  4402  	realServers1, rsErr1 := ipvs.GetRealServers(virtualServers1[0])
  4403  	assert.Nil(t, rsErr1, "Expected no error getting real servers")
  4404  	assert.Len(t, realServers1, 3, "Expected 3 real servers")
  4405  	assert.Equal(t, realServers1[0].String(), "10.0.1.1:80")
  4406  	assert.Equal(t, realServers1[1].String(), "10.0.1.2:80")
  4407  	assert.Equal(t, realServers1[2].String(), "10.0.1.3:80")
  4408  
  4409  	fp.OnEndpointSliceDelete(endpointSlice)
  4410  	fp.syncProxyRules()
  4411  
  4412  	// Ensure that Proxier updates ipvs appropriately after EndpointSlice delete
  4413  	assert.NotNil(t, fp.ipsetList["KUBE-LOOP-BACK"])
  4414  	activeEntries2 := fp.ipsetList["KUBE-LOOP-BACK"].activeEntries
  4415  	assert.Equal(t, 0, activeEntries2.Len(), "Expected 0 active entries in KUBE-LOOP-BACK")
  4416  	virtualServers2, vsErr2 := ipvs.GetVirtualServers()
  4417  	assert.Nil(t, vsErr2, "Expected no error getting virtual servers")
  4418  	assert.Len(t, virtualServers2, 1, "Expected 1 virtual server")
  4419  	realServers2, rsErr2 := ipvs.GetRealServers(virtualServers2[0])
  4420  	assert.Nil(t, rsErr2, "Expected no error getting real servers")
  4421  	assert.Len(t, realServers2, 0, "Expected 0 real servers")
  4422  }
  4423  
  4424  func TestHealthCheckNodePortE2E(t *testing.T) {
  4425  	ipt := iptablestest.NewFake()
  4426  	ipvs := ipvstest.NewFake()
  4427  	ipset := ipsettest.NewFake(testIPSetVersion)
  4428  	fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol)
  4429  	fp.servicesSynced = true
  4430  	fp.endpointSlicesSynced = true
  4431  
  4432  	// Add initial service
  4433  	serviceName := "svc1"
  4434  	namespaceName := "ns1"
  4435  
  4436  	svc := v1.Service{
  4437  		ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: namespaceName},
  4438  		Spec: v1.ServiceSpec{
  4439  			ClusterIP:             "172.20.1.1",
  4440  			Selector:              map[string]string{"foo": "bar"},
  4441  			Ports:                 []v1.ServicePort{{Name: "", TargetPort: intstr.FromInt32(80), Protocol: v1.ProtocolTCP}},
  4442  			Type:                  "LoadBalancer",
  4443  			HealthCheckNodePort:   30000,
  4444  			ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyLocal,
  4445  		},
  4446  	}
  4447  	fp.OnServiceAdd(&svc)
  4448  	fp.syncProxyRules()
  4449  
  4450  	// Ensure that Proxier updates ipvs appropriately after service's being created
  4451  	assert.NotNil(t, fp.ipsetList["KUBE-HEALTH-CHECK-NODE-PORT"])
  4452  	activeEntries1 := fp.ipsetList["KUBE-HEALTH-CHECK-NODE-PORT"].activeEntries
  4453  	assert.Equal(t, 1, activeEntries1.Len(), "Expected 1 active entry in KUBE-HEALTH-CHECK-NODE-PORT")
  4454  	assert.Equal(t, true, activeEntries1.Has("30000"), "Expected activeEntries to reference hc node port in spec")
  4455  
  4456  	// Update health check node port in the spec
  4457  	newSvc := svc
  4458  	newSvc.Spec.HealthCheckNodePort = 30001
  4459  	fp.OnServiceUpdate(&svc, &newSvc)
  4460  	fp.syncProxyRules()
  4461  
  4462  	// Ensure that Proxier updates ipvs appropriately after service's being updated
  4463  	assert.NotNil(t, fp.ipsetList["KUBE-HEALTH-CHECK-NODE-PORT"])
  4464  	activeEntries2 := fp.ipsetList["KUBE-HEALTH-CHECK-NODE-PORT"].activeEntries
  4465  	assert.Equal(t, 1, activeEntries2.Len(), "Expected 1 active entry in KUBE-HEALTH-CHECK-NODE-PORT")
  4466  	assert.Equal(t, true, activeEntries2.Has("30001"), "Expected activeEntries to reference updated hc node port in spec")
  4467  
  4468  	fp.OnServiceDelete(&svc)
  4469  	fp.syncProxyRules()
  4470  
  4471  	// Ensure that Proxier updates ipvs appropriately after EndpointSlice delete
  4472  	assert.NotNil(t, fp.ipsetList["KUBE-HEALTH-CHECK-NODE-PORT"])
  4473  	activeEntries3 := fp.ipsetList["KUBE-HEALTH-CHECK-NODE-PORT"].activeEntries
  4474  	assert.Equal(t, 0, activeEntries3.Len(), "Expected 0 active entries in KUBE-HEALTH-CHECK-NODE-PORT")
  4475  }
  4476  
  4477  // Test_HealthCheckNodePortWhenTerminating tests that health check node ports are not enabled when all local endpoints are terminating
  4478  func Test_HealthCheckNodePortWhenTerminating(t *testing.T) {
  4479  	ipt := iptablestest.NewFake()
  4480  	ipvs := ipvstest.NewFake()
  4481  	ipset := ipsettest.NewFake(testIPSetVersion)
  4482  	fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol)
  4483  	fp.servicesSynced = true
  4484  	// fp.endpointsSynced = true
  4485  	fp.endpointSlicesSynced = true
  4486  
  4487  	serviceName := "svc1"
  4488  	namespaceName := "ns1"
  4489  
  4490  	fp.OnServiceAdd(&v1.Service{
  4491  		ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: namespaceName},
  4492  		Spec: v1.ServiceSpec{
  4493  			ClusterIP: "172.20.1.1",
  4494  			Selector:  map[string]string{"foo": "bar"},
  4495  			Ports:     []v1.ServicePort{{Name: "", TargetPort: intstr.FromInt32(80), Protocol: v1.ProtocolTCP}},
  4496  		},
  4497  	})
  4498  
  4499  	endpointSlice := &discovery.EndpointSlice{
  4500  		ObjectMeta: metav1.ObjectMeta{
  4501  			Name:      fmt.Sprintf("%s-1", serviceName),
  4502  			Namespace: namespaceName,
  4503  			Labels:    map[string]string{discovery.LabelServiceName: serviceName},
  4504  		},
  4505  		Ports: []discovery.EndpointPort{{
  4506  			Name:     ptr.To(""),
  4507  			Port:     ptr.To[int32](80),
  4508  			Protocol: ptr.To(v1.ProtocolTCP),
  4509  		}},
  4510  		AddressType: discovery.AddressTypeIPv4,
  4511  		Endpoints: []discovery.Endpoint{{
  4512  			Addresses:  []string{"10.0.1.1"},
  4513  			Conditions: discovery.EndpointConditions{Ready: ptr.To(true)},
  4514  			NodeName:   ptr.To(testHostname),
  4515  		}, {
  4516  			Addresses:  []string{"10.0.1.2"},
  4517  			Conditions: discovery.EndpointConditions{Ready: ptr.To(true)},
  4518  			NodeName:   ptr.To(testHostname),
  4519  		}, {
  4520  			Addresses:  []string{"10.0.1.3"},
  4521  			Conditions: discovery.EndpointConditions{Ready: ptr.To(true)},
  4522  		}, { // not ready endpoints should be ignored
  4523  			Addresses:  []string{"10.0.1.4"},
  4524  			Conditions: discovery.EndpointConditions{Ready: ptr.To(false)},
  4525  			NodeName:   ptr.To(testHostname),
  4526  		}},
  4527  	}
  4528  
  4529  	fp.OnEndpointSliceAdd(endpointSlice)
  4530  	_ = fp.endpointsMap.Update(fp.endpointsChanges)
  4531  	localReadyEndpoints := fp.endpointsMap.LocalReadyEndpoints()
  4532  	if len(localReadyEndpoints) != 1 {
  4533  		t.Errorf("unexpected number of health check node ports, expected 1 but got: %d", len(localReadyEndpoints))
  4534  	}
  4535  
  4536  	// set all endpoints to terminating
  4537  	endpointSliceTerminating := &discovery.EndpointSlice{
  4538  		ObjectMeta: metav1.ObjectMeta{
  4539  			Name:      fmt.Sprintf("%s-1", serviceName),
  4540  			Namespace: namespaceName,
  4541  			Labels:    map[string]string{discovery.LabelServiceName: serviceName},
  4542  		},
  4543  		Ports: []discovery.EndpointPort{{
  4544  			Name:     ptr.To(""),
  4545  			Port:     ptr.To[int32](80),
  4546  			Protocol: ptr.To(v1.ProtocolTCP),
  4547  		}},
  4548  		AddressType: discovery.AddressTypeIPv4,
  4549  		Endpoints: []discovery.Endpoint{{
  4550  			Addresses: []string{"10.0.1.1"},
  4551  			Conditions: discovery.EndpointConditions{
  4552  				Ready:       ptr.To(false),
  4553  				Serving:     ptr.To(true),
  4554  				Terminating: ptr.To(false),
  4555  			},
  4556  			NodeName: ptr.To(testHostname),
  4557  		}, {
  4558  			Addresses: []string{"10.0.1.2"},
  4559  			Conditions: discovery.EndpointConditions{
  4560  				Ready:       ptr.To(false),
  4561  				Serving:     ptr.To(true),
  4562  				Terminating: ptr.To(true),
  4563  			},
  4564  			NodeName: ptr.To(testHostname),
  4565  		}, {
  4566  			Addresses: []string{"10.0.1.3"},
  4567  			Conditions: discovery.EndpointConditions{
  4568  				Ready:       ptr.To(false),
  4569  				Serving:     ptr.To(true),
  4570  				Terminating: ptr.To(true),
  4571  			},
  4572  			NodeName: ptr.To(testHostname),
  4573  		}, { // not ready endpoints should be ignored
  4574  			Addresses: []string{"10.0.1.4"},
  4575  			Conditions: discovery.EndpointConditions{
  4576  				Ready:       ptr.To(false),
  4577  				Serving:     ptr.To(false),
  4578  				Terminating: ptr.To(true),
  4579  			},
  4580  			NodeName: ptr.To(testHostname),
  4581  		}},
  4582  	}
  4583  
  4584  	fp.OnEndpointSliceUpdate(endpointSlice, endpointSliceTerminating)
  4585  	_ = fp.endpointsMap.Update(fp.endpointsChanges)
  4586  	localReadyEndpoints = fp.endpointsMap.LocalReadyEndpoints()
  4587  	if len(localReadyEndpoints) != 0 {
  4588  		t.Errorf("unexpected number of health check node ports, expected 0 but got: %d", len(localReadyEndpoints))
  4589  	}
  4590  }
  4591  
  4592  func TestFilterCIDRs(t *testing.T) {
  4593  	var cidrList []string
  4594  	var cidrs []string
  4595  	var expected []string
  4596  	cidrs = filterCIDRs(true, []string{})
  4597  	if len(cidrs) > 0 {
  4598  		t.Errorf("An empty list produces a non-empty return %v", cidrs)
  4599  	}
  4600  
  4601  	cidrList = []string{"1000::/64", "10.0.0.0/16", "11.0.0.0/16", "2000::/64"}
  4602  	expected = []string{"1000::/64", "2000::/64"}
  4603  	cidrs = filterCIDRs(true, cidrList)
  4604  	if !reflect.DeepEqual(cidrs, expected) {
  4605  		t.Errorf("cidrs %v is not expected %v", cidrs, expected)
  4606  	}
  4607  
  4608  	expected = []string{"10.0.0.0/16", "11.0.0.0/16"}
  4609  	cidrs = filterCIDRs(false, cidrList)
  4610  	if !reflect.DeepEqual(cidrs, expected) {
  4611  		t.Errorf("cidrs %v is not expected %v", cidrs, expected)
  4612  	}
  4613  
  4614  	cidrList = []string{"1000::/64", "2000::/64"}
  4615  	expected = []string{}
  4616  	cidrs = filterCIDRs(false, cidrList)
  4617  	if len(cidrs) > 0 {
  4618  		t.Errorf("cidrs %v is not expected %v", cidrs, expected)
  4619  	}
  4620  }
  4621  
  4622  func TestCreateAndLinkKubeChain(t *testing.T) {
  4623  	ipt := iptablestest.NewFake()
  4624  	ipvs := ipvstest.NewFake()
  4625  	ipset := ipsettest.NewFake(testIPSetVersion)
  4626  	fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol)
  4627  	fp.createAndLinkKubeChain()
  4628  	expectedNATChains := `:KUBE-SERVICES - [0:0]
  4629  :KUBE-POSTROUTING - [0:0]
  4630  :KUBE-NODE-PORT - [0:0]
  4631  :KUBE-LOAD-BALANCER - [0:0]
  4632  :KUBE-MARK-MASQ - [0:0]
  4633  `
  4634  	expectedFilterChains := `:KUBE-FORWARD - [0:0]
  4635  :KUBE-NODE-PORT - [0:0]
  4636  :KUBE-PROXY-FIREWALL - [0:0]
  4637  :KUBE-SOURCE-RANGES-FIREWALL - [0:0]
  4638  :KUBE-IPVS-FILTER - [0:0]
  4639  :KUBE-IPVS-OUT-FILTER - [0:0]
  4640  `
  4641  	assert.Equal(t, expectedNATChains, fp.natChains.String())
  4642  	assert.Equal(t, expectedFilterChains, fp.filterChains.String())
  4643  }
  4644  
  4645  // This test ensures that the iptables proxier supports translating Endpoints to
  4646  // iptables output when internalTrafficPolicy is specified
  4647  func TestTestInternalTrafficPolicyE2E(t *testing.T) {
  4648  	type endpoint struct {
  4649  		ip       string
  4650  		hostname string
  4651  	}
  4652  
  4653  	testCases := []struct {
  4654  		name                     string
  4655  		internalTrafficPolicy    *v1.ServiceInternalTrafficPolicy
  4656  		endpoints                []endpoint
  4657  		expectVirtualServer      bool
  4658  		expectLocalEntries       bool
  4659  		expectLocalRealServerNum int
  4660  		expectLocalRealServers   []string
  4661  	}{
  4662  		{
  4663  			name:                  "internalTrafficPolicy is cluster with non-zero local endpoints",
  4664  			internalTrafficPolicy: ptr.To(v1.ServiceInternalTrafficPolicyCluster),
  4665  			endpoints: []endpoint{
  4666  				{"10.0.1.1", testHostname},
  4667  				{"10.0.1.2", "host1"},
  4668  				{"10.0.1.3", "host2"},
  4669  			},
  4670  			expectVirtualServer:      true,
  4671  			expectLocalEntries:       true,
  4672  			expectLocalRealServerNum: 3,
  4673  			expectLocalRealServers: []string{
  4674  				"10.0.1.1:80",
  4675  				"10.0.1.2:80",
  4676  				"10.0.1.3:80",
  4677  			},
  4678  		},
  4679  		{
  4680  			name:                  "internalTrafficPolicy is cluster with zero local endpoints",
  4681  			internalTrafficPolicy: ptr.To(v1.ServiceInternalTrafficPolicyCluster),
  4682  			endpoints: []endpoint{
  4683  				{"10.0.1.1", "host0"},
  4684  				{"10.0.1.2", "host1"},
  4685  				{"10.0.1.3", "host2"},
  4686  			},
  4687  			expectVirtualServer:      false,
  4688  			expectLocalEntries:       false,
  4689  			expectLocalRealServerNum: 3,
  4690  			expectLocalRealServers: []string{
  4691  				"10.0.1.1:80",
  4692  				"10.0.1.2:80",
  4693  				"10.0.1.3:80",
  4694  			},
  4695  		},
  4696  		{
  4697  			name:                  "internalTrafficPolicy is local with non-zero local endpoints",
  4698  			internalTrafficPolicy: ptr.To(v1.ServiceInternalTrafficPolicyLocal),
  4699  			endpoints: []endpoint{
  4700  				{"10.0.1.1", testHostname},
  4701  				{"10.0.1.2", "host1"},
  4702  				{"10.0.1.3", "host2"},
  4703  			},
  4704  			expectVirtualServer:      true,
  4705  			expectLocalEntries:       true,
  4706  			expectLocalRealServerNum: 1,
  4707  			expectLocalRealServers: []string{
  4708  				"10.0.1.1:80",
  4709  			},
  4710  		},
  4711  		{
  4712  			name:                  "internalTrafficPolicy is local with zero local endpoints",
  4713  			internalTrafficPolicy: ptr.To(v1.ServiceInternalTrafficPolicyLocal),
  4714  			endpoints: []endpoint{
  4715  				{"10.0.1.1", "host0"},
  4716  				{"10.0.1.2", "host1"},
  4717  				{"10.0.1.3", "host2"},
  4718  			},
  4719  			expectVirtualServer:      false,
  4720  			expectLocalEntries:       false,
  4721  			expectLocalRealServerNum: 0,
  4722  			expectLocalRealServers:   []string{},
  4723  		},
  4724  	}
  4725  	for _, tc := range testCases {
  4726  		ipt := iptablestest.NewFake()
  4727  		ipvs := ipvstest.NewFake()
  4728  		ipset := ipsettest.NewFake(testIPSetVersion)
  4729  		fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol)
  4730  		fp.servicesSynced = true
  4731  		// fp.endpointsSynced = true
  4732  		fp.endpointSlicesSynced = true
  4733  
  4734  		// Add initial service
  4735  		serviceName := "svc1"
  4736  		namespaceName := "ns1"
  4737  
  4738  		svc := &v1.Service{
  4739  			ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: namespaceName},
  4740  			Spec: v1.ServiceSpec{
  4741  				ClusterIP: "172.20.1.1",
  4742  				Selector:  map[string]string{"foo": "bar"},
  4743  				Ports:     []v1.ServicePort{{Name: "", TargetPort: intstr.FromInt32(80), Protocol: v1.ProtocolTCP}},
  4744  			},
  4745  		}
  4746  		if tc.internalTrafficPolicy != nil {
  4747  			svc.Spec.InternalTrafficPolicy = tc.internalTrafficPolicy
  4748  		}
  4749  
  4750  		fp.OnServiceAdd(svc)
  4751  
  4752  		// Add initial endpoint slice
  4753  		endpointSlice := &discovery.EndpointSlice{
  4754  			ObjectMeta: metav1.ObjectMeta{
  4755  				Name:      fmt.Sprintf("%s-1", serviceName),
  4756  				Namespace: namespaceName,
  4757  				Labels:    map[string]string{discovery.LabelServiceName: serviceName},
  4758  			},
  4759  			Ports: []discovery.EndpointPort{{
  4760  				Name:     ptr.To(""),
  4761  				Port:     ptr.To[int32](80),
  4762  				Protocol: ptr.To(v1.ProtocolTCP),
  4763  			}},
  4764  			AddressType: discovery.AddressTypeIPv4,
  4765  		}
  4766  
  4767  		for _, ep := range tc.endpoints {
  4768  			endpointSlice.Endpoints = append(endpointSlice.Endpoints, discovery.Endpoint{
  4769  				Addresses:  []string{ep.ip},
  4770  				Conditions: discovery.EndpointConditions{Ready: ptr.To(true)},
  4771  				NodeName:   ptr.To(ep.hostname),
  4772  			})
  4773  		}
  4774  
  4775  		fp.OnEndpointSliceAdd(endpointSlice)
  4776  		fp.syncProxyRules()
  4777  
  4778  		// Ensure that Proxier updates ipvs appropriately after EndpointSlice update
  4779  		assert.NotNil(t, fp.ipsetList["KUBE-LOOP-BACK"])
  4780  
  4781  		activeEntries1 := fp.ipsetList["KUBE-LOOP-BACK"].activeEntries
  4782  
  4783  		if tc.expectLocalEntries {
  4784  			assert.Equal(t, 1, activeEntries1.Len(), "Expected 1 active entry in KUBE-LOOP-BACK")
  4785  		} else {
  4786  			assert.Equal(t, 0, activeEntries1.Len(), "Expected no active entry in KUBE-LOOP-BACK")
  4787  		}
  4788  
  4789  		if tc.expectVirtualServer {
  4790  			virtualServers1, vsErr1 := ipvs.GetVirtualServers()
  4791  			assert.Nil(t, vsErr1, "Expected no error getting virtual servers")
  4792  
  4793  			assert.Len(t, virtualServers1, 1, "Expected 1 virtual server")
  4794  			realServers1, rsErr1 := ipvs.GetRealServers(virtualServers1[0])
  4795  			assert.Nil(t, rsErr1, "Expected no error getting real servers")
  4796  
  4797  			assert.Len(t, realServers1, tc.expectLocalRealServerNum, fmt.Sprintf("Expected %d real servers", tc.expectLocalRealServerNum))
  4798  			for i := 0; i < tc.expectLocalRealServerNum; i++ {
  4799  				assert.Equal(t, realServers1[i].String(), tc.expectLocalRealServers[i])
  4800  			}
  4801  		}
  4802  
  4803  		fp.OnEndpointSliceDelete(endpointSlice)
  4804  		fp.syncProxyRules()
  4805  
  4806  		// Ensure that Proxier updates ipvs appropriately after EndpointSlice delete
  4807  		assert.NotNil(t, fp.ipsetList["KUBE-LOOP-BACK"])
  4808  		activeEntries3 := fp.ipsetList["KUBE-LOOP-BACK"].activeEntries
  4809  		assert.Equal(t, 0, activeEntries3.Len(), "Expected 0 active entries in KUBE-LOOP-BACK")
  4810  		virtualServers2, vsErr2 := ipvs.GetVirtualServers()
  4811  		assert.Nil(t, vsErr2, "Expected no error getting virtual servers")
  4812  		assert.Len(t, virtualServers2, 1, "Expected 1 virtual server")
  4813  		realServers2, rsErr2 := ipvs.GetRealServers(virtualServers2[0])
  4814  		assert.Nil(t, rsErr2, "Expected no error getting real servers")
  4815  		assert.Len(t, realServers2, 0, "Expected 0 real servers")
  4816  	}
  4817  }
  4818  
  4819  // Test_EndpointSliceReadyAndTerminatingCluster tests that when there are ready and ready + terminating
  4820  // endpoints and the traffic policy is "Cluster", only the ready endpoints are used.
  4821  func Test_EndpointSliceReadyAndTerminatingCluster(t *testing.T) {
  4822  
  4823  	ipt := iptablestest.NewFake()
  4824  	ipvs := ipvstest.NewFake()
  4825  	ipset := ipsettest.NewFake(testIPSetVersion)
  4826  	fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol)
  4827  	fp.servicesSynced = true
  4828  	// fp.endpointsSynced = true
  4829  	fp.endpointSlicesSynced = true
  4830  
  4831  	serviceName := "svc1"
  4832  	// Add initial service
  4833  	namespaceName := "ns1"
  4834  	fp.OnServiceAdd(&v1.Service{
  4835  		ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: namespaceName},
  4836  		Spec: v1.ServiceSpec{
  4837  			ClusterIP:             "172.20.1.1",
  4838  			Selector:              map[string]string{"foo": "bar"},
  4839  			Type:                  v1.ServiceTypeNodePort,
  4840  			ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyCluster,
  4841  			InternalTrafficPolicy: ptr.To(v1.ServiceInternalTrafficPolicyCluster),
  4842  			ExternalIPs: []string{
  4843  				"1.2.3.4",
  4844  			},
  4845  			Ports: []v1.ServicePort{
  4846  				{
  4847  					Name:       "",
  4848  					Port:       80,
  4849  					TargetPort: intstr.FromInt32(80),
  4850  					Protocol:   v1.ProtocolTCP,
  4851  				},
  4852  			},
  4853  		},
  4854  	})
  4855  
  4856  	// Add initial endpoint slice
  4857  	endpointSlice := &discovery.EndpointSlice{
  4858  		ObjectMeta: metav1.ObjectMeta{
  4859  			Name:      fmt.Sprintf("%s-1", serviceName),
  4860  			Namespace: namespaceName,
  4861  			Labels:    map[string]string{discovery.LabelServiceName: serviceName},
  4862  		},
  4863  		Ports: []discovery.EndpointPort{{
  4864  			Name:     ptr.To(""),
  4865  			Port:     ptr.To[int32](80),
  4866  			Protocol: ptr.To(v1.ProtocolTCP),
  4867  		}},
  4868  		AddressType: discovery.AddressTypeIPv4,
  4869  		Endpoints: []discovery.Endpoint{
  4870  			{
  4871  				Addresses: []string{"10.0.1.1"},
  4872  				Conditions: discovery.EndpointConditions{
  4873  					Ready:       ptr.To(true),
  4874  					Serving:     ptr.To(true),
  4875  					Terminating: ptr.To(false),
  4876  				},
  4877  				NodeName: ptr.To(testHostname),
  4878  			},
  4879  			{
  4880  				Addresses: []string{"10.0.1.2"},
  4881  				Conditions: discovery.EndpointConditions{
  4882  					Ready:       ptr.To(true),
  4883  					Serving:     ptr.To(true),
  4884  					Terminating: ptr.To(false),
  4885  				},
  4886  				NodeName: ptr.To(testHostname),
  4887  			},
  4888  			{
  4889  				Addresses: []string{"10.0.1.3"},
  4890  				Conditions: discovery.EndpointConditions{
  4891  					Ready:       ptr.To(false),
  4892  					Serving:     ptr.To(true),
  4893  					Terminating: ptr.To(true),
  4894  				},
  4895  				NodeName: ptr.To(testHostname),
  4896  			},
  4897  			{
  4898  				Addresses: []string{"10.0.1.4"},
  4899  				Conditions: discovery.EndpointConditions{
  4900  					Ready:       ptr.To(false),
  4901  					Serving:     ptr.To(false),
  4902  					Terminating: ptr.To(true),
  4903  				},
  4904  				NodeName: ptr.To(testHostname),
  4905  			},
  4906  			{
  4907  				Addresses: []string{"10.0.1.5"},
  4908  				Conditions: discovery.EndpointConditions{
  4909  					Ready:       ptr.To(true),
  4910  					Serving:     ptr.To(true),
  4911  					Terminating: ptr.To(false),
  4912  				},
  4913  				NodeName: ptr.To("another-host"),
  4914  			},
  4915  		},
  4916  	}
  4917  
  4918  	fp.OnEndpointSliceAdd(endpointSlice)
  4919  	fp.syncProxyRules()
  4920  
  4921  	// Ensure that Proxier updates ipvs appropriately after EndpointSlice update
  4922  	assert.NotNil(t, fp.ipsetList["KUBE-LOOP-BACK"])
  4923  	activeEntries1 := fp.ipsetList["KUBE-LOOP-BACK"].activeEntries
  4924  	assert.Equal(t, 4, activeEntries1.Len(), "Expected 4 active entry in KUBE-LOOP-BACK")
  4925  	assert.Equal(t, true, activeEntries1.Has("10.0.1.1,tcp:80,10.0.1.1"), "Expected activeEntries to reference first pod")
  4926  	assert.Equal(t, true, activeEntries1.Has("10.0.1.2,tcp:80,10.0.1.2"), "Expected activeEntries to reference second pod")
  4927  	assert.Equal(t, true, activeEntries1.Has("10.0.1.3,tcp:80,10.0.1.3"), "Expected activeEntries to reference third pod")
  4928  	assert.Equal(t, true, activeEntries1.Has("10.0.1.4,tcp:80,10.0.1.4"), "Expected activeEntries to reference fourth pod")
  4929  
  4930  	virtualServers, vsErr := ipvs.GetVirtualServers()
  4931  	assert.Nil(t, vsErr, "Expected no error getting virtual servers")
  4932  	assert.Len(t, virtualServers, 2, "Expected 2 virtual server")
  4933  
  4934  	var clusterIPServer, externalIPServer *utilipvs.VirtualServer
  4935  	for _, virtualServer := range virtualServers {
  4936  		if virtualServer.Address.String() == "172.20.1.1" {
  4937  			clusterIPServer = virtualServer
  4938  		}
  4939  
  4940  		if virtualServer.Address.String() == "1.2.3.4" {
  4941  			externalIPServer = virtualServer
  4942  		}
  4943  	}
  4944  
  4945  	// clusterIP should route to cluster-wide ready endpoints
  4946  	realServers1, rsErr1 := ipvs.GetRealServers(clusterIPServer)
  4947  	assert.Nil(t, rsErr1, "Expected no error getting real servers")
  4948  	assert.Len(t, realServers1, 3, "Expected 3 real servers")
  4949  	assert.Equal(t, realServers1[0].String(), "10.0.1.1:80")
  4950  	assert.Equal(t, realServers1[1].String(), "10.0.1.2:80")
  4951  	assert.Equal(t, realServers1[2].String(), "10.0.1.5:80")
  4952  
  4953  	// externalIP should route to cluster-wide ready endpoints
  4954  	realServers2, rsErr2 := ipvs.GetRealServers(externalIPServer)
  4955  	assert.Nil(t, rsErr2, "Expected no error getting real servers")
  4956  	assert.Len(t, realServers2, 3, "Expected 3 real servers")
  4957  	assert.Equal(t, realServers2[0].String(), "10.0.1.1:80")
  4958  	assert.Equal(t, realServers2[1].String(), "10.0.1.2:80")
  4959  	assert.Equal(t, realServers1[2].String(), "10.0.1.5:80")
  4960  
  4961  	fp.OnEndpointSliceDelete(endpointSlice)
  4962  	fp.syncProxyRules()
  4963  
  4964  	// Ensure that Proxier updates ipvs appropriately after EndpointSlice delete
  4965  	assert.NotNil(t, fp.ipsetList["KUBE-LOOP-BACK"])
  4966  	activeEntries2 := fp.ipsetList["KUBE-LOOP-BACK"].activeEntries
  4967  	assert.Equal(t, 0, activeEntries2.Len(), "Expected 0 active entries in KUBE-LOOP-BACK")
  4968  
  4969  	virtualServers, vsErr = ipvs.GetVirtualServers()
  4970  	assert.Nil(t, vsErr, "Expected no error getting virtual servers")
  4971  	assert.Len(t, virtualServers, 2, "Expected 2 virtual server")
  4972  
  4973  	for _, virtualServer := range virtualServers {
  4974  		if virtualServer.Address.String() == "172.20.1.1" {
  4975  			clusterIPServer = virtualServer
  4976  		}
  4977  
  4978  		if virtualServer.Address.String() == "1.2.3.4" {
  4979  			externalIPServer = virtualServer
  4980  		}
  4981  	}
  4982  
  4983  	realServers1, rsErr1 = ipvs.GetRealServers(clusterIPServer)
  4984  	assert.Nil(t, rsErr1, "Expected no error getting real servers")
  4985  	assert.Len(t, realServers1, 0, "Expected 0 real servers")
  4986  
  4987  	realServers2, rsErr2 = ipvs.GetRealServers(externalIPServer)
  4988  	assert.Nil(t, rsErr2, "Expected no error getting real servers")
  4989  	assert.Len(t, realServers2, 0, "Expected 0 real servers")
  4990  }
  4991  
  4992  // Test_EndpointSliceReadyAndTerminatingLocal tests that when there are local ready and ready + terminating
  4993  // endpoints, only the ready endpoints are used.
  4994  func Test_EndpointSliceReadyAndTerminatingLocal(t *testing.T) {
  4995  
  4996  	ipt := iptablestest.NewFake()
  4997  	ipvs := ipvstest.NewFake()
  4998  	ipset := ipsettest.NewFake(testIPSetVersion)
  4999  	fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol)
  5000  	fp.servicesSynced = true
  5001  	// fp.endpointsSynced = true
  5002  	fp.endpointSlicesSynced = true
  5003  
  5004  	serviceName := "svc1"
  5005  	// Add initial service
  5006  	namespaceName := "ns1"
  5007  	fp.OnServiceAdd(&v1.Service{
  5008  		ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: namespaceName},
  5009  		Spec: v1.ServiceSpec{
  5010  			ClusterIP:             "172.20.1.1",
  5011  			Selector:              map[string]string{"foo": "bar"},
  5012  			Type:                  v1.ServiceTypeNodePort,
  5013  			ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyLocal,
  5014  			InternalTrafficPolicy: ptr.To(v1.ServiceInternalTrafficPolicyCluster),
  5015  			ExternalIPs: []string{
  5016  				"1.2.3.4",
  5017  			},
  5018  			Ports: []v1.ServicePort{
  5019  				{
  5020  					Name:       "",
  5021  					Port:       80,
  5022  					TargetPort: intstr.FromInt32(80),
  5023  					Protocol:   v1.ProtocolTCP,
  5024  				},
  5025  			},
  5026  		},
  5027  	})
  5028  
  5029  	// Add initial endpoint slice
  5030  	endpointSlice := &discovery.EndpointSlice{
  5031  		ObjectMeta: metav1.ObjectMeta{
  5032  			Name:      fmt.Sprintf("%s-1", serviceName),
  5033  			Namespace: namespaceName,
  5034  			Labels:    map[string]string{discovery.LabelServiceName: serviceName},
  5035  		},
  5036  		Ports: []discovery.EndpointPort{{
  5037  			Name:     ptr.To(""),
  5038  			Port:     ptr.To[int32](80),
  5039  			Protocol: ptr.To(v1.ProtocolTCP),
  5040  		}},
  5041  		AddressType: discovery.AddressTypeIPv4,
  5042  		Endpoints: []discovery.Endpoint{
  5043  			{
  5044  				Addresses: []string{"10.0.1.1"},
  5045  				Conditions: discovery.EndpointConditions{
  5046  					Ready:       ptr.To(true),
  5047  					Serving:     ptr.To(true),
  5048  					Terminating: ptr.To(false),
  5049  				},
  5050  				NodeName: ptr.To(testHostname),
  5051  			},
  5052  			{
  5053  				Addresses: []string{"10.0.1.2"},
  5054  				Conditions: discovery.EndpointConditions{
  5055  					Ready:       ptr.To(true),
  5056  					Serving:     ptr.To(true),
  5057  					Terminating: ptr.To(false),
  5058  				},
  5059  				NodeName: ptr.To(testHostname),
  5060  			},
  5061  			{
  5062  				Addresses: []string{"10.0.1.3"},
  5063  				Conditions: discovery.EndpointConditions{
  5064  					Ready:       ptr.To(false),
  5065  					Serving:     ptr.To(true),
  5066  					Terminating: ptr.To(true),
  5067  				},
  5068  				NodeName: ptr.To(testHostname),
  5069  			},
  5070  			{
  5071  				Addresses: []string{"10.0.1.4"},
  5072  				Conditions: discovery.EndpointConditions{
  5073  					Ready:       ptr.To(false),
  5074  					Serving:     ptr.To(false),
  5075  					Terminating: ptr.To(true),
  5076  				},
  5077  				NodeName: ptr.To(testHostname),
  5078  			},
  5079  			{
  5080  				Addresses: []string{"10.0.1.5"},
  5081  				Conditions: discovery.EndpointConditions{
  5082  					Ready:       ptr.To(true),
  5083  					Serving:     ptr.To(true),
  5084  					Terminating: ptr.To(false),
  5085  				},
  5086  				NodeName: ptr.To("another-host"),
  5087  			},
  5088  		},
  5089  	}
  5090  
  5091  	fp.OnEndpointSliceAdd(endpointSlice)
  5092  	fp.syncProxyRules()
  5093  
  5094  	// Ensure that Proxier updates ipvs appropriately after EndpointSlice update
  5095  	assert.NotNil(t, fp.ipsetList["KUBE-LOOP-BACK"])
  5096  	activeEntries1 := fp.ipsetList["KUBE-LOOP-BACK"].activeEntries
  5097  	assert.Equal(t, 4, activeEntries1.Len(), "Expected 3 active entry in KUBE-LOOP-BACK")
  5098  	assert.Equal(t, true, activeEntries1.Has("10.0.1.1,tcp:80,10.0.1.1"), "Expected activeEntries to reference first (local) pod")
  5099  	assert.Equal(t, true, activeEntries1.Has("10.0.1.2,tcp:80,10.0.1.2"), "Expected activeEntries to reference second (local) pod")
  5100  	assert.Equal(t, true, activeEntries1.Has("10.0.1.3,tcp:80,10.0.1.3"), "Expected activeEntries to reference second (local) pod")
  5101  	assert.Equal(t, true, activeEntries1.Has("10.0.1.4,tcp:80,10.0.1.4"), "Expected activeEntries to reference second (local) pod")
  5102  
  5103  	virtualServers, vsErr := ipvs.GetVirtualServers()
  5104  	assert.Nil(t, vsErr, "Expected no error getting virtual servers")
  5105  	assert.Len(t, virtualServers, 2, "Expected 2 virtual server")
  5106  
  5107  	var clusterIPServer, externalIPServer *utilipvs.VirtualServer
  5108  	for _, virtualServer := range virtualServers {
  5109  		if virtualServer.Address.String() == "172.20.1.1" {
  5110  			clusterIPServer = virtualServer
  5111  		}
  5112  
  5113  		if virtualServer.Address.String() == "1.2.3.4" {
  5114  			externalIPServer = virtualServer
  5115  		}
  5116  	}
  5117  
  5118  	// clusterIP should route to cluster-wide ready endpoints
  5119  	realServers1, rsErr1 := ipvs.GetRealServers(clusterIPServer)
  5120  	assert.Nil(t, rsErr1, "Expected no error getting real servers")
  5121  	assert.Len(t, realServers1, 3, "Expected 3 real servers")
  5122  	assert.Equal(t, realServers1[0].String(), "10.0.1.1:80")
  5123  	assert.Equal(t, realServers1[1].String(), "10.0.1.2:80")
  5124  	assert.Equal(t, realServers1[2].String(), "10.0.1.5:80")
  5125  
  5126  	// externalIP should route to local ready + non-terminating endpoints if they exist
  5127  	realServers2, rsErr2 := ipvs.GetRealServers(externalIPServer)
  5128  	assert.Nil(t, rsErr2, "Expected no error getting real servers")
  5129  	assert.Len(t, realServers2, 2, "Expected 2 real servers")
  5130  	assert.Equal(t, realServers2[0].String(), "10.0.1.1:80")
  5131  	assert.Equal(t, realServers2[1].String(), "10.0.1.2:80")
  5132  
  5133  	fp.OnEndpointSliceDelete(endpointSlice)
  5134  	fp.syncProxyRules()
  5135  
  5136  	// Ensure that Proxier updates ipvs appropriately after EndpointSlice delete
  5137  	assert.NotNil(t, fp.ipsetList["KUBE-LOOP-BACK"])
  5138  	activeEntries2 := fp.ipsetList["KUBE-LOOP-BACK"].activeEntries
  5139  	assert.Equal(t, 0, activeEntries2.Len(), "Expected 0 active entries in KUBE-LOOP-BACK")
  5140  
  5141  	virtualServers, vsErr = ipvs.GetVirtualServers()
  5142  	assert.Nil(t, vsErr, "Expected no error getting virtual servers")
  5143  	assert.Len(t, virtualServers, 2, "Expected 2 virtual server")
  5144  
  5145  	for _, virtualServer := range virtualServers {
  5146  		if virtualServer.Address.String() == "172.20.1.1" {
  5147  			clusterIPServer = virtualServer
  5148  		}
  5149  
  5150  		if virtualServer.Address.String() == "1.2.3.4" {
  5151  			externalIPServer = virtualServer
  5152  		}
  5153  	}
  5154  
  5155  	realServers1, rsErr1 = ipvs.GetRealServers(clusterIPServer)
  5156  	assert.Nil(t, rsErr1, "Expected no error getting real servers")
  5157  	assert.Len(t, realServers1, 0, "Expected 0 real servers")
  5158  
  5159  	realServers2, rsErr2 = ipvs.GetRealServers(externalIPServer)
  5160  	assert.Nil(t, rsErr2, "Expected no error getting real servers")
  5161  	assert.Len(t, realServers2, 0, "Expected 0 real servers")
  5162  }
  5163  
  5164  // Test_EndpointSliceOnlyReadyTerminatingCluster tests that when there are only ready terminating
  5165  // endpoints and the traffic policy is "Cluster",  we fall back to terminating endpoints.
  5166  func Test_EndpointSliceOnlyReadyAndTerminatingCluster(t *testing.T) {
  5167  
  5168  	ipt := iptablestest.NewFake()
  5169  	ipvs := ipvstest.NewFake()
  5170  	ipset := ipsettest.NewFake(testIPSetVersion)
  5171  	fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol)
  5172  	fp.servicesSynced = true
  5173  	// fp.endpointsSynced = true
  5174  	fp.endpointSlicesSynced = true
  5175  
  5176  	// Add initial service
  5177  	serviceName := "svc1"
  5178  	namespaceName := "ns1"
  5179  	fp.OnServiceAdd(&v1.Service{
  5180  		ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: namespaceName},
  5181  		Spec: v1.ServiceSpec{
  5182  			ClusterIP:             "172.20.1.1",
  5183  			Selector:              map[string]string{"foo": "bar"},
  5184  			Type:                  v1.ServiceTypeNodePort,
  5185  			ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyCluster,
  5186  			InternalTrafficPolicy: ptr.To(v1.ServiceInternalTrafficPolicyCluster),
  5187  			ExternalIPs: []string{
  5188  				"1.2.3.4",
  5189  			},
  5190  			Ports: []v1.ServicePort{
  5191  				{
  5192  					Name:       "",
  5193  					Port:       80,
  5194  					TargetPort: intstr.FromInt32(80),
  5195  					Protocol:   v1.ProtocolTCP,
  5196  				},
  5197  			},
  5198  		},
  5199  	})
  5200  
  5201  	// Add initial endpoint slice
  5202  	endpointSlice := &discovery.EndpointSlice{
  5203  		ObjectMeta: metav1.ObjectMeta{
  5204  			Name:      fmt.Sprintf("%s-1", serviceName),
  5205  			Namespace: namespaceName,
  5206  			Labels:    map[string]string{discovery.LabelServiceName: serviceName},
  5207  		},
  5208  		Ports: []discovery.EndpointPort{{
  5209  			Name:     ptr.To(""),
  5210  			Port:     ptr.To[int32](80),
  5211  			Protocol: ptr.To(v1.ProtocolTCP),
  5212  		}},
  5213  		AddressType: discovery.AddressTypeIPv4,
  5214  		Endpoints: []discovery.Endpoint{
  5215  			{
  5216  				Addresses: []string{"10.0.1.1"},
  5217  				Conditions: discovery.EndpointConditions{
  5218  					Ready:       ptr.To(false),
  5219  					Serving:     ptr.To(true),
  5220  					Terminating: ptr.To(true),
  5221  				},
  5222  				NodeName: ptr.To(testHostname),
  5223  			},
  5224  			{
  5225  				Addresses: []string{"10.0.1.2"},
  5226  				Conditions: discovery.EndpointConditions{
  5227  					Ready:       ptr.To(false),
  5228  					Serving:     ptr.To(true),
  5229  					Terminating: ptr.To(true),
  5230  				},
  5231  				NodeName: ptr.To(testHostname),
  5232  			},
  5233  			{
  5234  				Addresses: []string{"10.0.1.3"},
  5235  				Conditions: discovery.EndpointConditions{
  5236  					Ready:       ptr.To(false),
  5237  					Serving:     ptr.To(false),
  5238  					Terminating: ptr.To(true),
  5239  				},
  5240  				NodeName: ptr.To(testHostname),
  5241  			},
  5242  			{
  5243  				Addresses: []string{"10.0.1.4"},
  5244  				Conditions: discovery.EndpointConditions{
  5245  					Ready:       ptr.To(false),
  5246  					Serving:     ptr.To(true),
  5247  					Terminating: ptr.To(true),
  5248  				},
  5249  				NodeName: ptr.To("another-host"),
  5250  			},
  5251  			{
  5252  				Addresses: []string{"10.0.1.5"},
  5253  				Conditions: discovery.EndpointConditions{
  5254  					Ready:       ptr.To(false),
  5255  					Serving:     ptr.To(false),
  5256  					Terminating: ptr.To(false),
  5257  				},
  5258  				NodeName: ptr.To("another-host"),
  5259  			},
  5260  		},
  5261  	}
  5262  
  5263  	fp.OnEndpointSliceAdd(endpointSlice)
  5264  	fp.syncProxyRules()
  5265  
  5266  	// Ensure that Proxier updates ipvs appropriately after EndpointSlice update
  5267  	assert.NotNil(t, fp.ipsetList["KUBE-LOOP-BACK"])
  5268  	activeEntries1 := fp.ipsetList["KUBE-LOOP-BACK"].activeEntries
  5269  	assert.Equal(t, 3, activeEntries1.Len(), "Expected 3 active entry in KUBE-LOOP-BACK")
  5270  	assert.Equal(t, true, activeEntries1.Has("10.0.1.1,tcp:80,10.0.1.1"), "Expected activeEntries to reference first (local) pod")
  5271  	assert.Equal(t, true, activeEntries1.Has("10.0.1.2,tcp:80,10.0.1.2"), "Expected activeEntries to reference second (local) pod")
  5272  	assert.Equal(t, true, activeEntries1.Has("10.0.1.3,tcp:80,10.0.1.3"), "Expected activeEntries to reference second (local) pod")
  5273  
  5274  	virtualServers, vsErr := ipvs.GetVirtualServers()
  5275  	assert.Nil(t, vsErr, "Expected no error getting virtual servers")
  5276  	assert.Len(t, virtualServers, 2, "Expected 2 virtual server")
  5277  
  5278  	var clusterIPServer, externalIPServer *utilipvs.VirtualServer
  5279  	for _, virtualServer := range virtualServers {
  5280  		if virtualServer.Address.String() == "172.20.1.1" {
  5281  			clusterIPServer = virtualServer
  5282  		}
  5283  
  5284  		if virtualServer.Address.String() == "1.2.3.4" {
  5285  			externalIPServer = virtualServer
  5286  		}
  5287  	}
  5288  
  5289  	// clusterIP should fall back to cluster-wide ready + terminating endpoints
  5290  	realServers1, rsErr1 := ipvs.GetRealServers(clusterIPServer)
  5291  	assert.Nil(t, rsErr1, "Expected no error getting real servers")
  5292  	assert.Len(t, realServers1, 3, "Expected 1 real servers")
  5293  	assert.Equal(t, realServers1[0].String(), "10.0.1.1:80")
  5294  	assert.Equal(t, realServers1[1].String(), "10.0.1.2:80")
  5295  	assert.Equal(t, realServers1[2].String(), "10.0.1.4:80")
  5296  
  5297  	// externalIP should fall back to ready + terminating endpoints
  5298  	realServers2, rsErr2 := ipvs.GetRealServers(externalIPServer)
  5299  	assert.Nil(t, rsErr2, "Expected no error getting real servers")
  5300  	assert.Len(t, realServers2, 3, "Expected 2 real servers")
  5301  	assert.Equal(t, realServers2[0].String(), "10.0.1.1:80")
  5302  	assert.Equal(t, realServers2[1].String(), "10.0.1.2:80")
  5303  	assert.Equal(t, realServers2[2].String(), "10.0.1.4:80")
  5304  
  5305  	fp.OnEndpointSliceDelete(endpointSlice)
  5306  	fp.syncProxyRules()
  5307  
  5308  	// Ensure that Proxier updates ipvs appropriately after EndpointSlice delete
  5309  	assert.NotNil(t, fp.ipsetList["KUBE-LOOP-BACK"])
  5310  	activeEntries2 := fp.ipsetList["KUBE-LOOP-BACK"].activeEntries
  5311  	assert.Equal(t, 0, activeEntries2.Len(), "Expected 0 active entries in KUBE-LOOP-BACK")
  5312  
  5313  	virtualServers, vsErr = ipvs.GetVirtualServers()
  5314  	assert.Nil(t, vsErr, "Expected no error getting virtual servers")
  5315  	assert.Len(t, virtualServers, 2, "Expected 2 virtual server")
  5316  
  5317  	for _, virtualServer := range virtualServers {
  5318  		if virtualServer.Address.String() == "172.20.1.1" {
  5319  			clusterIPServer = virtualServer
  5320  		}
  5321  
  5322  		if virtualServer.Address.String() == "1.2.3.4" {
  5323  			externalIPServer = virtualServer
  5324  		}
  5325  	}
  5326  
  5327  	realServers1, rsErr1 = ipvs.GetRealServers(clusterIPServer)
  5328  	assert.Nil(t, rsErr1, "Expected no error getting real servers")
  5329  	assert.Len(t, realServers1, 0, "Expected 0 real servers")
  5330  
  5331  	realServers2, rsErr2 = ipvs.GetRealServers(externalIPServer)
  5332  	assert.Nil(t, rsErr2, "Expected no error getting real servers")
  5333  	assert.Len(t, realServers2, 0, "Expected 0 real servers")
  5334  }
  5335  
  5336  // Test_EndpointSliceOnlyReadyTerminatingLocal tests that when there are only local ready terminating
  5337  // endpoints, we fall back to those endpoints.
  5338  func Test_EndpointSliceOnlyReadyAndTerminatingLocal(t *testing.T) {
  5339  
  5340  	ipt := iptablestest.NewFake()
  5341  	ipvs := ipvstest.NewFake()
  5342  	ipset := ipsettest.NewFake(testIPSetVersion)
  5343  	fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, v1.IPv4Protocol)
  5344  	fp.servicesSynced = true
  5345  	// fp.endpointsSynced = true
  5346  	fp.endpointSlicesSynced = true
  5347  
  5348  	// Add initial service
  5349  	serviceName := "svc1"
  5350  	namespaceName := "ns1"
  5351  	fp.OnServiceAdd(&v1.Service{
  5352  		ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: namespaceName},
  5353  		Spec: v1.ServiceSpec{
  5354  			ClusterIP:             "172.20.1.1",
  5355  			Selector:              map[string]string{"foo": "bar"},
  5356  			Type:                  v1.ServiceTypeNodePort,
  5357  			ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyLocal,
  5358  			InternalTrafficPolicy: ptr.To(v1.ServiceInternalTrafficPolicyCluster),
  5359  			ExternalIPs: []string{
  5360  				"1.2.3.4",
  5361  			},
  5362  			Ports: []v1.ServicePort{
  5363  				{
  5364  					Name:       "",
  5365  					Port:       80,
  5366  					TargetPort: intstr.FromInt32(80),
  5367  					Protocol:   v1.ProtocolTCP,
  5368  				},
  5369  			},
  5370  		},
  5371  	})
  5372  
  5373  	// Add initial endpoint slice
  5374  	endpointSlice := &discovery.EndpointSlice{
  5375  		ObjectMeta: metav1.ObjectMeta{
  5376  			Name:      fmt.Sprintf("%s-1", serviceName),
  5377  			Namespace: namespaceName,
  5378  			Labels:    map[string]string{discovery.LabelServiceName: serviceName},
  5379  		},
  5380  		Ports: []discovery.EndpointPort{{
  5381  			Name:     ptr.To(""),
  5382  			Port:     ptr.To[int32](80),
  5383  			Protocol: ptr.To(v1.ProtocolTCP),
  5384  		}},
  5385  		AddressType: discovery.AddressTypeIPv4,
  5386  		Endpoints: []discovery.Endpoint{
  5387  			{
  5388  				Addresses: []string{"10.0.1.1"},
  5389  				Conditions: discovery.EndpointConditions{
  5390  					Ready:       ptr.To(false),
  5391  					Serving:     ptr.To(true),
  5392  					Terminating: ptr.To(true),
  5393  				},
  5394  				NodeName: ptr.To(testHostname),
  5395  			},
  5396  			{
  5397  				Addresses: []string{"10.0.1.2"},
  5398  				Conditions: discovery.EndpointConditions{
  5399  					Ready:       ptr.To(false),
  5400  					Serving:     ptr.To(true),
  5401  					Terminating: ptr.To(true),
  5402  				},
  5403  				NodeName: ptr.To(testHostname),
  5404  			},
  5405  			{
  5406  				Addresses: []string{"10.0.1.3"},
  5407  				Conditions: discovery.EndpointConditions{
  5408  					Ready:       ptr.To(false),
  5409  					Serving:     ptr.To(false),
  5410  					Terminating: ptr.To(true),
  5411  				},
  5412  				NodeName: ptr.To(testHostname),
  5413  			},
  5414  			{
  5415  				Addresses: []string{"10.0.1.4"},
  5416  				Conditions: discovery.EndpointConditions{
  5417  					Ready:       ptr.To(false),
  5418  					Serving:     ptr.To(true),
  5419  					Terminating: ptr.To(true),
  5420  				},
  5421  				NodeName: ptr.To("another-host"),
  5422  			},
  5423  			{
  5424  				Addresses: []string{"10.0.1.5"},
  5425  				Conditions: discovery.EndpointConditions{
  5426  					Ready:       ptr.To(true),
  5427  					Serving:     ptr.To(true),
  5428  					Terminating: ptr.To(false),
  5429  				},
  5430  				NodeName: ptr.To("another-host"),
  5431  			},
  5432  		},
  5433  	}
  5434  
  5435  	fp.OnEndpointSliceAdd(endpointSlice)
  5436  	fp.syncProxyRules()
  5437  
  5438  	// Ensure that Proxier updates ipvs appropriately after EndpointSlice update
  5439  	assert.NotNil(t, fp.ipsetList["KUBE-LOOP-BACK"])
  5440  	activeEntries1 := fp.ipsetList["KUBE-LOOP-BACK"].activeEntries
  5441  	assert.Equal(t, 3, activeEntries1.Len(), "Expected 3 active entry in KUBE-LOOP-BACK")
  5442  	assert.Equal(t, true, activeEntries1.Has("10.0.1.1,tcp:80,10.0.1.1"), "Expected activeEntries to reference first (local) pod")
  5443  	assert.Equal(t, true, activeEntries1.Has("10.0.1.2,tcp:80,10.0.1.2"), "Expected activeEntries to reference second (local) pod")
  5444  	assert.Equal(t, true, activeEntries1.Has("10.0.1.3,tcp:80,10.0.1.3"), "Expected activeEntries to reference second (local) pod")
  5445  
  5446  	virtualServers, vsErr := ipvs.GetVirtualServers()
  5447  	assert.Nil(t, vsErr, "Expected no error getting virtual servers")
  5448  	assert.Len(t, virtualServers, 2, "Expected 2 virtual server")
  5449  
  5450  	var clusterIPServer, externalIPServer *utilipvs.VirtualServer
  5451  	for _, virtualServer := range virtualServers {
  5452  		if virtualServer.Address.String() == "172.20.1.1" {
  5453  			clusterIPServer = virtualServer
  5454  		}
  5455  
  5456  		if virtualServer.Address.String() == "1.2.3.4" {
  5457  			externalIPServer = virtualServer
  5458  		}
  5459  	}
  5460  
  5461  	// clusterIP should route to cluster-wide Ready endpoints
  5462  	realServers1, rsErr1 := ipvs.GetRealServers(clusterIPServer)
  5463  	assert.Nil(t, rsErr1, "Expected no error getting real servers")
  5464  	assert.Len(t, realServers1, 1, "Expected 1 real servers")
  5465  	assert.Equal(t, realServers1[0].String(), "10.0.1.5:80")
  5466  
  5467  	// externalIP should fall back to local ready + terminating endpoints
  5468  	realServers2, rsErr2 := ipvs.GetRealServers(externalIPServer)
  5469  	assert.Nil(t, rsErr2, "Expected no error getting real servers")
  5470  	assert.Len(t, realServers2, 2, "Expected 2 real servers")
  5471  	assert.Equal(t, realServers2[0].String(), "10.0.1.1:80")
  5472  	assert.Equal(t, realServers2[1].String(), "10.0.1.2:80")
  5473  
  5474  	fp.OnEndpointSliceDelete(endpointSlice)
  5475  	fp.syncProxyRules()
  5476  
  5477  	// Ensure that Proxier updates ipvs appropriately after EndpointSlice delete
  5478  	assert.NotNil(t, fp.ipsetList["KUBE-LOOP-BACK"])
  5479  	activeEntries2 := fp.ipsetList["KUBE-LOOP-BACK"].activeEntries
  5480  	assert.Equal(t, 0, activeEntries2.Len(), "Expected 0 active entries in KUBE-LOOP-BACK")
  5481  
  5482  	virtualServers, vsErr = ipvs.GetVirtualServers()
  5483  	assert.Nil(t, vsErr, "Expected no error getting virtual servers")
  5484  	assert.Len(t, virtualServers, 2, "Expected 2 virtual server")
  5485  
  5486  	for _, virtualServer := range virtualServers {
  5487  		if virtualServer.Address.String() == "172.20.1.1" {
  5488  			clusterIPServer = virtualServer
  5489  		}
  5490  
  5491  		if virtualServer.Address.String() == "1.2.3.4" {
  5492  			externalIPServer = virtualServer
  5493  		}
  5494  	}
  5495  
  5496  	realServers1, rsErr1 = ipvs.GetRealServers(clusterIPServer)
  5497  	assert.Nil(t, rsErr1, "Expected no error getting real servers")
  5498  	assert.Len(t, realServers1, 0, "Expected 0 real servers")
  5499  
  5500  	realServers2, rsErr2 = ipvs.GetRealServers(externalIPServer)
  5501  	assert.Nil(t, rsErr2, "Expected no error getting real servers")
  5502  	assert.Len(t, realServers2, 0, "Expected 0 real servers")
  5503  }
  5504  
  5505  func TestIpIsValidForSet(t *testing.T) {
  5506  	testCases := []struct {
  5507  		isIPv6 bool
  5508  		ip     string
  5509  		res    bool
  5510  	}{
  5511  		{
  5512  			false,
  5513  			"127.0.0.1",
  5514  			false,
  5515  		},
  5516  		{
  5517  			false,
  5518  			"127.0.0.0",
  5519  			false,
  5520  		},
  5521  		{
  5522  			false,
  5523  			"127.6.7.8",
  5524  			false,
  5525  		},
  5526  		{
  5527  			false,
  5528  			"8.8.8.8",
  5529  			true,
  5530  		},
  5531  		{
  5532  			false,
  5533  			"192.168.0.1",
  5534  			true,
  5535  		},
  5536  		{
  5537  			false,
  5538  			"169.254.0.0",
  5539  			true,
  5540  		},
  5541  		{
  5542  			false,
  5543  			"::ffff:169.254.0.0", // IPv6 mapped IPv4
  5544  			true,
  5545  		},
  5546  		{
  5547  			false,
  5548  			"1000::",
  5549  			false,
  5550  		},
  5551  		// IPv6
  5552  		{
  5553  			true,
  5554  			"::1",
  5555  			false,
  5556  		},
  5557  		{
  5558  			true,
  5559  			"1000::",
  5560  			true,
  5561  		},
  5562  		{
  5563  			true,
  5564  			"fe80::200:ff:fe01:1",
  5565  			false,
  5566  		},
  5567  		{
  5568  			true,
  5569  			"8.8.8.8",
  5570  			false,
  5571  		},
  5572  		{
  5573  			true,
  5574  			"::ffff:8.8.8.8",
  5575  			false,
  5576  		},
  5577  	}
  5578  
  5579  	for _, tc := range testCases {
  5580  		v := &netlinkHandle{}
  5581  		v.isIPv6 = tc.isIPv6
  5582  		ip := netutils.ParseIPSloppy(tc.ip)
  5583  		if ip == nil {
  5584  			t.Errorf("Parse error: %s", tc.ip)
  5585  		}
  5586  		if v.isValidForSet(ip) != tc.res {
  5587  			if tc.isIPv6 {
  5588  				t.Errorf("IPv6: %s", tc.ip)
  5589  			} else {
  5590  				t.Errorf("IPv4: %s", tc.ip)
  5591  			}
  5592  		}
  5593  	}
  5594  }
  5595  
  5596  func TestNoEndpointsMetric(t *testing.T) {
  5597  	type endpoint struct {
  5598  		ip       string
  5599  		hostname string
  5600  	}
  5601  
  5602  	metrics.RegisterMetrics()
  5603  
  5604  	testCases := []struct {
  5605  		name                                                string
  5606  		internalTrafficPolicy                               *v1.ServiceInternalTrafficPolicy
  5607  		externalTrafficPolicy                               v1.ServiceExternalTrafficPolicy
  5608  		endpoints                                           []endpoint
  5609  		expectedSyncProxyRulesNoLocalEndpointsTotalInternal int
  5610  		expectedSyncProxyRulesNoLocalEndpointsTotalExternal int
  5611  	}{
  5612  		{
  5613  			name:                  "internalTrafficPolicy is set and there are local endpoints",
  5614  			internalTrafficPolicy: ptr.To(v1.ServiceInternalTrafficPolicyLocal),
  5615  			endpoints: []endpoint{
  5616  				{"10.0.1.1", testHostname},
  5617  				{"10.0.1.2", "host1"},
  5618  				{"10.0.1.3", "host2"},
  5619  			},
  5620  		},
  5621  		{
  5622  			name:                  "externalTrafficPolicy is set and there are local endpoints",
  5623  			externalTrafficPolicy: v1.ServiceExternalTrafficPolicyLocal,
  5624  			endpoints: []endpoint{
  5625  				{"10.0.1.1", testHostname},
  5626  				{"10.0.1.2", "host1"},
  5627  				{"10.0.1.3", "host2"},
  5628  			},
  5629  		},
  5630  		{
  5631  			name:                  "both policies are set and there are local endpoints",
  5632  			internalTrafficPolicy: ptr.To(v1.ServiceInternalTrafficPolicyLocal),
  5633  			externalTrafficPolicy: v1.ServiceExternalTrafficPolicyLocal,
  5634  			endpoints: []endpoint{
  5635  				{"10.0.1.1", testHostname},
  5636  				{"10.0.1.2", "host1"},
  5637  				{"10.0.1.3", "host2"},
  5638  			},
  5639  		},
  5640  		{
  5641  			name:                  "internalTrafficPolicy is set and there are no local endpoints",
  5642  			internalTrafficPolicy: ptr.To(v1.ServiceInternalTrafficPolicyLocal),
  5643  			endpoints: []endpoint{
  5644  				{"10.0.1.1", "host0"},
  5645  				{"10.0.1.2", "host1"},
  5646  				{"10.0.1.3", "host2"},
  5647  			},
  5648  			expectedSyncProxyRulesNoLocalEndpointsTotalInternal: 1,
  5649  		},
  5650  		{
  5651  			name:                  "externalTrafficPolicy is set and there are no local endpoints",
  5652  			externalTrafficPolicy: v1.ServiceExternalTrafficPolicyLocal,
  5653  			endpoints: []endpoint{
  5654  				{"10.0.1.1", "host0"},
  5655  				{"10.0.1.2", "host1"},
  5656  				{"10.0.1.3", "host2"},
  5657  			},
  5658  			expectedSyncProxyRulesNoLocalEndpointsTotalExternal: 1,
  5659  		},
  5660  		{
  5661  			name:                  "Both policies are set and there are no local endpoints",
  5662  			internalTrafficPolicy: ptr.To(v1.ServiceInternalTrafficPolicyLocal),
  5663  			externalTrafficPolicy: v1.ServiceExternalTrafficPolicyLocal,
  5664  			endpoints: []endpoint{
  5665  				{"10.0.1.1", "host0"},
  5666  				{"10.0.1.2", "host1"},
  5667  				{"10.0.1.3", "host2"},
  5668  			},
  5669  			expectedSyncProxyRulesNoLocalEndpointsTotalInternal: 1,
  5670  			expectedSyncProxyRulesNoLocalEndpointsTotalExternal: 1,
  5671  		},
  5672  		{
  5673  			name:                  "Both policies are set and there are no endpoints at all",
  5674  			internalTrafficPolicy: ptr.To(v1.ServiceInternalTrafficPolicyLocal),
  5675  			externalTrafficPolicy: v1.ServiceExternalTrafficPolicyLocal,
  5676  			endpoints:             []endpoint{},
  5677  			expectedSyncProxyRulesNoLocalEndpointsTotalInternal: 0,
  5678  			expectedSyncProxyRulesNoLocalEndpointsTotalExternal: 0,
  5679  		},
  5680  	}
  5681  	for _, tc := range testCases {
  5682  		ipt := iptablestest.NewFake()
  5683  		ipvs := ipvstest.NewFake()
  5684  		ipset := ipsettest.NewFake(testIPSetVersion)
  5685  		fp := NewFakeProxier(ipt, ipvs, ipset, []string{"10.0.0.1"}, nil, v1.IPv4Protocol)
  5686  		fp.servicesSynced = true
  5687  		// fp.endpointsSynced = true
  5688  		fp.endpointSlicesSynced = true
  5689  
  5690  		// Add initial service
  5691  		serviceName := "svc1"
  5692  		namespaceName := "ns1"
  5693  
  5694  		svc := &v1.Service{
  5695  			ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: namespaceName},
  5696  			Spec: v1.ServiceSpec{
  5697  				ClusterIP: "172.20.1.1",
  5698  				Selector:  map[string]string{"foo": "bar"},
  5699  				Ports:     []v1.ServicePort{{Name: "p80", Port: 80, TargetPort: intstr.FromInt32(80), Protocol: v1.ProtocolTCP, NodePort: 30000}},
  5700  			},
  5701  		}
  5702  		if tc.internalTrafficPolicy != nil {
  5703  			svc.Spec.InternalTrafficPolicy = tc.internalTrafficPolicy
  5704  		}
  5705  		if tc.externalTrafficPolicy != "" {
  5706  			svc.Spec.Type = v1.ServiceTypeNodePort
  5707  			svc.Spec.ExternalTrafficPolicy = tc.externalTrafficPolicy
  5708  		}
  5709  
  5710  		fp.OnServiceAdd(svc)
  5711  
  5712  		// Add initial endpoint slice
  5713  		endpointSlice := &discovery.EndpointSlice{
  5714  			ObjectMeta: metav1.ObjectMeta{
  5715  				Name:      fmt.Sprintf("%s-1", serviceName),
  5716  				Namespace: namespaceName,
  5717  				Labels:    map[string]string{discovery.LabelServiceName: serviceName},
  5718  			},
  5719  			Ports: []discovery.EndpointPort{{
  5720  				Name:     ptr.To("p80"),
  5721  				Port:     ptr.To[int32](80),
  5722  				Protocol: ptr.To(v1.ProtocolTCP),
  5723  			}},
  5724  			AddressType: discovery.AddressTypeIPv4,
  5725  		}
  5726  
  5727  		for _, ep := range tc.endpoints {
  5728  			endpointSlice.Endpoints = append(endpointSlice.Endpoints, discovery.Endpoint{
  5729  				Addresses:  []string{ep.ip},
  5730  				Conditions: discovery.EndpointConditions{Ready: ptr.To(true)},
  5731  				NodeName:   ptr.To(ep.hostname),
  5732  			})
  5733  		}
  5734  
  5735  		fp.OnEndpointSliceAdd(endpointSlice)
  5736  		fp.syncProxyRules()
  5737  
  5738  		syncProxyRulesNoLocalEndpointsTotalInternal, err := testutil.GetGaugeMetricValue(metrics.SyncProxyRulesNoLocalEndpointsTotal.WithLabelValues("internal"))
  5739  		if err != nil {
  5740  			t.Errorf("failed to get %s value(internal), err: %v", metrics.SyncProxyRulesNoLocalEndpointsTotal.Name, err)
  5741  		}
  5742  
  5743  		if tc.expectedSyncProxyRulesNoLocalEndpointsTotalInternal != int(syncProxyRulesNoLocalEndpointsTotalInternal) {
  5744  			t.Errorf("sync_proxy_rules_no_endpoints_total metric mismatch(internal): got=%d, expected %d", int(syncProxyRulesNoLocalEndpointsTotalInternal), tc.expectedSyncProxyRulesNoLocalEndpointsTotalInternal)
  5745  		}
  5746  
  5747  		syncProxyRulesNoLocalEndpointsTotalExternal, err := testutil.GetGaugeMetricValue(metrics.SyncProxyRulesNoLocalEndpointsTotal.WithLabelValues("external"))
  5748  		if err != nil {
  5749  			t.Errorf("failed to get %s value(external), err: %v", metrics.SyncProxyRulesNoLocalEndpointsTotal.Name, err)
  5750  		}
  5751  
  5752  		if tc.expectedSyncProxyRulesNoLocalEndpointsTotalExternal != int(syncProxyRulesNoLocalEndpointsTotalExternal) {
  5753  			t.Errorf("sync_proxy_rules_no_endpoints_total metric mismatch(external): got=%d, expected %d", int(syncProxyRulesNoLocalEndpointsTotalExternal), tc.expectedSyncProxyRulesNoLocalEndpointsTotalExternal)
  5754  		}
  5755  	}
  5756  }
  5757  
  5758  func TestDismissLocalhostRuleExist(t *testing.T) {
  5759  	tests := []struct {
  5760  		name     string
  5761  		ipFamily v1.IPFamily
  5762  		src      string
  5763  	}{
  5764  		{
  5765  			name:     "ipv4 rule",
  5766  			ipFamily: v1.IPv4Protocol,
  5767  			src:      "127.0.0.0/8",
  5768  		},
  5769  		{
  5770  			name:     "ipv6 rule",
  5771  			ipFamily: v1.IPv6Protocol,
  5772  			src:      "::1/128",
  5773  		},
  5774  	}
  5775  	for _, test := range tests {
  5776  		t.Run(test.name, func(t *testing.T) {
  5777  			ipt := iptablestest.NewFake()
  5778  			if test.ipFamily == v1.IPv6Protocol {
  5779  				ipt = iptablestest.NewIPv6Fake()
  5780  			}
  5781  			ipvs := ipvstest.NewFake()
  5782  			ipset := ipsettest.NewFake(testIPSetVersion)
  5783  			fp := NewFakeProxier(ipt, ipvs, ipset, nil, nil, test.ipFamily)
  5784  
  5785  			fp.syncProxyRules()
  5786  
  5787  			rules := getRules(ipt, kubeServicesChain)
  5788  			if len(rules) <= 0 {
  5789  				t.Errorf("skip loop back ip in kubeservice chain not exist")
  5790  				return
  5791  			}
  5792  			if !rules[0].Jump.Matches("RETURN") || !rules[0].SourceAddress.Matches(test.src) {
  5793  				t.Errorf("rules not match, expect jump: %s, got: %s; expect source address: %s, got: %s", "RETURN", rules[0].Jump.String(), test.src, rules[0].SourceAddress.String())
  5794  			}
  5795  		})
  5796  	}
  5797  }
  5798  
  5799  func TestLoadBalancerIngressRouteTypeProxy(t *testing.T) {
  5800  	testCases := []struct {
  5801  		name             string
  5802  		ipModeEnabled    bool
  5803  		svcIP            string
  5804  		svcLBIP          string
  5805  		ipMode           *v1.LoadBalancerIPMode
  5806  		expectedServices int
  5807  	}{
  5808  		/* LoadBalancerIPMode disabled */
  5809  		{
  5810  			name:             "LoadBalancerIPMode disabled, ipMode Proxy",
  5811  			ipModeEnabled:    false,
  5812  			svcIP:            "10.20.30.41",
  5813  			svcLBIP:          "1.2.3.4",
  5814  			ipMode:           ptr.To(v1.LoadBalancerIPModeProxy),
  5815  			expectedServices: 2,
  5816  		},
  5817  		{
  5818  			name:             "LoadBalancerIPMode disabled, ipMode VIP",
  5819  			ipModeEnabled:    false,
  5820  			svcIP:            "10.20.30.42",
  5821  			svcLBIP:          "1.2.3.5",
  5822  			ipMode:           ptr.To(v1.LoadBalancerIPModeVIP),
  5823  			expectedServices: 2,
  5824  		},
  5825  		{
  5826  			name:             "LoadBalancerIPMode disabled, ipMode nil",
  5827  			ipModeEnabled:    false,
  5828  			svcIP:            "10.20.30.43",
  5829  			svcLBIP:          "1.2.3.6",
  5830  			ipMode:           nil,
  5831  			expectedServices: 2,
  5832  		},
  5833  		/* LoadBalancerIPMode enabled */
  5834  		{
  5835  			name:             "LoadBalancerIPMode enabled, ipMode Proxy",
  5836  			ipModeEnabled:    true,
  5837  			svcIP:            "10.20.30.41",
  5838  			svcLBIP:          "1.2.3.4",
  5839  			ipMode:           ptr.To(v1.LoadBalancerIPModeProxy),
  5840  			expectedServices: 1,
  5841  		},
  5842  		{
  5843  			name:             "LoadBalancerIPMode enabled, ipMode VIP",
  5844  			ipModeEnabled:    true,
  5845  			svcIP:            "10.20.30.42",
  5846  			svcLBIP:          "1.2.3.5",
  5847  			ipMode:           ptr.To(v1.LoadBalancerIPModeVIP),
  5848  			expectedServices: 2,
  5849  		},
  5850  		{
  5851  			name:             "LoadBalancerIPMode enabled, ipMode nil",
  5852  			ipModeEnabled:    true,
  5853  			svcIP:            "10.20.30.43",
  5854  			svcLBIP:          "1.2.3.6",
  5855  			ipMode:           nil,
  5856  			expectedServices: 2,
  5857  		},
  5858  	}
  5859  
  5860  	svcPort := 80
  5861  	svcNodePort := 3001
  5862  	svcPortName := proxy.ServicePortName{
  5863  		NamespacedName: makeNSN("ns1", "svc1"),
  5864  		Port:           "p80",
  5865  	}
  5866  
  5867  	for _, testCase := range testCases {
  5868  		t.Run(testCase.name, func(t *testing.T) {
  5869  			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LoadBalancerIPMode, testCase.ipModeEnabled)()
  5870  			_, fp := buildFakeProxier()
  5871  			makeServiceMap(fp,
  5872  				makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) {
  5873  					svc.Spec.Type = "LoadBalancer"
  5874  					svc.Spec.ClusterIP = testCase.svcIP
  5875  					svc.Spec.Ports = []v1.ServicePort{{
  5876  						Name:     svcPortName.Port,
  5877  						Port:     int32(svcPort),
  5878  						Protocol: v1.ProtocolTCP,
  5879  						NodePort: int32(svcNodePort),
  5880  					}}
  5881  					svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
  5882  						IP:     testCase.svcLBIP,
  5883  						IPMode: testCase.ipMode,
  5884  					}}
  5885  				}),
  5886  			)
  5887  
  5888  			makeEndpointSliceMap(fp,
  5889  				makeTestEndpointSlice("ns1", "svc1", 1, func(eps *discovery.EndpointSlice) {
  5890  					eps.AddressType = discovery.AddressTypeIPv4
  5891  					eps.Endpoints = []discovery.Endpoint{{
  5892  						Addresses: []string{"10.180.0.1"},
  5893  					}}
  5894  					eps.Ports = []discovery.EndpointPort{{
  5895  						Name:     ptr.To("p80"),
  5896  						Port:     ptr.To[int32](80),
  5897  						Protocol: ptr.To(v1.ProtocolTCP),
  5898  					}}
  5899  				}),
  5900  			)
  5901  
  5902  			fp.syncProxyRules()
  5903  
  5904  			services, err := fp.ipvs.GetVirtualServers()
  5905  			if err != nil {
  5906  				t.Errorf("Failed to get ipvs services, err: %v", err)
  5907  			}
  5908  			if len(services) != testCase.expectedServices {
  5909  				t.Errorf("Expected %d ipvs services, got %d", testCase.expectedServices, len(services))
  5910  			}
  5911  		})
  5912  	}
  5913  }