github.com/cilium/cilium@v1.16.2/pkg/k8s/watchers/service_test.go (about)

     1  // SPDX-License-Identifier: Apache-2.0
     2  // Copyright Authors of Cilium
     3  
     4  package watchers
     5  
     6  import (
     7  	"sort"
     8  	"testing"
     9  
    10  	"github.com/cilium/statedb"
    11  	"github.com/stretchr/testify/require"
    12  
    13  	cmtypes "github.com/cilium/cilium/pkg/clustermesh/types"
    14  	fakeTypes "github.com/cilium/cilium/pkg/datapath/fake/types"
    15  	datapathTables "github.com/cilium/cilium/pkg/datapath/tables"
    16  	"github.com/cilium/cilium/pkg/k8s"
    17  	slim_corev1 "github.com/cilium/cilium/pkg/k8s/slim/k8s/api/core/v1"
    18  	slim_metav1 "github.com/cilium/cilium/pkg/k8s/slim/k8s/apis/meta/v1"
    19  	"github.com/cilium/cilium/pkg/loadbalancer"
    20  	"github.com/cilium/cilium/pkg/lock"
    21  	"github.com/cilium/cilium/pkg/option"
    22  )
    23  
    24  type fakeSvcManager struct {
    25  	OnDeleteService func(frontend loadbalancer.L3n4Addr) (bool, error)
    26  	OnUpsertService func(*loadbalancer.SVC) (bool, loadbalancer.ID, error)
    27  }
    28  
    29  func (f *fakeSvcManager) DeleteService(frontend loadbalancer.L3n4Addr) (bool, error) {
    30  	if f.OnDeleteService != nil {
    31  		return f.OnDeleteService(frontend)
    32  	}
    33  	panic("OnDeleteService(loadbalancer.L3n4Addr) (bool, error) was called and is not set!")
    34  }
    35  
    36  func (f *fakeSvcManager) GetDeepCopyServiceByFrontend(frontend loadbalancer.L3n4Addr) (*loadbalancer.SVC, bool) {
    37  	return nil, false
    38  }
    39  
    40  func (f *fakeSvcManager) UpsertService(p *loadbalancer.SVC) (bool, loadbalancer.ID, error) {
    41  	if f.OnUpsertService != nil {
    42  		return f.OnUpsertService(p)
    43  	}
    44  	panic("OnUpsertService() was called and is not set!")
    45  }
    46  
    47  func newDB(t *testing.T) (*statedb.DB, statedb.Table[datapathTables.NodeAddress]) {
    48  	db := statedb.New()
    49  	nodeAddrs, err := datapathTables.NewNodeAddressTable()
    50  	require.NoError(t, err)
    51  	err = db.RegisterTable(nodeAddrs)
    52  	require.NoError(t, err)
    53  
    54  	txn := db.WriteTxn(nodeAddrs)
    55  	for _, addr := range datapathTables.TestAddresses {
    56  		nodeAddrs.Insert(txn, addr)
    57  	}
    58  	txn.Commit()
    59  
    60  	return db, nodeAddrs
    61  }
    62  
    63  func Test_addK8sSVCs_ClusterIP(t *testing.T) {
    64  	k8sSvc := &slim_corev1.Service{
    65  		ObjectMeta: slim_metav1.ObjectMeta{
    66  			Name:      "foo",
    67  			Namespace: "bar",
    68  			Labels: map[string]string{
    69  				"foo": "bar",
    70  			},
    71  		},
    72  		Spec: slim_corev1.ServiceSpec{
    73  			Ports: []slim_corev1.ServicePort{
    74  				{
    75  					Name:     "port-udp-80",
    76  					Protocol: slim_corev1.ProtocolUDP,
    77  					Port:     80,
    78  				},
    79  				// FIXME: We don't distinguish about the protocol being used
    80  				//        so we can't tell if a UDP/80 maps to port 8080/udp
    81  				//        or if TCP/80 maps to port 8081/TCP
    82  				// {
    83  				// 	Name:       "port-tcp-80",
    84  				// 	Protocol:  slim_corev1.ProtocolTCP,
    85  				// 	Port:       80,
    86  				// 	TargetPort: intstr.FromString("port-80-t"),
    87  				// },
    88  				{
    89  					Name:     "port-tcp-81",
    90  					Protocol: slim_corev1.ProtocolTCP,
    91  					Port:     81,
    92  				},
    93  			},
    94  			Selector:              nil,
    95  			ClusterIP:             "172.0.20.1",
    96  			Type:                  slim_corev1.ServiceTypeClusterIP,
    97  			ExternalIPs:           nil,
    98  			SessionAffinity:       "",
    99  			ExternalTrafficPolicy: "",
   100  			HealthCheckNodePort:   0,
   101  			SessionAffinityConfig: nil,
   102  		},
   103  	}
   104  
   105  	ep1stApply := &slim_corev1.Endpoints{
   106  		ObjectMeta: slim_metav1.ObjectMeta{
   107  			Name:      "foo",
   108  			Namespace: "bar",
   109  		},
   110  		Subsets: []slim_corev1.EndpointSubset{
   111  			{
   112  				Addresses: []slim_corev1.EndpointAddress{{IP: "10.0.0.2"}},
   113  				Ports: []slim_corev1.EndpointPort{
   114  					{
   115  						Name:     "port-udp-80",
   116  						Port:     8080,
   117  						Protocol: slim_corev1.ProtocolUDP,
   118  					},
   119  					// FIXME: We don't distinguish about the protocol being used
   120  					//        so we can't tell if a UDP/80 maps to port 8080/udp
   121  					//        or if TCP/80 maps to port 8081/TCP
   122  					// {
   123  					// 	Name:     "port-tcp-80",
   124  					// 	Protocol:slim_corev1.ProtocolTCP,
   125  					// 	Port:     8081,
   126  					// },
   127  					{
   128  						Name:     "port-tcp-81",
   129  						Protocol: slim_corev1.ProtocolTCP,
   130  						Port:     81,
   131  					},
   132  				},
   133  			},
   134  		},
   135  	}
   136  
   137  	lb1 := loadbalancer.NewL3n4AddrID(loadbalancer.UDP, cmtypes.MustParseAddrCluster("172.0.20.1"), 80, loadbalancer.ScopeExternal, 0)
   138  	// lb2 := loadbalancer.NewL3n4AddrID(loadbalancer.TCP, net.ParseIP("172.0.20.1"), 80, loadbalancer.ScopeExternal, 0)
   139  	lb3 := loadbalancer.NewL3n4AddrID(loadbalancer.TCP, cmtypes.MustParseAddrCluster("172.0.20.1"), 81, loadbalancer.ScopeExternal, 0)
   140  	upsert1stWanted := map[string]loadbalancer.SVC{
   141  		lb1.Hash(): {
   142  			Type:     loadbalancer.SVCTypeClusterIP,
   143  			Frontend: *lb1,
   144  			Backends: []*loadbalancer.Backend{
   145  				{
   146  					FEPortName: "port-udp-80",
   147  					L3n4Addr: loadbalancer.L3n4Addr{
   148  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
   149  						L4Addr: loadbalancer.L4Addr{
   150  							Protocol: loadbalancer.UDP,
   151  							Port:     8080,
   152  						},
   153  					},
   154  					Weight: loadbalancer.DefaultBackendWeight,
   155  				},
   156  			},
   157  		},
   158  		// FIXME: We don't distinguish about the protocol being used
   159  		//        so we can't tell if a UDP/80 maps to port 8080/udp
   160  		//        or if TCP/80 maps to port 8081/TCP
   161  		// lb2.Hash(): {
   162  		// 	Type:     loadbalancer.SVCTypeClusterIP,
   163  		// 	Frontend: *lb2,
   164  		// 	Backends: []*loadbalancer.Backend{
   165  		// 		{
   166  		// 			L3n4Addr: loadbalancer.L3n4Addr{
   167  		// 				IP: net.ParseIP("10.0.0.2"),
   168  		// 				L4Addr: loadbalancer.L4Addr{
   169  		// 					Protocol: loadbalancer.TCP,
   170  		// 					Port:     8081,
   171  		// 				},
   172  		// 			},
   173  		// 		},
   174  		// 	},
   175  		// },
   176  		lb3.Hash(): {
   177  			Type:     loadbalancer.SVCTypeClusterIP,
   178  			Frontend: *lb3,
   179  			Backends: []*loadbalancer.Backend{
   180  				{
   181  					FEPortName: "port-tcp-81",
   182  					L3n4Addr: loadbalancer.L3n4Addr{
   183  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
   184  						L4Addr: loadbalancer.L4Addr{
   185  							Protocol: loadbalancer.TCP,
   186  							Port:     81,
   187  						},
   188  					},
   189  					Weight: loadbalancer.DefaultBackendWeight,
   190  				},
   191  			},
   192  		},
   193  	}
   194  
   195  	ep2ndApply := ep1stApply.DeepCopy()
   196  	ep2ndApply.Subsets[0].Addresses = append(
   197  		ep2ndApply.Subsets[0].Addresses,
   198  		slim_corev1.EndpointAddress{IP: "10.0.0.3"},
   199  	)
   200  
   201  	upsert2ndWanted := map[string]loadbalancer.SVC{
   202  		lb1.Hash(): {
   203  			Type:     loadbalancer.SVCTypeClusterIP,
   204  			Frontend: *lb1,
   205  			Backends: []*loadbalancer.Backend{
   206  				{
   207  					FEPortName: "port-udp-80",
   208  					L3n4Addr: loadbalancer.L3n4Addr{
   209  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
   210  						L4Addr: loadbalancer.L4Addr{
   211  							Protocol: loadbalancer.UDP,
   212  							Port:     8080,
   213  						},
   214  					},
   215  					Weight: loadbalancer.DefaultBackendWeight,
   216  				},
   217  				{
   218  					FEPortName: "port-udp-80",
   219  					L3n4Addr: loadbalancer.L3n4Addr{
   220  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.3"),
   221  						L4Addr: loadbalancer.L4Addr{
   222  							Protocol: loadbalancer.UDP,
   223  							Port:     8080,
   224  						},
   225  					},
   226  					Weight: loadbalancer.DefaultBackendWeight,
   227  				},
   228  			},
   229  		},
   230  		// FIXME: We don't distinguish about the protocol being used
   231  		//        so we can't tell if a UDP/80 maps to port 8080/udp
   232  		//        or if TCP/80 maps to port 8081/TCP
   233  		// lb2.Hash(): {
   234  		// 	Type:     loadbalancer.SVCTypeClusterIP,
   235  		// 	Frontend: *lb2,
   236  		// 	Backends: []*loadbalancer.Backend{
   237  		// 		{
   238  		// 			L3n4Addr: loadbalancer.L3n4Addr{
   239  		// 				IP: net.ParseIP("10.0.0.2"),
   240  		// 				L4Addr: loadbalancer.L4Addr{
   241  		// 					Protocol: loadbalancer.TCP,
   242  		// 					Port:     8081,
   243  		// 				},
   244  		// 			},
   245  		// 		},
   246  		// 		{
   247  		// 			L3n4Addr: loadbalancer.L3n4Addr{
   248  		// 				IP: net.ParseIP("10.0.0.3"),
   249  		// 				L4Addr: loadbalancer.L4Addr{
   250  		// 					Protocol: loadbalancer.TCP,
   251  		// 					Port:     8081,
   252  		// 				},
   253  		// 			},
   254  		// 		},
   255  		// 	},
   256  		// },
   257  		lb3.Hash(): {
   258  			Type:     loadbalancer.SVCTypeClusterIP,
   259  			Frontend: *lb3,
   260  			Backends: []*loadbalancer.Backend{
   261  				{
   262  					FEPortName: "port-tcp-81",
   263  					L3n4Addr: loadbalancer.L3n4Addr{
   264  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
   265  						L4Addr: loadbalancer.L4Addr{
   266  							Protocol: loadbalancer.TCP,
   267  							Port:     81,
   268  						},
   269  					},
   270  					Weight: loadbalancer.DefaultBackendWeight,
   271  				},
   272  				{
   273  					FEPortName: "port-tcp-81",
   274  					L3n4Addr: loadbalancer.L3n4Addr{
   275  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.3"),
   276  						L4Addr: loadbalancer.L4Addr{
   277  							Protocol: loadbalancer.TCP,
   278  							Port:     81,
   279  						},
   280  					},
   281  					Weight: loadbalancer.DefaultBackendWeight,
   282  				},
   283  			},
   284  		},
   285  	}
   286  
   287  	del1stWanted := map[string]struct{}{
   288  		lb1.Hash(): {},
   289  		// lb2.Hash(): {},
   290  		lb3.Hash(): {},
   291  	}
   292  
   293  	upsert1st := map[string]loadbalancer.SVC{}
   294  	upsert2nd := map[string]loadbalancer.SVC{}
   295  	del1st := map[string]struct{}{}
   296  
   297  	svcUpsertManagerCalls, svcDeleteManagerCalls := 0, 0
   298  
   299  	svcManager := &fakeSvcManager{
   300  		OnUpsertService: func(p *loadbalancer.SVC) (bool, loadbalancer.ID, error) {
   301  			sort.Slice(p.Backends, func(i, j int) bool {
   302  				return p.Backends[i].AddrCluster.Less(p.Backends[j].AddrCluster)
   303  			})
   304  			switch {
   305  			// 1st update endpoints
   306  			case svcUpsertManagerCalls < len(upsert1stWanted):
   307  				upsert1st[p.Frontend.Hash()] = loadbalancer.SVC{
   308  					Frontend: p.Frontend,
   309  					Backends: p.Backends,
   310  					Type:     p.Type,
   311  				}
   312  			// 2nd update endpoints
   313  			case svcUpsertManagerCalls < len(upsert1stWanted)+len(upsert2ndWanted):
   314  				upsert2nd[p.Frontend.Hash()] = loadbalancer.SVC{
   315  					Frontend: p.Frontend,
   316  					Backends: p.Backends,
   317  					Type:     p.Type,
   318  				}
   319  			}
   320  			svcUpsertManagerCalls++
   321  			return false, 0, nil
   322  		},
   323  		OnDeleteService: func(fe loadbalancer.L3n4Addr) (b bool, e error) {
   324  			del1st[fe.Hash()] = struct{}{}
   325  			svcDeleteManagerCalls++
   326  			return true, nil
   327  		},
   328  	}
   329  
   330  	db, nodeAddrs := newDB(t)
   331  	k8sSvcCache := k8s.NewServiceCache(db, nodeAddrs)
   332  	svcWatcher := &K8sServiceWatcher{
   333  		k8sSvcCache: k8sSvcCache,
   334  		svcManager:  svcManager,
   335  	}
   336  
   337  	go svcWatcher.k8sServiceHandler()
   338  	swg := lock.NewStoppableWaitGroup()
   339  
   340  	k8sSvcCache.UpdateService(k8sSvc, swg)
   341  	k8sSvcCache.UpdateEndpoints(k8s.ParseEndpoints(ep1stApply), swg)
   342  	// Running a 2nd update should also trigger a new upsert service
   343  	k8sSvcCache.UpdateEndpoints(k8s.ParseEndpoints(ep2ndApply), swg)
   344  	// Running a 3rd update should also not trigger anything because the
   345  	// endpoints are the same
   346  	k8sSvcCache.UpdateEndpoints(k8s.ParseEndpoints(ep2ndApply), swg)
   347  
   348  	k8sSvcCache.DeleteService(k8sSvc, swg)
   349  
   350  	swg.Stop()
   351  	swg.Wait()
   352  	require.Equal(t, len(upsert1stWanted)+len(upsert2ndWanted), svcUpsertManagerCalls)
   353  	require.Equal(t, len(del1stWanted), svcDeleteManagerCalls)
   354  
   355  	require.EqualValues(t, upsert1stWanted, upsert1st)
   356  	require.EqualValues(t, upsert2ndWanted, upsert2nd)
   357  	require.EqualValues(t, del1stWanted, del1st)
   358  }
   359  
   360  func TestChangeSVCPort(t *testing.T) {
   361  	k8sSvc := &slim_corev1.Service{
   362  		ObjectMeta: slim_metav1.ObjectMeta{
   363  			Name:      "foo",
   364  			Namespace: "bar",
   365  			Labels: map[string]string{
   366  				"foo": "bar",
   367  			},
   368  		},
   369  		Spec: slim_corev1.ServiceSpec{
   370  			Ports: []slim_corev1.ServicePort{
   371  				{
   372  					Name:     "port-udp-80",
   373  					Protocol: slim_corev1.ProtocolUDP,
   374  					Port:     80,
   375  				},
   376  			},
   377  			ClusterIP: "172.0.20.1",
   378  			Type:      slim_corev1.ServiceTypeClusterIP,
   379  		},
   380  	}
   381  
   382  	ep1stApply := &slim_corev1.Endpoints{
   383  		ObjectMeta: slim_metav1.ObjectMeta{
   384  			Name:      "foo",
   385  			Namespace: "bar",
   386  		},
   387  		Subsets: []slim_corev1.EndpointSubset{
   388  			{
   389  				Addresses: []slim_corev1.EndpointAddress{{IP: "10.0.0.2"}},
   390  				Ports: []slim_corev1.EndpointPort{
   391  					{
   392  						Name:     "port-udp-80",
   393  						Port:     8080,
   394  						Protocol: slim_corev1.ProtocolUDP,
   395  					},
   396  				},
   397  			},
   398  		},
   399  	}
   400  
   401  	lb1 := loadbalancer.NewL3n4AddrID(loadbalancer.UDP, cmtypes.MustParseAddrCluster("172.0.20.1"), 80, loadbalancer.ScopeExternal, 0)
   402  	lb2 := loadbalancer.NewL3n4AddrID(loadbalancer.UDP, cmtypes.MustParseAddrCluster("172.0.20.1"), 81, loadbalancer.ScopeExternal, 0)
   403  	upsertsWanted := []loadbalancer.SVC{
   404  		{
   405  			Type:     loadbalancer.SVCTypeClusterIP,
   406  			Frontend: *lb1,
   407  			Backends: []*loadbalancer.Backend{
   408  				{
   409  					FEPortName: "port-udp-80",
   410  					L3n4Addr: loadbalancer.L3n4Addr{
   411  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
   412  						L4Addr: loadbalancer.L4Addr{
   413  							Protocol: loadbalancer.UDP,
   414  							Port:     8080,
   415  						},
   416  					},
   417  					Weight: loadbalancer.DefaultBackendWeight,
   418  				},
   419  			},
   420  		},
   421  		{
   422  			Type:     loadbalancer.SVCTypeClusterIP,
   423  			Frontend: *lb2,
   424  			Backends: []*loadbalancer.Backend{
   425  				{
   426  					FEPortName: "port-udp-80",
   427  					L3n4Addr: loadbalancer.L3n4Addr{
   428  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
   429  						L4Addr: loadbalancer.L4Addr{
   430  							Protocol: loadbalancer.UDP,
   431  							Port:     8080,
   432  						},
   433  					},
   434  					Weight: loadbalancer.DefaultBackendWeight,
   435  				},
   436  			},
   437  		},
   438  	}
   439  
   440  	k8sSvcChanged := k8sSvc.DeepCopy()
   441  	k8sSvcChanged.Spec.Ports[0].Port = 81
   442  
   443  	upserts := []loadbalancer.SVC{}
   444  
   445  	svcUpsertManagerCalls := 0
   446  
   447  	svcManager := &fakeSvcManager{
   448  		OnUpsertService: func(p *loadbalancer.SVC) (bool, loadbalancer.ID, error) {
   449  			upserts = append(upserts, loadbalancer.SVC{
   450  				Frontend: p.Frontend,
   451  				Backends: p.Backends,
   452  				Type:     p.Type,
   453  			})
   454  			svcUpsertManagerCalls++
   455  			return false, 0, nil
   456  		},
   457  		OnDeleteService: func(fe loadbalancer.L3n4Addr) (b bool, e error) {
   458  			return false, nil
   459  		},
   460  	}
   461  
   462  	db, nodeAddrs := newDB(t)
   463  	k8sSvcCache := k8s.NewServiceCache(db, nodeAddrs)
   464  	svcWatcher := &K8sServiceWatcher{
   465  		k8sSvcCache: k8sSvcCache,
   466  		svcManager:  svcManager,
   467  	}
   468  
   469  	go svcWatcher.k8sServiceHandler()
   470  	swg := lock.NewStoppableWaitGroup()
   471  
   472  	k8sSvcCache.UpdateService(k8sSvc, swg)
   473  	k8sSvcCache.UpdateEndpoints(k8s.ParseEndpoints(ep1stApply), swg)
   474  	k8sSvcCache.UpdateService(k8sSvcChanged, swg)
   475  
   476  	swg.Stop()
   477  	swg.Wait()
   478  	require.Equal(t, 2, svcUpsertManagerCalls) // Add and Update events
   479  	require.EqualValues(t, upsertsWanted, upserts)
   480  }
   481  
   482  func Test_addK8sSVCs_NodePort(t *testing.T) {
   483  	enableNodePortBak := option.Config.EnableNodePort
   484  	option.Config.EnableNodePort = true
   485  	defer func() {
   486  		option.Config.EnableNodePort = enableNodePortBak
   487  	}()
   488  
   489  	k8sSvc := &slim_corev1.Service{
   490  		ObjectMeta: slim_metav1.ObjectMeta{
   491  			Name:      "foo",
   492  			Namespace: "bar",
   493  			Labels: map[string]string{
   494  				"foo": "bar",
   495  			},
   496  		},
   497  		Spec: slim_corev1.ServiceSpec{
   498  			Ports: []slim_corev1.ServicePort{
   499  				{
   500  					Name:     "port-udp-80",
   501  					Protocol: slim_corev1.ProtocolUDP,
   502  					Port:     80,
   503  					NodePort: 18080,
   504  				},
   505  				// FIXME: We don't distinguish about the protocol being used
   506  				//        so we can't tell if a UDP/80 maps to port 8080/udp
   507  				//        or if TCP/80 maps to port 8081/TCP
   508  				// {
   509  				// 	Name:       "port-tcp-80",
   510  				// 	Protocol:  slim_corev1.ProtocolTCP,
   511  				// 	Port:       80,
   512  				// 	TargetPort: intstr.FromString("port-80-t"),
   513  				//  NodePort:   18080,
   514  				// },
   515  				{
   516  					Name:     "port-tcp-81",
   517  					Protocol: slim_corev1.ProtocolTCP,
   518  					Port:     81,
   519  					NodePort: 18081,
   520  				},
   521  			},
   522  			Selector:              nil,
   523  			ClusterIP:             "172.0.20.1",
   524  			Type:                  slim_corev1.ServiceTypeNodePort,
   525  			ExternalIPs:           nil,
   526  			SessionAffinity:       "",
   527  			ExternalTrafficPolicy: "",
   528  			HealthCheckNodePort:   0,
   529  			SessionAffinityConfig: nil,
   530  		},
   531  	}
   532  
   533  	ep1stApply := &slim_corev1.Endpoints{
   534  		ObjectMeta: slim_metav1.ObjectMeta{
   535  			Name:      "foo",
   536  			Namespace: "bar",
   537  		},
   538  		Subsets: []slim_corev1.EndpointSubset{
   539  			{
   540  				Addresses: []slim_corev1.EndpointAddress{{IP: "10.0.0.2"}},
   541  				Ports: []slim_corev1.EndpointPort{
   542  					{
   543  						Name:     "port-udp-80",
   544  						Port:     8080,
   545  						Protocol: slim_corev1.ProtocolUDP,
   546  					},
   547  					// FIXME: We don't distinguish about the protocol being used
   548  					//        so we can't tell if a UDP/80 maps to port 8080/udp
   549  					//        or if TCP/80 maps to port 8081/TCP
   550  					// {
   551  					// 	Name:     "port-tcp-80",
   552  					// 	Protocol:slim_corev1.ProtocolTCP,
   553  					// 	Port:     8081,
   554  					// },
   555  					{
   556  						Name:     "port-tcp-81",
   557  						Protocol: slim_corev1.ProtocolTCP,
   558  						Port:     8081,
   559  					},
   560  				},
   561  			},
   562  		},
   563  	}
   564  
   565  	clusterIP1 := loadbalancer.NewL3n4AddrID(loadbalancer.UDP, cmtypes.MustParseAddrCluster("172.0.20.1"), 80, loadbalancer.ScopeExternal, 0)
   566  	// clusterIP2 := loadbalancer.NewL3n4AddrID(loadbalancer.TCP, net.ParseIP("172.0.20.1"), 80, loadbalancer.ScopeExternal, 0)
   567  	clusterIP3 := loadbalancer.NewL3n4AddrID(loadbalancer.TCP, cmtypes.MustParseAddrCluster("172.0.20.1"), 81, loadbalancer.ScopeExternal, 0)
   568  
   569  	upsert1stWanted := map[string]loadbalancer.SVC{
   570  		clusterIP1.Hash(): {
   571  			Type:     loadbalancer.SVCTypeClusterIP,
   572  			Frontend: *clusterIP1,
   573  			Backends: []*loadbalancer.Backend{
   574  				{
   575  					FEPortName: "port-udp-80",
   576  					L3n4Addr: loadbalancer.L3n4Addr{
   577  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
   578  						L4Addr: loadbalancer.L4Addr{
   579  							Protocol: loadbalancer.UDP,
   580  							Port:     8080,
   581  						},
   582  					},
   583  					Weight: loadbalancer.DefaultBackendWeight,
   584  				},
   585  			},
   586  		},
   587  		// FIXME: We don't distinguish about the protocol being used
   588  		//        so we can't tell if a UDP/80 maps to port 8080/udp
   589  		//        or if TCP/80 maps to port 8081/TCP
   590  		// clusterIP2.Hash(): {
   591  		// 	Type:     loadbalancer.SVCTypeClusterIP,
   592  		// 	Frontend: *clusterIP2,
   593  		// 	Backends: []*loadbalancer.Backend{
   594  		// 		{
   595  		// 			L3n4Addr: loadbalancer.L3n4Addr{
   596  		// 				IP: net.ParseIP("10.0.0.2"),
   597  		// 				L4Addr: loadbalancer.L4Addr{
   598  		// 					Protocol: loadbalancer.TCP,
   599  		// 					Port:     8081,
   600  		// 				},
   601  		// 			},
   602  		// 		},
   603  		// 	},
   604  		// },
   605  		clusterIP3.Hash(): {
   606  			Type:     loadbalancer.SVCTypeClusterIP,
   607  			Frontend: *clusterIP3,
   608  			Backends: []*loadbalancer.Backend{
   609  				{
   610  					FEPortName: "port-tcp-81",
   611  					L3n4Addr: loadbalancer.L3n4Addr{
   612  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
   613  						L4Addr: loadbalancer.L4Addr{
   614  							Protocol: loadbalancer.TCP,
   615  							Port:     8081,
   616  						},
   617  					},
   618  					Weight: loadbalancer.DefaultBackendWeight,
   619  				},
   620  			},
   621  		},
   622  	}
   623  
   624  	ipv4NodePortAddrCluster := cmtypes.MustAddrClusterFromIP(fakeTypes.IPv4NodePortAddress)
   625  	ipv4InternalAddrCluster := cmtypes.MustAddrClusterFromIP(fakeTypes.IPv4InternalAddress)
   626  
   627  	nodePortIPs1 := []*loadbalancer.L3n4AddrID{
   628  		loadbalancer.NewL3n4AddrID(loadbalancer.UDP, cmtypes.MustParseAddrCluster("0.0.0.0"), 18080, loadbalancer.ScopeExternal, 0),
   629  		loadbalancer.NewL3n4AddrID(loadbalancer.UDP, ipv4NodePortAddrCluster, 18080, loadbalancer.ScopeExternal, 0),
   630  		loadbalancer.NewL3n4AddrID(loadbalancer.UDP, ipv4InternalAddrCluster, 18080, loadbalancer.ScopeExternal, 0),
   631  	}
   632  	for _, nodePort := range nodePortIPs1 {
   633  		upsert1stWanted[nodePort.Hash()] = loadbalancer.SVC{
   634  			Type:     loadbalancer.SVCTypeNodePort,
   635  			Frontend: *nodePort,
   636  			Backends: []*loadbalancer.Backend{
   637  				{
   638  					FEPortName: "port-udp-80",
   639  					L3n4Addr: loadbalancer.L3n4Addr{
   640  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
   641  						L4Addr: loadbalancer.L4Addr{
   642  							Protocol: loadbalancer.UDP,
   643  							Port:     8080,
   644  						},
   645  					},
   646  					Weight: loadbalancer.DefaultBackendWeight,
   647  				},
   648  			},
   649  		}
   650  	}
   651  	// nodePortIPs2 := []*loadbalancer.L3n4AddrID{
   652  	// 	loadbalancer.NewL3n4AddrID(loadbalancer.TCP, net.ParseIP("0.0.0.0"), 18080, loadbalancer.ScopeExternal, 0),
   653  	// 	loadbalancer.NewL3n4AddrID(loadbalancer.TCP, fakeDatapath.Pv4NodePortAddress, 18080, loadbalancer.ScopeExternal, 0),
   654  	// 	loadbalancer.NewL3n4AddrID(loadbalancer.TCP, fakeDatapath.IPv4InternalAddress, 18080, loadbalancer.ScopeExternal, 0),
   655  	// }
   656  	// for _, nodePort := range nodePortIPs2 {
   657  	// 	upsert1stWanted[nodePort.Hash()] = loadbalancer.SVC{
   658  	// 		Type:     loadbalancer.SVCTypeNodePort,
   659  	// 		Frontend: *nodePort,
   660  	// 		Backends: []*loadbalancer.Backend{
   661  	// 			{
   662  	// 				L3n4Addr: loadbalancer.L3n4Addr{
   663  	// 					IP: net.ParseIP("10.0.0.2"),
   664  	// 					L4Addr: loadbalancer.L4Addr{
   665  	// 						Protocol: loadbalancer.TCP,
   666  	// 						Port:     8081,
   667  	// 					},
   668  	// 				},
   669  	// 			},
   670  	// 		},
   671  	// 	}
   672  	// }
   673  	nodePortIPs3 := []*loadbalancer.L3n4AddrID{
   674  		loadbalancer.NewL3n4AddrID(loadbalancer.TCP, cmtypes.MustParseAddrCluster("0.0.0.0"), 18081, loadbalancer.ScopeExternal, 0),
   675  		loadbalancer.NewL3n4AddrID(loadbalancer.TCP, ipv4NodePortAddrCluster, 18081, loadbalancer.ScopeExternal, 0),
   676  		loadbalancer.NewL3n4AddrID(loadbalancer.TCP, ipv4InternalAddrCluster, 18081, loadbalancer.ScopeExternal, 0),
   677  	}
   678  	for _, nodePort := range nodePortIPs3 {
   679  		upsert1stWanted[nodePort.Hash()] = loadbalancer.SVC{
   680  			Type:     loadbalancer.SVCTypeNodePort,
   681  			Frontend: *nodePort,
   682  			Backends: []*loadbalancer.Backend{
   683  				{
   684  					FEPortName: "port-tcp-81",
   685  					L3n4Addr: loadbalancer.L3n4Addr{
   686  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
   687  						L4Addr: loadbalancer.L4Addr{
   688  							Protocol: loadbalancer.TCP,
   689  							Port:     8081,
   690  						},
   691  					},
   692  					Weight: loadbalancer.DefaultBackendWeight,
   693  				},
   694  			},
   695  		}
   696  	}
   697  
   698  	ep2ndApply := ep1stApply.DeepCopy()
   699  	ep2ndApply.Subsets[0].Addresses = append(
   700  		ep2ndApply.Subsets[0].Addresses,
   701  		slim_corev1.EndpointAddress{IP: "10.0.0.3"},
   702  	)
   703  
   704  	upsert2ndWanted := map[string]loadbalancer.SVC{
   705  		clusterIP1.Hash(): {
   706  			Type:     loadbalancer.SVCTypeClusterIP,
   707  			Frontend: *clusterIP1,
   708  			Backends: []*loadbalancer.Backend{
   709  				{
   710  					FEPortName: "port-udp-80",
   711  					L3n4Addr: loadbalancer.L3n4Addr{
   712  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
   713  						L4Addr: loadbalancer.L4Addr{
   714  							Protocol: loadbalancer.UDP,
   715  							Port:     8080,
   716  						},
   717  					},
   718  					Weight: loadbalancer.DefaultBackendWeight,
   719  				},
   720  				{
   721  					FEPortName: "port-udp-80",
   722  					L3n4Addr: loadbalancer.L3n4Addr{
   723  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.3"),
   724  						L4Addr: loadbalancer.L4Addr{
   725  							Protocol: loadbalancer.UDP,
   726  							Port:     8080,
   727  						},
   728  					},
   729  					Weight: loadbalancer.DefaultBackendWeight,
   730  				},
   731  			},
   732  		},
   733  		// FIXME: We don't distinguish about the protocol being used
   734  		//        so we can't tell if a UDP/80 maps to port 8080/udp
   735  		//        or if TCP/80 maps to port 8081/TCP
   736  		// clusterIP2.Hash(): {
   737  		// 	Type:     loadbalancer.SVCTypeClusterIP,
   738  		// 	Frontend: *clusterIP2,
   739  		// 	Backends: []*loadbalancer.Backend{
   740  		// 		{
   741  		// 			L3n4Addr: loadbalancer.L3n4Addr{
   742  		// 				IP: net.ParseIP("10.0.0.2"),
   743  		// 				L4Addr: loadbalancer.L4Addr{
   744  		// 					Protocol: loadbalancer.TCP,
   745  		// 					Port:     8081,
   746  		// 				},
   747  		// 			},
   748  		// 		},
   749  		// 		{
   750  		// 			L3n4Addr: loadbalancer.L3n4Addr{
   751  		// 				IP: net.ParseIP("10.0.0.3"),
   752  		// 				L4Addr: loadbalancer.L4Addr{
   753  		// 					Protocol: loadbalancer.TCP,
   754  		// 					Port:     8081,
   755  		// 				},
   756  		// 			},
   757  		// 		},
   758  		// 	},
   759  		// },
   760  		clusterIP3.Hash(): {
   761  			Type:     loadbalancer.SVCTypeClusterIP,
   762  			Frontend: *clusterIP3,
   763  			Backends: []*loadbalancer.Backend{
   764  				{
   765  					FEPortName: "port-tcp-81",
   766  					L3n4Addr: loadbalancer.L3n4Addr{
   767  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
   768  						L4Addr: loadbalancer.L4Addr{
   769  							Protocol: loadbalancer.TCP,
   770  							Port:     8081,
   771  						},
   772  					},
   773  					Weight: loadbalancer.DefaultBackendWeight,
   774  				},
   775  				{
   776  					FEPortName: "port-tcp-81",
   777  					L3n4Addr: loadbalancer.L3n4Addr{
   778  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.3"),
   779  						L4Addr: loadbalancer.L4Addr{
   780  							Protocol: loadbalancer.TCP,
   781  							Port:     8081,
   782  						},
   783  					},
   784  					Weight: loadbalancer.DefaultBackendWeight,
   785  				},
   786  			},
   787  		},
   788  	}
   789  
   790  	for _, nodePort := range nodePortIPs1 {
   791  		upsert2ndWanted[nodePort.Hash()] = loadbalancer.SVC{
   792  			Type:     loadbalancer.SVCTypeNodePort,
   793  			Frontend: *nodePort,
   794  			Backends: []*loadbalancer.Backend{
   795  				{
   796  					FEPortName: "port-udp-80",
   797  					L3n4Addr: loadbalancer.L3n4Addr{
   798  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
   799  						L4Addr: loadbalancer.L4Addr{
   800  							Protocol: loadbalancer.UDP,
   801  							Port:     8080,
   802  						},
   803  					},
   804  					Weight: loadbalancer.DefaultBackendWeight,
   805  				},
   806  				{
   807  					FEPortName: "port-udp-80",
   808  					L3n4Addr: loadbalancer.L3n4Addr{
   809  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.3"),
   810  						L4Addr: loadbalancer.L4Addr{
   811  							Protocol: loadbalancer.UDP,
   812  							Port:     8080,
   813  						},
   814  					},
   815  					Weight: loadbalancer.DefaultBackendWeight,
   816  				},
   817  			},
   818  		}
   819  	}
   820  	// for _, nodePort := range nodePortIPs2 {
   821  	// 	upsert2ndWanted[nodePort.Hash()] = loadbalancer.SVC{
   822  	// 		Type:     loadbalancer.SVCTypeNodePort,
   823  	// 		Frontend: *nodePort,
   824  	// 		Backends: []*loadbalancer.Backend{
   825  	// 			{
   826  	// 				L3n4Addr: loadbalancer.L3n4Addr{
   827  	// 					IP: net.ParseIP("10.0.0.2"),
   828  	// 					L4Addr: loadbalancer.L4Addr{
   829  	// 						Protocol: loadbalancer.TCP,
   830  	// 						Port:     8081,
   831  	// 					},
   832  	// 				},
   833  	// 			},
   834  	// 			{
   835  	// 				L3n4Addr: loadbalancer.L3n4Addr{
   836  	// 					IP: net.ParseIP("10.0.0.3"),
   837  	// 					L4Addr: loadbalancer.L4Addr{
   838  	// 						Protocol: loadbalancer.TCP,
   839  	// 						Port:     8081,
   840  	// 					},
   841  	// 				},
   842  	// 			},
   843  	// 		},
   844  	// 	}
   845  	// }
   846  	for _, nodePort := range nodePortIPs3 {
   847  		upsert2ndWanted[nodePort.Hash()] = loadbalancer.SVC{
   848  			Type:     loadbalancer.SVCTypeNodePort,
   849  			Frontend: *nodePort,
   850  			Backends: []*loadbalancer.Backend{
   851  				{
   852  					FEPortName: "port-tcp-81",
   853  					L3n4Addr: loadbalancer.L3n4Addr{
   854  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
   855  						L4Addr: loadbalancer.L4Addr{
   856  							Protocol: loadbalancer.TCP,
   857  							Port:     8081,
   858  						},
   859  					},
   860  					Weight: loadbalancer.DefaultBackendWeight,
   861  				},
   862  				{
   863  					FEPortName: "port-tcp-81",
   864  					L3n4Addr: loadbalancer.L3n4Addr{
   865  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.3"),
   866  						L4Addr: loadbalancer.L4Addr{
   867  							Protocol: loadbalancer.TCP,
   868  							Port:     8081,
   869  						},
   870  					},
   871  					Weight: loadbalancer.DefaultBackendWeight,
   872  				},
   873  			},
   874  		}
   875  	}
   876  
   877  	del1stWanted := map[string]struct{}{
   878  		clusterIP1.Hash(): {},
   879  		// clusterIP2.Hash(): {},
   880  		clusterIP3.Hash(): {},
   881  	}
   882  	for _, nodePort := range append(nodePortIPs1, nodePortIPs3...) {
   883  		del1stWanted[nodePort.Hash()] = struct{}{}
   884  	}
   885  
   886  	upsert1st := map[string]loadbalancer.SVC{}
   887  	upsert2nd := map[string]loadbalancer.SVC{}
   888  	del1st := map[string]struct{}{}
   889  
   890  	svcUpsertManagerCalls, svcDeleteManagerCalls := 0, 0
   891  
   892  	svcManager := &fakeSvcManager{
   893  		OnUpsertService: func(p *loadbalancer.SVC) (bool, loadbalancer.ID, error) {
   894  			sort.Slice(p.Backends, func(i, j int) bool {
   895  				return p.Backends[i].AddrCluster.Less(p.Backends[j].AddrCluster)
   896  			})
   897  			switch {
   898  			// 1st update endpoints
   899  			case svcUpsertManagerCalls < len(upsert1stWanted):
   900  				upsert1st[p.Frontend.Hash()] = loadbalancer.SVC{
   901  					Frontend: p.Frontend,
   902  					Backends: p.Backends,
   903  					Type:     p.Type,
   904  				}
   905  			// 2nd update endpoints
   906  			case svcUpsertManagerCalls < len(upsert1stWanted)+len(upsert2ndWanted):
   907  				upsert2nd[p.Frontend.Hash()] = loadbalancer.SVC{
   908  					Frontend: p.Frontend,
   909  					Backends: p.Backends,
   910  					Type:     p.Type,
   911  				}
   912  			}
   913  			svcUpsertManagerCalls++
   914  			return false, 0, nil
   915  		},
   916  		OnDeleteService: func(fe loadbalancer.L3n4Addr) (b bool, e error) {
   917  			del1st[fe.Hash()] = struct{}{}
   918  			svcDeleteManagerCalls++
   919  			return true, nil
   920  		},
   921  	}
   922  
   923  	db, nodeAddrs := newDB(t)
   924  	k8sSvcCache := k8s.NewServiceCache(db, nodeAddrs)
   925  	svcWatcher := &K8sServiceWatcher{
   926  		k8sSvcCache: k8sSvcCache,
   927  		svcManager:  svcManager,
   928  	}
   929  
   930  	go svcWatcher.k8sServiceHandler()
   931  	swg := lock.NewStoppableWaitGroup()
   932  
   933  	k8sSvcCache.UpdateService(k8sSvc, swg)
   934  	k8sSvcCache.UpdateEndpoints(k8s.ParseEndpoints(ep1stApply), swg)
   935  	// Running a 2nd update should also trigger a new upsert service
   936  	k8sSvcCache.UpdateEndpoints(k8s.ParseEndpoints(ep2ndApply), swg)
   937  	// Running a 3rd update should also not trigger anything because the
   938  	// endpoints are the same
   939  	k8sSvcCache.UpdateEndpoints(k8s.ParseEndpoints(ep2ndApply), swg)
   940  
   941  	k8sSvcCache.DeleteService(k8sSvc, swg)
   942  
   943  	swg.Stop()
   944  	swg.Wait()
   945  	require.Equal(t, len(upsert1stWanted)+len(upsert2ndWanted), svcUpsertManagerCalls)
   946  	require.Equal(t, len(del1stWanted), svcDeleteManagerCalls)
   947  
   948  	require.EqualValues(t, upsert1stWanted, upsert1st)
   949  	require.EqualValues(t, upsert2ndWanted, upsert2nd)
   950  	require.EqualValues(t, del1stWanted, del1st)
   951  }
   952  
   953  func Test_addK8sSVCs_GH9576_1(t *testing.T) {
   954  	// Adding service without any endpoints and later on modifying the service,
   955  	// cilium should:
   956  	// 1) delete the non existing services from the datapath.
   957  
   958  	enableNodePortBak := option.Config.EnableNodePort
   959  	option.Config.EnableNodePort = true
   960  	defer func() {
   961  		option.Config.EnableNodePort = enableNodePortBak
   962  	}()
   963  
   964  	k8sSvc1stApply := &slim_corev1.Service{
   965  		ObjectMeta: slim_metav1.ObjectMeta{
   966  			Name:      "foo",
   967  			Namespace: "bar",
   968  			Labels: map[string]string{
   969  				"foo": "bar",
   970  			},
   971  		},
   972  		Spec: slim_corev1.ServiceSpec{
   973  			Ports: []slim_corev1.ServicePort{
   974  				{
   975  					Name:     "port-udp-80",
   976  					Protocol: slim_corev1.ProtocolUDP,
   977  					Port:     80,
   978  					NodePort: 18080,
   979  				},
   980  				{
   981  					Name:     "port-tcp-81",
   982  					Protocol: slim_corev1.ProtocolTCP,
   983  					Port:     81,
   984  					NodePort: 18081,
   985  				},
   986  			},
   987  			ClusterIP: "172.0.20.1",
   988  			Type:      slim_corev1.ServiceTypeNodePort,
   989  		},
   990  	}
   991  
   992  	k8sSvc2ndApply := &slim_corev1.Service{
   993  		ObjectMeta: slim_metav1.ObjectMeta{
   994  			Name:      "foo",
   995  			Namespace: "bar",
   996  			Labels: map[string]string{
   997  				"foo": "bar",
   998  			},
   999  		},
  1000  		Spec: slim_corev1.ServiceSpec{
  1001  			Ports: []slim_corev1.ServicePort{
  1002  				{
  1003  					Name:     "port-udp-80",
  1004  					Protocol: slim_corev1.ProtocolUDP,
  1005  					Port:     8083,
  1006  				},
  1007  				{
  1008  					Name:     "port-tcp-81",
  1009  					Protocol: slim_corev1.ProtocolTCP,
  1010  					Port:     81,
  1011  				},
  1012  			},
  1013  			ClusterIP: "172.0.20.1",
  1014  			Type:      slim_corev1.ServiceTypeClusterIP,
  1015  		},
  1016  	}
  1017  
  1018  	ep1stApply := &slim_corev1.Endpoints{
  1019  		ObjectMeta: slim_metav1.ObjectMeta{
  1020  			Name:      "foo",
  1021  			Namespace: "bar",
  1022  		},
  1023  		Subsets: []slim_corev1.EndpointSubset{
  1024  			{
  1025  				Addresses: []slim_corev1.EndpointAddress{{IP: "10.0.0.2"}},
  1026  				Ports: []slim_corev1.EndpointPort{
  1027  					{
  1028  						Name:     "port-udp-80",
  1029  						Port:     8080,
  1030  						Protocol: slim_corev1.ProtocolUDP,
  1031  					},
  1032  					{
  1033  						Name:     "port-tcp-81",
  1034  						Protocol: slim_corev1.ProtocolTCP,
  1035  						Port:     8081,
  1036  					},
  1037  				},
  1038  			},
  1039  		},
  1040  	}
  1041  
  1042  	clusterIP1 := loadbalancer.NewL3n4AddrID(loadbalancer.UDP, cmtypes.MustParseAddrCluster("172.0.20.1"), 80, loadbalancer.ScopeExternal, 0)
  1043  	clusterIP2 := loadbalancer.NewL3n4AddrID(loadbalancer.TCP, cmtypes.MustParseAddrCluster("172.0.20.1"), 81, loadbalancer.ScopeExternal, 0)
  1044  	ipv4NodePortAddrCluster := cmtypes.MustAddrClusterFromIP(fakeTypes.IPv4NodePortAddress)
  1045  	ipv4InternalAddrCluster := cmtypes.MustAddrClusterFromIP(fakeTypes.IPv4InternalAddress)
  1046  
  1047  	nodePortIPs1 := []*loadbalancer.L3n4AddrID{
  1048  		loadbalancer.NewL3n4AddrID(loadbalancer.UDP, cmtypes.MustParseAddrCluster("0.0.0.0"), 18080, loadbalancer.ScopeExternal, 0),
  1049  		loadbalancer.NewL3n4AddrID(loadbalancer.UDP, ipv4NodePortAddrCluster, 18080, loadbalancer.ScopeExternal, 0),
  1050  		loadbalancer.NewL3n4AddrID(loadbalancer.UDP, ipv4InternalAddrCluster, 18080, loadbalancer.ScopeExternal, 0),
  1051  	}
  1052  	nodePortIPs2 := []*loadbalancer.L3n4AddrID{
  1053  		loadbalancer.NewL3n4AddrID(loadbalancer.TCP, cmtypes.MustParseAddrCluster("0.0.0.0"), 18081, loadbalancer.ScopeExternal, 0),
  1054  		loadbalancer.NewL3n4AddrID(loadbalancer.TCP, ipv4NodePortAddrCluster, 18081, loadbalancer.ScopeExternal, 0),
  1055  		loadbalancer.NewL3n4AddrID(loadbalancer.TCP, ipv4InternalAddrCluster, 18081, loadbalancer.ScopeExternal, 0),
  1056  	}
  1057  
  1058  	upsert1stWanted := map[string]loadbalancer.SVC{
  1059  		clusterIP1.Hash(): {
  1060  			Type:     loadbalancer.SVCTypeClusterIP,
  1061  			Frontend: *clusterIP1,
  1062  			Backends: []*loadbalancer.Backend{
  1063  				{
  1064  					FEPortName: "port-udp-80",
  1065  					L3n4Addr: loadbalancer.L3n4Addr{
  1066  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
  1067  						L4Addr: loadbalancer.L4Addr{
  1068  							Protocol: loadbalancer.UDP,
  1069  							Port:     8080,
  1070  						},
  1071  					},
  1072  					Weight: loadbalancer.DefaultBackendWeight,
  1073  				},
  1074  			},
  1075  		},
  1076  		clusterIP2.Hash(): {
  1077  			Type:     loadbalancer.SVCTypeClusterIP,
  1078  			Frontend: *clusterIP2,
  1079  			Backends: []*loadbalancer.Backend{
  1080  				{
  1081  					FEPortName: "port-tcp-81",
  1082  					L3n4Addr: loadbalancer.L3n4Addr{
  1083  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
  1084  						L4Addr: loadbalancer.L4Addr{
  1085  							Protocol: loadbalancer.TCP,
  1086  							Port:     8081,
  1087  						},
  1088  					},
  1089  					Weight: loadbalancer.DefaultBackendWeight,
  1090  				},
  1091  			},
  1092  		},
  1093  	}
  1094  	for _, nodePort := range nodePortIPs1 {
  1095  		upsert1stWanted[nodePort.Hash()] = loadbalancer.SVC{
  1096  			Type:     loadbalancer.SVCTypeNodePort,
  1097  			Frontend: *nodePort,
  1098  			Backends: []*loadbalancer.Backend{
  1099  				{
  1100  					FEPortName: "port-udp-80",
  1101  					L3n4Addr: loadbalancer.L3n4Addr{
  1102  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
  1103  						L4Addr: loadbalancer.L4Addr{
  1104  							Protocol: loadbalancer.UDP,
  1105  							Port:     8080,
  1106  						},
  1107  					},
  1108  					Weight: loadbalancer.DefaultBackendWeight,
  1109  				},
  1110  			},
  1111  		}
  1112  	}
  1113  	for _, nodePort := range nodePortIPs2 {
  1114  		upsert1stWanted[nodePort.Hash()] = loadbalancer.SVC{
  1115  			Type:     loadbalancer.SVCTypeNodePort,
  1116  			Frontend: *nodePort,
  1117  			Backends: []*loadbalancer.Backend{
  1118  				{
  1119  					FEPortName: "port-tcp-81",
  1120  					L3n4Addr: loadbalancer.L3n4Addr{
  1121  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
  1122  						L4Addr: loadbalancer.L4Addr{
  1123  							Protocol: loadbalancer.TCP,
  1124  							Port:     8081,
  1125  						},
  1126  					},
  1127  					Weight: loadbalancer.DefaultBackendWeight,
  1128  				},
  1129  			},
  1130  		}
  1131  	}
  1132  
  1133  	clusterIP3 := loadbalancer.NewL3n4AddrID(loadbalancer.UDP, cmtypes.MustParseAddrCluster("172.0.20.1"), 8083, loadbalancer.ScopeExternal, 0)
  1134  
  1135  	upsert2ndWanted := map[string]loadbalancer.SVC{
  1136  		clusterIP2.Hash(): {
  1137  			Type:     loadbalancer.SVCTypeClusterIP,
  1138  			Frontend: *clusterIP2,
  1139  			Backends: []*loadbalancer.Backend{
  1140  				{
  1141  					FEPortName: "port-tcp-81",
  1142  					L3n4Addr: loadbalancer.L3n4Addr{
  1143  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
  1144  						L4Addr: loadbalancer.L4Addr{
  1145  							Protocol: loadbalancer.TCP,
  1146  							Port:     8081,
  1147  						},
  1148  					},
  1149  					Weight: loadbalancer.DefaultBackendWeight,
  1150  				},
  1151  			},
  1152  		},
  1153  		clusterIP3.Hash(): {
  1154  			Type:     loadbalancer.SVCTypeClusterIP,
  1155  			Frontend: *clusterIP3,
  1156  			Backends: []*loadbalancer.Backend{
  1157  				{
  1158  					FEPortName: "port-udp-80",
  1159  					L3n4Addr: loadbalancer.L3n4Addr{
  1160  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
  1161  						L4Addr: loadbalancer.L4Addr{
  1162  							Protocol: loadbalancer.UDP,
  1163  							Port:     8080,
  1164  						},
  1165  					},
  1166  					Weight: loadbalancer.DefaultBackendWeight,
  1167  				},
  1168  			},
  1169  		},
  1170  	}
  1171  
  1172  	del1stWanted := map[string]loadbalancer.L3n4Addr{
  1173  		clusterIP1.Hash(): clusterIP1.L3n4Addr,
  1174  	}
  1175  	for _, nodePort := range append(nodePortIPs1, nodePortIPs2...) {
  1176  		del1stWanted[nodePort.Hash()] = nodePort.L3n4Addr
  1177  	}
  1178  
  1179  	upsert1st := map[string]loadbalancer.SVC{}
  1180  	upsert2nd := map[string]loadbalancer.SVC{}
  1181  	del1st := map[string]loadbalancer.L3n4Addr{}
  1182  
  1183  	svcUpsertManagerCalls, svcDeleteManagerCalls := 0, 0
  1184  	wantSvcUpsertManagerCalls := len(upsert1stWanted) + len(upsert2ndWanted)
  1185  	wantSvcDeleteManagerCalls := len(del1stWanted)
  1186  
  1187  	svcManager := &fakeSvcManager{
  1188  		OnUpsertService: func(p *loadbalancer.SVC) (bool, loadbalancer.ID, error) {
  1189  			sort.Slice(p.Backends, func(i, j int) bool {
  1190  				return p.Backends[i].AddrCluster.Less(p.Backends[j].AddrCluster)
  1191  			})
  1192  			switch {
  1193  			// 1st update service-endpoints
  1194  			case svcUpsertManagerCalls < len(upsert1stWanted):
  1195  				upsert1st[p.Frontend.Hash()] = loadbalancer.SVC{
  1196  					Frontend: p.Frontend,
  1197  					Backends: p.Backends,
  1198  					Type:     p.Type,
  1199  				}
  1200  			// 2nd update services
  1201  			case svcUpsertManagerCalls < len(upsert1stWanted)+len(upsert2ndWanted):
  1202  				upsert2nd[p.Frontend.Hash()] = loadbalancer.SVC{
  1203  					Frontend: p.Frontend,
  1204  					Backends: p.Backends,
  1205  					Type:     p.Type,
  1206  				}
  1207  			}
  1208  			svcUpsertManagerCalls++
  1209  			return false, 0, nil
  1210  		},
  1211  		OnDeleteService: func(fe loadbalancer.L3n4Addr) (b bool, e error) {
  1212  			del1st[fe.Hash()] = fe
  1213  			svcDeleteManagerCalls++
  1214  			return true, nil
  1215  		},
  1216  	}
  1217  
  1218  	db, nodeAddrs := newDB(t)
  1219  	k8sSvcCache := k8s.NewServiceCache(db, nodeAddrs)
  1220  	svcWatcher := &K8sServiceWatcher{
  1221  		k8sSvcCache: k8sSvcCache,
  1222  		svcManager:  svcManager,
  1223  	}
  1224  
  1225  	go svcWatcher.k8sServiceHandler()
  1226  	swg := lock.NewStoppableWaitGroup()
  1227  
  1228  	k8sSvcCache.UpdateService(k8sSvc1stApply, swg)
  1229  	k8sSvcCache.UpdateEndpoints(k8s.ParseEndpoints(ep1stApply), swg)
  1230  
  1231  	k8sSvcCache.UpdateService(k8sSvc2ndApply, swg)
  1232  
  1233  	swg.Stop()
  1234  	swg.Wait()
  1235  	require.Equal(t, wantSvcUpsertManagerCalls, svcUpsertManagerCalls)
  1236  	require.Equal(t, wantSvcDeleteManagerCalls, svcDeleteManagerCalls)
  1237  
  1238  	require.EqualValues(t, upsert1stWanted, upsert1st)
  1239  	require.EqualValues(t, upsert2ndWanted, upsert2nd)
  1240  	require.EqualValues(t, del1stWanted, del1st)
  1241  }
  1242  
  1243  func Test_addK8sSVCs_GH9576_2(t *testing.T) {
  1244  	// Adding service without any endpoints and later on modifying the service,
  1245  	// cilium should:
  1246  	// 1) delete the non existing endpoints from the datapath, i.e., updating
  1247  	//    services without any backend.
  1248  
  1249  	enableNodePortBak := option.Config.EnableNodePort
  1250  	option.Config.EnableNodePort = true
  1251  	defer func() {
  1252  		option.Config.EnableNodePort = enableNodePortBak
  1253  	}()
  1254  
  1255  	k8sSvc1stApply := &slim_corev1.Service{
  1256  		ObjectMeta: slim_metav1.ObjectMeta{
  1257  			Name:      "foo",
  1258  			Namespace: "bar",
  1259  			Labels: map[string]string{
  1260  				"foo": "bar",
  1261  			},
  1262  		},
  1263  		Spec: slim_corev1.ServiceSpec{
  1264  			Ports: []slim_corev1.ServicePort{
  1265  				{
  1266  					Name:     "port-udp-80",
  1267  					Protocol: slim_corev1.ProtocolUDP,
  1268  					Port:     80,
  1269  					NodePort: 18080,
  1270  				},
  1271  				{
  1272  					Name:     "port-tcp-81",
  1273  					Protocol: slim_corev1.ProtocolTCP,
  1274  					Port:     81,
  1275  					NodePort: 18081,
  1276  				},
  1277  			},
  1278  			ClusterIP: "172.0.20.1",
  1279  			Type:      slim_corev1.ServiceTypeNodePort,
  1280  		},
  1281  	}
  1282  
  1283  	ep1stApply := &slim_corev1.Endpoints{
  1284  		ObjectMeta: slim_metav1.ObjectMeta{
  1285  			Name:      "foo",
  1286  			Namespace: "bar",
  1287  		},
  1288  		Subsets: []slim_corev1.EndpointSubset{
  1289  			{
  1290  				Addresses: []slim_corev1.EndpointAddress{{IP: "10.0.0.2"}},
  1291  				Ports: []slim_corev1.EndpointPort{
  1292  					{
  1293  						Name:     "port-udp-80",
  1294  						Port:     8080,
  1295  						Protocol: slim_corev1.ProtocolUDP,
  1296  					},
  1297  					{
  1298  						Name:     "port-tcp-81",
  1299  						Protocol: slim_corev1.ProtocolTCP,
  1300  						Port:     8081,
  1301  					},
  1302  				},
  1303  			},
  1304  		},
  1305  	}
  1306  
  1307  	ep2ndApply := &slim_corev1.Endpoints{
  1308  		ObjectMeta: slim_metav1.ObjectMeta{
  1309  			Name:      "foo",
  1310  			Namespace: "bar",
  1311  		},
  1312  		Subsets: []slim_corev1.EndpointSubset{
  1313  			{
  1314  				Addresses: []slim_corev1.EndpointAddress{{IP: "10.0.0.3"}},
  1315  				Ports: []slim_corev1.EndpointPort{
  1316  					{
  1317  						Name:     "port-udp-80",
  1318  						Port:     8080,
  1319  						Protocol: slim_corev1.ProtocolUDP,
  1320  					},
  1321  				},
  1322  			},
  1323  		},
  1324  	}
  1325  
  1326  	clusterIP1 := loadbalancer.NewL3n4AddrID(loadbalancer.UDP, cmtypes.MustParseAddrCluster("172.0.20.1"), 80, loadbalancer.ScopeExternal, 0)
  1327  	clusterIP2 := loadbalancer.NewL3n4AddrID(loadbalancer.TCP, cmtypes.MustParseAddrCluster("172.0.20.1"), 81, loadbalancer.ScopeExternal, 0)
  1328  	ipv4NodePortAddrCluster := cmtypes.MustAddrClusterFromIP(fakeTypes.IPv4NodePortAddress)
  1329  	ipv4InternalAddrCluster := cmtypes.MustAddrClusterFromIP(fakeTypes.IPv4InternalAddress)
  1330  
  1331  	nodePortIPs1 := []*loadbalancer.L3n4AddrID{
  1332  		loadbalancer.NewL3n4AddrID(loadbalancer.UDP, cmtypes.MustParseAddrCluster("0.0.0.0"), 18080, loadbalancer.ScopeExternal, 0),
  1333  		loadbalancer.NewL3n4AddrID(loadbalancer.UDP, ipv4NodePortAddrCluster, 18080, loadbalancer.ScopeExternal, 0),
  1334  		loadbalancer.NewL3n4AddrID(loadbalancer.UDP, ipv4InternalAddrCluster, 18080, loadbalancer.ScopeExternal, 0),
  1335  	}
  1336  	nodePortIPs2 := []*loadbalancer.L3n4AddrID{
  1337  		loadbalancer.NewL3n4AddrID(loadbalancer.TCP, cmtypes.MustParseAddrCluster("0.0.0.0"), 18081, loadbalancer.ScopeExternal, 0),
  1338  		loadbalancer.NewL3n4AddrID(loadbalancer.TCP, ipv4NodePortAddrCluster, 18081, loadbalancer.ScopeExternal, 0),
  1339  		loadbalancer.NewL3n4AddrID(loadbalancer.TCP, ipv4InternalAddrCluster, 18081, loadbalancer.ScopeExternal, 0),
  1340  	}
  1341  
  1342  	upsert1stWanted := map[string]loadbalancer.SVC{
  1343  		clusterIP1.Hash(): {
  1344  			Type:     loadbalancer.SVCTypeClusterIP,
  1345  			Frontend: *clusterIP1,
  1346  			Backends: []*loadbalancer.Backend{
  1347  				{
  1348  					FEPortName: "port-udp-80",
  1349  					L3n4Addr: loadbalancer.L3n4Addr{
  1350  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
  1351  						L4Addr: loadbalancer.L4Addr{
  1352  							Protocol: loadbalancer.UDP,
  1353  							Port:     8080,
  1354  						},
  1355  					},
  1356  					Weight: loadbalancer.DefaultBackendWeight,
  1357  				},
  1358  			},
  1359  		},
  1360  		clusterIP2.Hash(): {
  1361  			Type:     loadbalancer.SVCTypeClusterIP,
  1362  			Frontend: *clusterIP2,
  1363  			Backends: []*loadbalancer.Backend{
  1364  				{
  1365  					FEPortName: "port-tcp-81",
  1366  					L3n4Addr: loadbalancer.L3n4Addr{
  1367  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
  1368  						L4Addr: loadbalancer.L4Addr{
  1369  							Protocol: loadbalancer.TCP,
  1370  							Port:     8081,
  1371  						},
  1372  					},
  1373  					Weight: loadbalancer.DefaultBackendWeight,
  1374  				},
  1375  			},
  1376  		},
  1377  	}
  1378  	for _, nodePort := range nodePortIPs1 {
  1379  		upsert1stWanted[nodePort.Hash()] = loadbalancer.SVC{
  1380  			Type:     loadbalancer.SVCTypeNodePort,
  1381  			Frontend: *nodePort,
  1382  			Backends: []*loadbalancer.Backend{
  1383  				{
  1384  					FEPortName: "port-udp-80",
  1385  					L3n4Addr: loadbalancer.L3n4Addr{
  1386  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
  1387  						L4Addr: loadbalancer.L4Addr{
  1388  							Protocol: loadbalancer.UDP,
  1389  							Port:     8080,
  1390  						},
  1391  					},
  1392  					Weight: loadbalancer.DefaultBackendWeight,
  1393  				},
  1394  			},
  1395  		}
  1396  	}
  1397  	for _, nodePort := range nodePortIPs2 {
  1398  		upsert1stWanted[nodePort.Hash()] = loadbalancer.SVC{
  1399  			Type:     loadbalancer.SVCTypeNodePort,
  1400  			Frontend: *nodePort,
  1401  			Backends: []*loadbalancer.Backend{
  1402  				{
  1403  					FEPortName: "port-tcp-81",
  1404  					L3n4Addr: loadbalancer.L3n4Addr{
  1405  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
  1406  						L4Addr: loadbalancer.L4Addr{
  1407  							Protocol: loadbalancer.TCP,
  1408  							Port:     8081,
  1409  						},
  1410  					},
  1411  					Weight: loadbalancer.DefaultBackendWeight,
  1412  				},
  1413  			},
  1414  		}
  1415  	}
  1416  
  1417  	upsert2ndWanted := map[string]loadbalancer.SVC{
  1418  		clusterIP1.Hash(): {
  1419  			Type:     loadbalancer.SVCTypeClusterIP,
  1420  			Frontend: *clusterIP1,
  1421  			Backends: []*loadbalancer.Backend{
  1422  				{
  1423  					FEPortName: "port-udp-80",
  1424  					L3n4Addr: loadbalancer.L3n4Addr{
  1425  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.3"),
  1426  						L4Addr: loadbalancer.L4Addr{
  1427  							Protocol: loadbalancer.UDP,
  1428  							Port:     8080,
  1429  						},
  1430  					},
  1431  					Weight: loadbalancer.DefaultBackendWeight,
  1432  				},
  1433  			},
  1434  		},
  1435  		clusterIP2.Hash(): {
  1436  			Type:     loadbalancer.SVCTypeClusterIP,
  1437  			Frontend: *clusterIP2,
  1438  		},
  1439  	}
  1440  	for _, nodePort := range nodePortIPs1 {
  1441  		upsert2ndWanted[nodePort.Hash()] = loadbalancer.SVC{
  1442  			Type:     loadbalancer.SVCTypeNodePort,
  1443  			Frontend: *nodePort,
  1444  			Backends: []*loadbalancer.Backend{
  1445  				{
  1446  					FEPortName: "port-udp-80",
  1447  					L3n4Addr: loadbalancer.L3n4Addr{
  1448  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.3"),
  1449  						L4Addr: loadbalancer.L4Addr{
  1450  							Protocol: loadbalancer.UDP,
  1451  							Port:     8080,
  1452  						},
  1453  					},
  1454  					Weight: loadbalancer.DefaultBackendWeight,
  1455  				},
  1456  			},
  1457  		}
  1458  	}
  1459  	for _, nodePort := range nodePortIPs2 {
  1460  		upsert2ndWanted[nodePort.Hash()] = loadbalancer.SVC{
  1461  			Type:     loadbalancer.SVCTypeNodePort,
  1462  			Frontend: *nodePort,
  1463  		}
  1464  	}
  1465  
  1466  	del1stWanted := map[string]loadbalancer.L3n4Addr{}
  1467  	upsert1st := map[string]loadbalancer.SVC{}
  1468  	upsert2nd := map[string]loadbalancer.SVC{}
  1469  	del1st := map[string]loadbalancer.L3n4Addr{}
  1470  
  1471  	svcUpsertManagerCalls, svcDeleteManagerCalls := 0, 0
  1472  	wantSvcUpsertManagerCalls := len(upsert1stWanted) + len(upsert2ndWanted)
  1473  	wantSvcDeleteManagerCalls := len(del1stWanted)
  1474  
  1475  	svcManager := &fakeSvcManager{
  1476  		OnUpsertService: func(p *loadbalancer.SVC) (bool, loadbalancer.ID, error) {
  1477  			sort.Slice(p.Backends, func(i, j int) bool {
  1478  				return p.Backends[i].AddrCluster.Less(p.Backends[j].AddrCluster)
  1479  			})
  1480  			switch {
  1481  			// 1st update service-endpoints
  1482  			case svcUpsertManagerCalls < len(upsert1stWanted):
  1483  				upsert1st[p.Frontend.Hash()] = loadbalancer.SVC{
  1484  					Frontend: p.Frontend,
  1485  					Backends: p.Backends,
  1486  					Type:     p.Type,
  1487  				}
  1488  			// 2nd update services
  1489  			case svcUpsertManagerCalls < len(upsert1stWanted)+len(upsert2ndWanted):
  1490  				upsert2nd[p.Frontend.Hash()] = loadbalancer.SVC{
  1491  					Frontend: p.Frontend,
  1492  					Backends: p.Backends,
  1493  					Type:     p.Type,
  1494  				}
  1495  			}
  1496  			svcUpsertManagerCalls++
  1497  			return false, 0, nil
  1498  		},
  1499  		OnDeleteService: func(fe loadbalancer.L3n4Addr) (b bool, e error) {
  1500  			del1st[fe.Hash()] = fe
  1501  			svcDeleteManagerCalls++
  1502  			return true, nil
  1503  		},
  1504  	}
  1505  
  1506  	db, nodeAddrs := newDB(t)
  1507  	k8sSvcCache := k8s.NewServiceCache(db, nodeAddrs)
  1508  	svcWatcher := &K8sServiceWatcher{
  1509  		k8sSvcCache: k8sSvcCache,
  1510  		svcManager:  svcManager,
  1511  	}
  1512  
  1513  	go svcWatcher.k8sServiceHandler()
  1514  	swg := lock.NewStoppableWaitGroup()
  1515  
  1516  	k8sSvcCache.UpdateService(k8sSvc1stApply, swg)
  1517  	k8sSvcCache.UpdateEndpoints(k8s.ParseEndpoints(ep1stApply), swg)
  1518  	k8sSvcCache.UpdateEndpoints(k8s.ParseEndpoints(ep2ndApply), swg)
  1519  
  1520  	swg.Stop()
  1521  	swg.Wait()
  1522  
  1523  	require.Equal(t, wantSvcUpsertManagerCalls, svcUpsertManagerCalls)
  1524  	require.Equal(t, wantSvcDeleteManagerCalls, svcDeleteManagerCalls)
  1525  
  1526  	require.EqualValues(t, upsert1stWanted, upsert1st)
  1527  	require.EqualValues(t, upsert2ndWanted, upsert2nd)
  1528  	require.EqualValues(t, del1stWanted, del1st)
  1529  }
  1530  
  1531  func Test_addK8sSVCs_ExternalIPs(t *testing.T) {
  1532  	enableNodePortBak := option.Config.EnableNodePort
  1533  	option.Config.EnableNodePort = true
  1534  	defer func() {
  1535  		option.Config.EnableNodePort = enableNodePortBak
  1536  	}()
  1537  
  1538  	svc1stApply := &slim_corev1.Service{
  1539  		ObjectMeta: slim_metav1.ObjectMeta{
  1540  			Name:      "foo",
  1541  			Namespace: "bar",
  1542  			Labels: map[string]string{
  1543  				"foo": "bar",
  1544  			},
  1545  		},
  1546  		Spec: slim_corev1.ServiceSpec{
  1547  			Ports: []slim_corev1.ServicePort{
  1548  				{
  1549  					Name:     "port-udp-80",
  1550  					Protocol: slim_corev1.ProtocolUDP,
  1551  					Port:     80,
  1552  					NodePort: 18080,
  1553  				},
  1554  				// FIXME: We don't distinguish about the protocol being used
  1555  				//        so we can't tell if a UDP/80 maps to port 8080/udp
  1556  				//        or if TCP/80 maps to port 8081/TCP
  1557  				// {
  1558  				// 	Name:       "port-tcp-80",
  1559  				// 	Protocol:  slim_corev1.ProtocolTCP,
  1560  				// 	Port:       80,
  1561  				// 	TargetPort: intstr.FromString("port-80-t"),
  1562  				//  NodePort:   18080,
  1563  				// },
  1564  				{
  1565  					Name:     "port-tcp-81",
  1566  					Protocol: slim_corev1.ProtocolTCP,
  1567  					Port:     81,
  1568  					NodePort: 18081,
  1569  				},
  1570  			},
  1571  			Selector:              nil,
  1572  			ClusterIP:             "172.0.20.1",
  1573  			Type:                  slim_corev1.ServiceTypeNodePort,
  1574  			ExternalIPs:           []string{"127.8.8.8", "127.9.9.9"},
  1575  			SessionAffinity:       "",
  1576  			ExternalTrafficPolicy: "",
  1577  			HealthCheckNodePort:   0,
  1578  			SessionAffinityConfig: nil,
  1579  		},
  1580  	}
  1581  
  1582  	svc2ndApply := svc1stApply.DeepCopy()
  1583  	svc2ndApply.Spec.ExternalIPs = []string{"127.8.8.8"}
  1584  
  1585  	ep1stApply := &slim_corev1.Endpoints{
  1586  		ObjectMeta: slim_metav1.ObjectMeta{
  1587  			Name:      "foo",
  1588  			Namespace: "bar",
  1589  		},
  1590  		Subsets: []slim_corev1.EndpointSubset{
  1591  			{
  1592  				Addresses: []slim_corev1.EndpointAddress{{IP: "10.0.0.2"}},
  1593  				Ports: []slim_corev1.EndpointPort{
  1594  					{
  1595  						Name:     "port-udp-80",
  1596  						Port:     8080,
  1597  						Protocol: slim_corev1.ProtocolUDP,
  1598  					},
  1599  					// FIXME: We don't distinguish about the protocol being used
  1600  					//        so we can't tell if a UDP/80 maps to port 8080/udp
  1601  					//        or if TCP/80 maps to port 8081/TCP
  1602  					// {
  1603  					// 	Name:     "port-tcp-80",
  1604  					// 	Protocol:slim_corev1.ProtocolTCP,
  1605  					// 	Port:     8081,
  1606  					// },
  1607  					{
  1608  						Name:     "port-tcp-81",
  1609  						Protocol: slim_corev1.ProtocolTCP,
  1610  						Port:     8081,
  1611  					},
  1612  				},
  1613  			},
  1614  		},
  1615  	}
  1616  
  1617  	ep2ndApply := ep1stApply.DeepCopy()
  1618  	ep2ndApply.Subsets[0].Addresses = append(
  1619  		ep2ndApply.Subsets[0].Addresses,
  1620  		slim_corev1.EndpointAddress{IP: "10.0.0.3"},
  1621  	)
  1622  
  1623  	clusterIP1 := loadbalancer.NewL3n4AddrID(loadbalancer.UDP, cmtypes.MustParseAddrCluster("172.0.20.1"), 80, loadbalancer.ScopeExternal, 0)
  1624  	// clusterIP2 := loadbalancer.NewL3n4AddrID(loadbalancer.TCP, net.ParseIP("172.0.20.1"), 80, loadbalancer.ScopeExternal, 0)
  1625  	clusterIP3 := loadbalancer.NewL3n4AddrID(loadbalancer.TCP, cmtypes.MustParseAddrCluster("172.0.20.1"), 81, loadbalancer.ScopeExternal, 0)
  1626  
  1627  	upsert1stWanted := map[string]loadbalancer.SVC{
  1628  		clusterIP1.Hash(): {
  1629  			Type:     loadbalancer.SVCTypeClusterIP,
  1630  			Frontend: *clusterIP1,
  1631  			Backends: []*loadbalancer.Backend{
  1632  				{
  1633  					FEPortName: "port-udp-80",
  1634  					L3n4Addr: loadbalancer.L3n4Addr{
  1635  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
  1636  						L4Addr: loadbalancer.L4Addr{
  1637  							Protocol: loadbalancer.UDP,
  1638  							Port:     8080,
  1639  						},
  1640  					},
  1641  					Weight: loadbalancer.DefaultBackendWeight,
  1642  				},
  1643  			},
  1644  		},
  1645  		// FIXME: We don't distinguish about the protocol being used
  1646  		//        so we can't tell if a UDP/80 maps to port 8080/udp
  1647  		//        or if TCP/80 maps to port 8081/TCP
  1648  		// clusterIP2.Hash(): {
  1649  		// 	Type:     loadbalancer.SVCTypeClusterIP,
  1650  		// 	Frontend: *clusterIP2,
  1651  		// 	Backends: []*loadbalancer.Backend{
  1652  		// 		{
  1653  		// 			L3n4Addr: loadbalancer.L3n4Addr{
  1654  		// 				IP: net.ParseIP("10.0.0.2"),
  1655  		// 				L4Addr: loadbalancer.L4Addr{
  1656  		// 					Protocol: loadbalancer.TCP,
  1657  		// 					Port:     8081,
  1658  		// 				},
  1659  		// 			},
  1660  		// 		},
  1661  		// 	},
  1662  		// },
  1663  		clusterIP3.Hash(): {
  1664  			Type:     loadbalancer.SVCTypeClusterIP,
  1665  			Frontend: *clusterIP3,
  1666  			Backends: []*loadbalancer.Backend{
  1667  				{
  1668  					FEPortName: "port-tcp-81",
  1669  					L3n4Addr: loadbalancer.L3n4Addr{
  1670  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
  1671  						L4Addr: loadbalancer.L4Addr{
  1672  							Protocol: loadbalancer.TCP,
  1673  							Port:     8081,
  1674  						},
  1675  					},
  1676  					Weight: loadbalancer.DefaultBackendWeight,
  1677  				},
  1678  			},
  1679  		},
  1680  	}
  1681  
  1682  	externalIP1 := loadbalancer.NewL3n4AddrID(loadbalancer.UDP, cmtypes.MustParseAddrCluster("127.8.8.8"), 80, loadbalancer.ScopeExternal, 0)
  1683  	// externalIP2 := loadbalancer.NewL3n4AddrID(loadbalancer.TCP, net.ParseIP("127.8.8.8"), 80, loadbalancer.ScopeExternal, 0)
  1684  	externalIP3 := loadbalancer.NewL3n4AddrID(loadbalancer.TCP, cmtypes.MustParseAddrCluster("127.8.8.8"), 81, loadbalancer.ScopeExternal, 0)
  1685  	externalIP4 := loadbalancer.NewL3n4AddrID(loadbalancer.UDP, cmtypes.MustParseAddrCluster("127.9.9.9"), 80, loadbalancer.ScopeExternal, 0)
  1686  	// externalIP5 := loadbalancer.NewL3n4AddrID(loadbalancer.TCP, net.ParseIP("127.9.9.9"), 80, loadbalancer.ScopeExternal, 0)
  1687  	externalIP6 := loadbalancer.NewL3n4AddrID(loadbalancer.TCP, cmtypes.MustParseAddrCluster("127.9.9.9"), 81, loadbalancer.ScopeExternal, 0)
  1688  	for _, externalIP := range []*loadbalancer.L3n4AddrID{externalIP1, externalIP4} {
  1689  		upsert1stWanted[externalIP.Hash()] = loadbalancer.SVC{
  1690  			Type:     loadbalancer.SVCTypeExternalIPs,
  1691  			Frontend: *externalIP,
  1692  			Backends: []*loadbalancer.Backend{
  1693  				{
  1694  					FEPortName: "port-udp-80",
  1695  					L3n4Addr: loadbalancer.L3n4Addr{
  1696  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
  1697  						L4Addr: loadbalancer.L4Addr{
  1698  							Protocol: loadbalancer.UDP,
  1699  							Port:     8080,
  1700  						},
  1701  					},
  1702  					Weight: loadbalancer.DefaultBackendWeight,
  1703  				},
  1704  			},
  1705  		}
  1706  	}
  1707  	// for _, externalIP := range []*loadbalancer.L3n4AddrID{externalIP2, externalIP5} {
  1708  	// 	upsert1stWanted[externalIP.Hash()] = loadbalancer.SVC{
  1709  	// 		Type:     loadbalancer.SVCTypeExternalIPs,
  1710  	// 		Frontend: *externalIP,
  1711  	// 		Backends: []*loadbalancer.Backend{
  1712  	// 			{
  1713  	// 				L3n4Addr: loadbalancer.L3n4Addr{
  1714  	// 					IP: net.ParseIP("10.0.0.2"),
  1715  	// 					L4Addr: loadbalancer.L4Addr{
  1716  	// 						Protocol: loadbalancer.UDP,
  1717  	// 						Port:     8080,
  1718  	// 					},
  1719  	// 				},
  1720  	// 			},
  1721  	// 		},
  1722  	// 	}
  1723  	// }
  1724  	for _, externalIP := range []*loadbalancer.L3n4AddrID{externalIP3, externalIP6} {
  1725  		upsert1stWanted[externalIP.Hash()] = loadbalancer.SVC{
  1726  			Type:     loadbalancer.SVCTypeExternalIPs,
  1727  			Frontend: *externalIP,
  1728  			Backends: []*loadbalancer.Backend{
  1729  				{
  1730  					FEPortName: "port-tcp-81",
  1731  					L3n4Addr: loadbalancer.L3n4Addr{
  1732  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
  1733  						L4Addr: loadbalancer.L4Addr{
  1734  							Protocol: loadbalancer.TCP,
  1735  							Port:     8081,
  1736  						},
  1737  					},
  1738  					Weight: loadbalancer.DefaultBackendWeight,
  1739  				},
  1740  			},
  1741  		}
  1742  	}
  1743  
  1744  	ipv4NodePortAddrCluster := cmtypes.MustAddrClusterFromIP(fakeTypes.IPv4NodePortAddress)
  1745  	ipv4InternalAddrCluster := cmtypes.MustAddrClusterFromIP(fakeTypes.IPv4InternalAddress)
  1746  
  1747  	nodePortIPs1 := []*loadbalancer.L3n4AddrID{
  1748  		loadbalancer.NewL3n4AddrID(loadbalancer.UDP, cmtypes.MustParseAddrCluster("0.0.0.0"), 18080, loadbalancer.ScopeExternal, 0),
  1749  		loadbalancer.NewL3n4AddrID(loadbalancer.UDP, ipv4NodePortAddrCluster, 18080, loadbalancer.ScopeExternal, 0),
  1750  		loadbalancer.NewL3n4AddrID(loadbalancer.UDP, ipv4InternalAddrCluster, 18080, loadbalancer.ScopeExternal, 0),
  1751  	}
  1752  	for _, nodePort := range nodePortIPs1 {
  1753  		upsert1stWanted[nodePort.Hash()] = loadbalancer.SVC{
  1754  			Type:     loadbalancer.SVCTypeNodePort,
  1755  			Frontend: *nodePort,
  1756  			Backends: []*loadbalancer.Backend{
  1757  				{
  1758  					FEPortName: "port-udp-80",
  1759  					L3n4Addr: loadbalancer.L3n4Addr{
  1760  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
  1761  						L4Addr: loadbalancer.L4Addr{
  1762  							Protocol: loadbalancer.UDP,
  1763  							Port:     8080,
  1764  						},
  1765  					},
  1766  					Weight: loadbalancer.DefaultBackendWeight,
  1767  				},
  1768  			},
  1769  		}
  1770  	}
  1771  	// nodePortIPs2 := []*loadbalancer.L3n4AddrID{
  1772  	// 	loadbalancer.NewL3n4AddrID(loadbalancer.TCP, net.ParseIP("0.0.0.0"), 18080, loadbalancer.ScopeExternal, 0),
  1773  	// 	loadbalancer.NewL3n4AddrID(loadbalancer.TCP, fakeDatapath.IPv4NodePortAddress, 18080, loadbalancer.ScopeExternal, 0),
  1774  	// 	loadbalancer.NewL3n4AddrID(loadbalancer.TCP, fakeDatapath.IPv4InternalAddress, 18080, loadbalancer.ScopeExternal, 0),
  1775  	// }
  1776  	// for _, nodePort := range nodePortIPs2 {
  1777  	// 	upsert1stWanted[nodePort.Hash()] = loadbalancer.SVC{
  1778  	// 		Type:     loadbalancer.SVCTypeNodePort,
  1779  	// 		Frontend: *nodePort,
  1780  	// 		Backends: []*loadbalancer.Backend{
  1781  	// 			{
  1782  	// 				L3n4Addr: loadbalancer.L3n4Addr{
  1783  	// 					IP: net.ParseIP("10.0.0.2"),
  1784  	// 					L4Addr: loadbalancer.L4Addr{
  1785  	// 						Protocol: loadbalancer.TCP,
  1786  	// 						Port:     8081,
  1787  	// 					},
  1788  	// 				},
  1789  	// 			},
  1790  	// 		},
  1791  	// 	}
  1792  	// }
  1793  	nodePortIPs3 := []*loadbalancer.L3n4AddrID{
  1794  		loadbalancer.NewL3n4AddrID(loadbalancer.TCP, cmtypes.MustParseAddrCluster("0.0.0.0"), 18081, loadbalancer.ScopeExternal, 0),
  1795  		loadbalancer.NewL3n4AddrID(loadbalancer.TCP, ipv4NodePortAddrCluster, 18081, loadbalancer.ScopeExternal, 0),
  1796  		loadbalancer.NewL3n4AddrID(loadbalancer.TCP, ipv4InternalAddrCluster, 18081, loadbalancer.ScopeExternal, 0),
  1797  	}
  1798  	for _, nodePort := range nodePortIPs3 {
  1799  		upsert1stWanted[nodePort.Hash()] = loadbalancer.SVC{
  1800  			Type:     loadbalancer.SVCTypeNodePort,
  1801  			Frontend: *nodePort,
  1802  			Backends: []*loadbalancer.Backend{
  1803  				{
  1804  					FEPortName: "port-tcp-81",
  1805  					L3n4Addr: loadbalancer.L3n4Addr{
  1806  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
  1807  						L4Addr: loadbalancer.L4Addr{
  1808  							Protocol: loadbalancer.TCP,
  1809  							Port:     8081,
  1810  						},
  1811  					},
  1812  					Weight: loadbalancer.DefaultBackendWeight,
  1813  				},
  1814  			},
  1815  		}
  1816  	}
  1817  
  1818  	upsert2ndWanted := map[string]loadbalancer.SVC{
  1819  		clusterIP1.Hash(): {
  1820  			Type:     loadbalancer.SVCTypeClusterIP,
  1821  			Frontend: *clusterIP1,
  1822  			Backends: []*loadbalancer.Backend{
  1823  				{
  1824  					FEPortName: "port-udp-80",
  1825  					L3n4Addr: loadbalancer.L3n4Addr{
  1826  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
  1827  						L4Addr: loadbalancer.L4Addr{
  1828  							Protocol: loadbalancer.UDP,
  1829  							Port:     8080,
  1830  						},
  1831  					},
  1832  					Weight: loadbalancer.DefaultBackendWeight,
  1833  				},
  1834  				{
  1835  					FEPortName: "port-udp-80",
  1836  					L3n4Addr: loadbalancer.L3n4Addr{
  1837  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.3"),
  1838  						L4Addr: loadbalancer.L4Addr{
  1839  							Protocol: loadbalancer.UDP,
  1840  							Port:     8080,
  1841  						},
  1842  					},
  1843  					Weight: loadbalancer.DefaultBackendWeight,
  1844  				},
  1845  			},
  1846  		},
  1847  		// FIXME: We don't distinguish about the protocol being used
  1848  		//        so we can't tell if a UDP/80 maps to port 8080/udp
  1849  		//        or if TCP/80 maps to port 8081/TCP
  1850  		// clusterIP2.Hash(): {
  1851  		// 	Type:     loadbalancer.SVCTypeClusterIP,
  1852  		// 	Frontend: *clusterIP2,
  1853  		// 	Backends: []*loadbalancer.Backend{
  1854  		// 		{
  1855  		// 			L3n4Addr: loadbalancer.L3n4Addr{
  1856  		// 				IP: net.ParseIP("10.0.0.2"),
  1857  		// 				L4Addr: loadbalancer.L4Addr{
  1858  		// 					Protocol: loadbalancer.TCP,
  1859  		// 					Port:     8081,
  1860  		// 				},
  1861  		// 			},
  1862  		// 		},
  1863  		// 		{
  1864  		// 			L3n4Addr: loadbalancer.L3n4Addr{
  1865  		// 				IP: net.ParseIP("10.0.0.3"),
  1866  		// 				L4Addr: loadbalancer.L4Addr{
  1867  		// 					Protocol: loadbalancer.TCP,
  1868  		// 					Port:     8081,
  1869  		// 				},
  1870  		// 			},
  1871  		// 		},
  1872  		// 	},
  1873  		// },
  1874  		clusterIP3.Hash(): {
  1875  			Type:     loadbalancer.SVCTypeClusterIP,
  1876  			Frontend: *clusterIP3,
  1877  			Backends: []*loadbalancer.Backend{
  1878  				{
  1879  					FEPortName: "port-tcp-81",
  1880  					L3n4Addr: loadbalancer.L3n4Addr{
  1881  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
  1882  						L4Addr: loadbalancer.L4Addr{
  1883  							Protocol: loadbalancer.TCP,
  1884  							Port:     8081,
  1885  						},
  1886  					},
  1887  					Weight: loadbalancer.DefaultBackendWeight,
  1888  				},
  1889  				{
  1890  					FEPortName: "port-tcp-81",
  1891  					L3n4Addr: loadbalancer.L3n4Addr{
  1892  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.3"),
  1893  						L4Addr: loadbalancer.L4Addr{
  1894  							Protocol: loadbalancer.TCP,
  1895  							Port:     8081,
  1896  						},
  1897  					},
  1898  					Weight: loadbalancer.DefaultBackendWeight,
  1899  				},
  1900  			},
  1901  		},
  1902  	}
  1903  
  1904  	for _, externalIP := range []*loadbalancer.L3n4AddrID{externalIP1, externalIP4} {
  1905  		upsert2ndWanted[externalIP.Hash()] = loadbalancer.SVC{
  1906  			Type:     loadbalancer.SVCTypeExternalIPs,
  1907  			Frontend: *externalIP,
  1908  			Backends: []*loadbalancer.Backend{
  1909  				{
  1910  					FEPortName: "port-udp-80",
  1911  					L3n4Addr: loadbalancer.L3n4Addr{
  1912  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
  1913  						L4Addr: loadbalancer.L4Addr{
  1914  							Protocol: loadbalancer.UDP,
  1915  							Port:     8080,
  1916  						},
  1917  					},
  1918  					Weight: loadbalancer.DefaultBackendWeight,
  1919  				},
  1920  				{
  1921  					FEPortName: "port-udp-80",
  1922  					L3n4Addr: loadbalancer.L3n4Addr{
  1923  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.3"),
  1924  						L4Addr: loadbalancer.L4Addr{
  1925  							Protocol: loadbalancer.UDP,
  1926  							Port:     8080,
  1927  						},
  1928  					},
  1929  					Weight: loadbalancer.DefaultBackendWeight,
  1930  				},
  1931  			},
  1932  		}
  1933  	}
  1934  	// for _, externalIP := range []*loadbalancer.L3n4AddrID{externalIP2, externalIP5} {
  1935  	// 	upsert2ndWanted[externalIP.Hash()] = loadbalancer.SVC{
  1936  	// 		Type:     loadbalancer.SVCTypeExternalIPs,
  1937  	// 		Frontend: *externalIP,
  1938  	// 		Backends: []*loadbalancer.Backend{
  1939  	// 			{
  1940  	// 				L3n4Addr: loadbalancer.L3n4Addr{
  1941  	// 					IP: net.ParseIP("10.0.0.2"),
  1942  	// 					L4Addr: loadbalancer.L4Addr{
  1943  	// 						Protocol: loadbalancer.UDP,
  1944  	// 						Port:     8080,
  1945  	// 					},
  1946  	// 				},
  1947  	// 			},
  1948  	// 			{
  1949  	// 				L3n4Addr: loadbalancer.L3n4Addr{
  1950  	// 					IP: net.ParseIP("10.0.0.3"),
  1951  	// 					L4Addr: loadbalancer.L4Addr{
  1952  	// 						Protocol: loadbalancer.TCP,
  1953  	// 						Port:     8081,
  1954  	// 					},
  1955  	// 				},
  1956  	// 			},
  1957  	// 		},
  1958  	// 	}
  1959  	// }
  1960  	for _, externalIP := range []*loadbalancer.L3n4AddrID{externalIP3, externalIP6} {
  1961  		upsert2ndWanted[externalIP.Hash()] = loadbalancer.SVC{
  1962  			Type:     loadbalancer.SVCTypeExternalIPs,
  1963  			Frontend: *externalIP,
  1964  			Backends: []*loadbalancer.Backend{
  1965  				{
  1966  					FEPortName: "port-tcp-81",
  1967  					L3n4Addr: loadbalancer.L3n4Addr{
  1968  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
  1969  						L4Addr: loadbalancer.L4Addr{
  1970  							Protocol: loadbalancer.TCP,
  1971  							Port:     8081,
  1972  						},
  1973  					},
  1974  					Weight: loadbalancer.DefaultBackendWeight,
  1975  				},
  1976  				{
  1977  					FEPortName: "port-tcp-81",
  1978  					L3n4Addr: loadbalancer.L3n4Addr{
  1979  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.3"),
  1980  						L4Addr: loadbalancer.L4Addr{
  1981  							Protocol: loadbalancer.TCP,
  1982  							Port:     8081,
  1983  						},
  1984  					},
  1985  					Weight: loadbalancer.DefaultBackendWeight,
  1986  				},
  1987  			},
  1988  		}
  1989  	}
  1990  
  1991  	for _, nodePort := range nodePortIPs1 {
  1992  		upsert2ndWanted[nodePort.Hash()] = loadbalancer.SVC{
  1993  			Type:     loadbalancer.SVCTypeNodePort,
  1994  			Frontend: *nodePort,
  1995  			Backends: []*loadbalancer.Backend{
  1996  				{
  1997  					FEPortName: "port-udp-80",
  1998  					L3n4Addr: loadbalancer.L3n4Addr{
  1999  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
  2000  						L4Addr: loadbalancer.L4Addr{
  2001  							Protocol: loadbalancer.UDP,
  2002  							Port:     8080,
  2003  						},
  2004  					},
  2005  					Weight: loadbalancer.DefaultBackendWeight,
  2006  				},
  2007  				{
  2008  					FEPortName: "port-udp-80",
  2009  					L3n4Addr: loadbalancer.L3n4Addr{
  2010  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.3"),
  2011  						L4Addr: loadbalancer.L4Addr{
  2012  							Protocol: loadbalancer.UDP,
  2013  							Port:     8080,
  2014  						},
  2015  					},
  2016  					Weight: loadbalancer.DefaultBackendWeight,
  2017  				},
  2018  			},
  2019  		}
  2020  	}
  2021  	// for _, nodePort := range nodePortIPs2 {
  2022  	// 	upsert2ndWanted[nodePort.Hash()] = loadbalancer.SVC{
  2023  	// 		Type:     loadbalancer.SVCTypeNodePort,
  2024  	// 		Frontend: *nodePort,
  2025  	// 		Backends: []*loadbalancer.Backend{
  2026  	// 			{
  2027  	// 				L3n4Addr: loadbalancer.L3n4Addr{
  2028  	// 					IP: net.ParseIP("10.0.0.2"),
  2029  	// 					L4Addr: loadbalancer.L4Addr{
  2030  	// 						Protocol: loadbalancer.TCP,
  2031  	// 						Port:     8081,
  2032  	// 					},
  2033  	// 				},
  2034  	// 			},
  2035  	// 			{
  2036  	// 				L3n4Addr: loadbalancer.L3n4Addr{
  2037  	// 					IP: net.ParseIP("10.0.0.3"),
  2038  	// 					L4Addr: loadbalancer.L4Addr{
  2039  	// 						Protocol: loadbalancer.TCP,
  2040  	// 						Port:     8081,
  2041  	// 					},
  2042  	// 				},
  2043  	// 			},
  2044  	// 		},
  2045  	// 	}
  2046  	// }
  2047  	for _, nodePort := range nodePortIPs3 {
  2048  		upsert2ndWanted[nodePort.Hash()] = loadbalancer.SVC{
  2049  			Type:     loadbalancer.SVCTypeNodePort,
  2050  			Frontend: *nodePort,
  2051  			Backends: []*loadbalancer.Backend{
  2052  				{
  2053  					FEPortName: "port-tcp-81",
  2054  					L3n4Addr: loadbalancer.L3n4Addr{
  2055  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
  2056  						L4Addr: loadbalancer.L4Addr{
  2057  							Protocol: loadbalancer.TCP,
  2058  							Port:     8081,
  2059  						},
  2060  					},
  2061  					Weight: loadbalancer.DefaultBackendWeight,
  2062  				},
  2063  				{
  2064  					FEPortName: "port-tcp-81",
  2065  					L3n4Addr: loadbalancer.L3n4Addr{
  2066  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.3"),
  2067  						L4Addr: loadbalancer.L4Addr{
  2068  							Protocol: loadbalancer.TCP,
  2069  							Port:     8081,
  2070  						},
  2071  					},
  2072  					Weight: loadbalancer.DefaultBackendWeight,
  2073  				},
  2074  			},
  2075  		}
  2076  	}
  2077  
  2078  	upsert3rdWanted := map[string]loadbalancer.SVC{
  2079  		clusterIP1.Hash(): {
  2080  			Type:     loadbalancer.SVCTypeClusterIP,
  2081  			Frontend: *clusterIP1,
  2082  			Backends: []*loadbalancer.Backend{
  2083  				{
  2084  					FEPortName: "port-udp-80",
  2085  					L3n4Addr: loadbalancer.L3n4Addr{
  2086  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
  2087  						L4Addr: loadbalancer.L4Addr{
  2088  							Protocol: loadbalancer.UDP,
  2089  							Port:     8080,
  2090  						},
  2091  					},
  2092  					Weight: loadbalancer.DefaultBackendWeight,
  2093  				},
  2094  				{
  2095  					FEPortName: "port-udp-80",
  2096  					L3n4Addr: loadbalancer.L3n4Addr{
  2097  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.3"),
  2098  						L4Addr: loadbalancer.L4Addr{
  2099  							Protocol: loadbalancer.UDP,
  2100  							Port:     8080,
  2101  						},
  2102  					},
  2103  					Weight: loadbalancer.DefaultBackendWeight,
  2104  				},
  2105  			},
  2106  		},
  2107  		// FIXME: We don't distinguish about the protocol being used
  2108  		//        so we can't tell if a UDP/80 maps to port 8080/udp
  2109  		//        or if TCP/80 maps to port 8081/TCP
  2110  		// clusterIP2.Hash(): {
  2111  		// 	Type:     loadbalancer.SVCTypeClusterIP,
  2112  		// 	Frontend: *clusterIP2,
  2113  		// 	Backends: []*loadbalancer.Backend{
  2114  		// 		{
  2115  		// 			L3n4Addr: loadbalancer.L3n4Addr{
  2116  		// 				IP: net.ParseIP("10.0.0.2"),
  2117  		// 				L4Addr: loadbalancer.L4Addr{
  2118  		// 					Protocol: loadbalancer.TCP,
  2119  		// 					Port:     8081,
  2120  		// 				},
  2121  		// 			},
  2122  		// 		},
  2123  		// 		{
  2124  		// 			L3n4Addr: loadbalancer.L3n4Addr{
  2125  		// 				IP: net.ParseIP("10.0.0.3"),
  2126  		// 				L4Addr: loadbalancer.L4Addr{
  2127  		// 					Protocol: loadbalancer.TCP,
  2128  		// 					Port:     8081,
  2129  		// 				},
  2130  		// 			},
  2131  		// 		},
  2132  		// 	},
  2133  		// },
  2134  		clusterIP3.Hash(): {
  2135  			Type:     loadbalancer.SVCTypeClusterIP,
  2136  			Frontend: *clusterIP3,
  2137  			Backends: []*loadbalancer.Backend{
  2138  				{
  2139  					FEPortName: "port-tcp-81",
  2140  					L3n4Addr: loadbalancer.L3n4Addr{
  2141  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
  2142  						L4Addr: loadbalancer.L4Addr{
  2143  							Protocol: loadbalancer.TCP,
  2144  							Port:     8081,
  2145  						},
  2146  					},
  2147  					Weight: loadbalancer.DefaultBackendWeight,
  2148  				},
  2149  				{
  2150  					FEPortName: "port-tcp-81",
  2151  					L3n4Addr: loadbalancer.L3n4Addr{
  2152  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.3"),
  2153  						L4Addr: loadbalancer.L4Addr{
  2154  							Protocol: loadbalancer.TCP,
  2155  							Port:     8081,
  2156  						},
  2157  					},
  2158  					Weight: loadbalancer.DefaultBackendWeight,
  2159  				},
  2160  			},
  2161  		},
  2162  	}
  2163  
  2164  	for _, externalIP := range []*loadbalancer.L3n4AddrID{externalIP1} {
  2165  		upsert3rdWanted[externalIP.Hash()] = loadbalancer.SVC{
  2166  			Type:     loadbalancer.SVCTypeExternalIPs,
  2167  			Frontend: *externalIP,
  2168  			Backends: []*loadbalancer.Backend{
  2169  				{
  2170  					FEPortName: "port-udp-80",
  2171  					L3n4Addr: loadbalancer.L3n4Addr{
  2172  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
  2173  						L4Addr: loadbalancer.L4Addr{
  2174  							Protocol: loadbalancer.UDP,
  2175  							Port:     8080,
  2176  						},
  2177  					},
  2178  					Weight: loadbalancer.DefaultBackendWeight,
  2179  				},
  2180  				{
  2181  					FEPortName: "port-udp-80",
  2182  					L3n4Addr: loadbalancer.L3n4Addr{
  2183  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.3"),
  2184  						L4Addr: loadbalancer.L4Addr{
  2185  							Protocol: loadbalancer.UDP,
  2186  							Port:     8080,
  2187  						},
  2188  					},
  2189  					Weight: loadbalancer.DefaultBackendWeight,
  2190  				},
  2191  			},
  2192  		}
  2193  	}
  2194  	// for _, externalIP := range []*loadbalancer.L3n4AddrID{externalIP2} {
  2195  	// 	upsert3rdWanted[externalIP.Hash()] = loadbalancer.SVC{
  2196  	// 		Type:     loadbalancer.SVCTypeExternalIPs,
  2197  	// 		Frontend: *externalIP,
  2198  	// 		Backends: []*loadbalancer.Backend{
  2199  	// 			{
  2200  	// 				L3n4Addr: loadbalancer.L3n4Addr{
  2201  	// 					IP: net.ParseIP("10.0.0.2"),
  2202  	// 					L4Addr: loadbalancer.L4Addr{
  2203  	// 						Protocol: loadbalancer.TCP,
  2204  	// 						Port:     8081,
  2205  	// 					},
  2206  	// 				},
  2207  	// 			},
  2208  	// 			{
  2209  	// 				L3n4Addr: loadbalancer.L3n4Addr{
  2210  	// 					IP: net.ParseIP("10.0.0.3"),
  2211  	// 					L4Addr: loadbalancer.L4Addr{
  2212  	// 						Protocol: loadbalancer.TCP,
  2213  	// 						Port:     8081,
  2214  	// 					},
  2215  	// 				},
  2216  	// 			},
  2217  	// 		},
  2218  	// 	}
  2219  	// }
  2220  	for _, externalIP := range []*loadbalancer.L3n4AddrID{externalIP3} {
  2221  		upsert3rdWanted[externalIP.Hash()] = loadbalancer.SVC{
  2222  			Type:     loadbalancer.SVCTypeExternalIPs,
  2223  			Frontend: *externalIP,
  2224  			Backends: []*loadbalancer.Backend{
  2225  				{
  2226  					FEPortName: "port-tcp-81",
  2227  					L3n4Addr: loadbalancer.L3n4Addr{
  2228  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
  2229  						L4Addr: loadbalancer.L4Addr{
  2230  							Protocol: loadbalancer.TCP,
  2231  							Port:     8081,
  2232  						},
  2233  					},
  2234  					Weight: loadbalancer.DefaultBackendWeight,
  2235  				},
  2236  				{
  2237  					FEPortName: "port-tcp-81",
  2238  					L3n4Addr: loadbalancer.L3n4Addr{
  2239  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.3"),
  2240  						L4Addr: loadbalancer.L4Addr{
  2241  							Protocol: loadbalancer.TCP,
  2242  							Port:     8081,
  2243  						},
  2244  					},
  2245  					Weight: loadbalancer.DefaultBackendWeight,
  2246  				},
  2247  			},
  2248  		}
  2249  	}
  2250  
  2251  	for _, nodePort := range nodePortIPs1 {
  2252  		upsert3rdWanted[nodePort.Hash()] = loadbalancer.SVC{
  2253  			Type:     loadbalancer.SVCTypeNodePort,
  2254  			Frontend: *nodePort,
  2255  			Backends: []*loadbalancer.Backend{
  2256  				{
  2257  					FEPortName: "port-udp-80",
  2258  					L3n4Addr: loadbalancer.L3n4Addr{
  2259  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
  2260  						L4Addr: loadbalancer.L4Addr{
  2261  							Protocol: loadbalancer.UDP,
  2262  							Port:     8080,
  2263  						},
  2264  					},
  2265  					Weight: loadbalancer.DefaultBackendWeight,
  2266  				},
  2267  				{
  2268  					FEPortName: "port-udp-80",
  2269  					L3n4Addr: loadbalancer.L3n4Addr{
  2270  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.3"),
  2271  						L4Addr: loadbalancer.L4Addr{
  2272  							Protocol: loadbalancer.UDP,
  2273  							Port:     8080,
  2274  						},
  2275  					},
  2276  					Weight: loadbalancer.DefaultBackendWeight,
  2277  				},
  2278  			},
  2279  		}
  2280  	}
  2281  	// for _, nodePort := range nodePortIPs2 {
  2282  	// 	upsert3rdWanted[nodePort.Hash()] = loadbalancer.SVC{
  2283  	// 		Type:     loadbalancer.SVCTypeNodePort,
  2284  	// 		Frontend: *nodePort,
  2285  	// 		Backends: []*loadbalancer.Backend{
  2286  	// 			{
  2287  	// 				L3n4Addr: loadbalancer.L3n4Addr{
  2288  	// 					IP: net.ParseIP("10.0.0.2"),
  2289  	// 					L4Addr: loadbalancer.L4Addr{
  2290  	// 						Protocol: loadbalancer.TCP,
  2291  	// 						Port:     8081,
  2292  	// 					},
  2293  	// 				},
  2294  	// 			},
  2295  	// 			{
  2296  	// 				L3n4Addr: loadbalancer.L3n4Addr{
  2297  	// 					IP: net.ParseIP("10.0.0.3"),
  2298  	// 					L4Addr: loadbalancer.L4Addr{
  2299  	// 						Protocol: loadbalancer.TCP,
  2300  	// 						Port:     8081,
  2301  	// 					},
  2302  	// 				},
  2303  	// 			},
  2304  	// 		},
  2305  	// 	}
  2306  	// }
  2307  	for _, nodePort := range nodePortIPs3 {
  2308  		upsert3rdWanted[nodePort.Hash()] = loadbalancer.SVC{
  2309  			Type:     loadbalancer.SVCTypeNodePort,
  2310  			Frontend: *nodePort,
  2311  			Backends: []*loadbalancer.Backend{
  2312  				{
  2313  					FEPortName: "port-tcp-81",
  2314  					L3n4Addr: loadbalancer.L3n4Addr{
  2315  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.2"),
  2316  						L4Addr: loadbalancer.L4Addr{
  2317  							Protocol: loadbalancer.TCP,
  2318  							Port:     8081,
  2319  						},
  2320  					},
  2321  					Weight: loadbalancer.DefaultBackendWeight,
  2322  				},
  2323  				{
  2324  					FEPortName: "port-tcp-81",
  2325  					L3n4Addr: loadbalancer.L3n4Addr{
  2326  						AddrCluster: cmtypes.MustParseAddrCluster("10.0.0.3"),
  2327  						L4Addr: loadbalancer.L4Addr{
  2328  							Protocol: loadbalancer.TCP,
  2329  							Port:     8081,
  2330  						},
  2331  					},
  2332  					Weight: loadbalancer.DefaultBackendWeight,
  2333  				},
  2334  			},
  2335  		}
  2336  	}
  2337  
  2338  	del1stWanted := map[string]struct{}{
  2339  		externalIP4.Hash(): {},
  2340  		// externalIP5.Hash():{},
  2341  		externalIP6.Hash(): {},
  2342  	}
  2343  	del2ndWanted := map[string]struct{}{
  2344  		clusterIP1.Hash(): {},
  2345  		// clusterIP2.Hash(): {},
  2346  		clusterIP3.Hash():  {},
  2347  		externalIP1.Hash(): {},
  2348  		// externalIP2.Hash():{},
  2349  		externalIP3.Hash(): {},
  2350  	}
  2351  	for _, nodePort := range append(nodePortIPs1, nodePortIPs3...) {
  2352  		del2ndWanted[nodePort.Hash()] = struct{}{}
  2353  	}
  2354  
  2355  	upsert1st := map[string]loadbalancer.SVC{}
  2356  	upsert2nd := map[string]loadbalancer.SVC{}
  2357  	upsert3rd := map[string]loadbalancer.SVC{}
  2358  	del1st := map[string]struct{}{}
  2359  	del2nd := map[string]struct{}{}
  2360  
  2361  	svcUpsertManagerCalls, svcDeleteManagerCalls := 0, 0
  2362  
  2363  	svcManager := &fakeSvcManager{
  2364  		OnUpsertService: func(p *loadbalancer.SVC) (bool, loadbalancer.ID, error) {
  2365  			sort.Slice(p.Backends, func(i, j int) bool {
  2366  				return p.Backends[i].AddrCluster.Less(p.Backends[j].AddrCluster)
  2367  			})
  2368  			switch {
  2369  			// 1st update endpoints
  2370  			case svcUpsertManagerCalls < len(upsert1stWanted):
  2371  				upsert1st[p.Frontend.Hash()] = loadbalancer.SVC{
  2372  					Frontend: p.Frontend,
  2373  					Backends: p.Backends,
  2374  					Type:     p.Type,
  2375  				}
  2376  			// 2nd update endpoints
  2377  			case svcUpsertManagerCalls < len(upsert1stWanted)+len(upsert2ndWanted):
  2378  				upsert2nd[p.Frontend.Hash()] = loadbalancer.SVC{
  2379  					Frontend: p.Frontend,
  2380  					Backends: p.Backends,
  2381  					Type:     p.Type,
  2382  				}
  2383  			// 3rd update services
  2384  			case svcUpsertManagerCalls < len(upsert1stWanted)+len(upsert2ndWanted)+len(upsert3rdWanted):
  2385  				upsert3rd[p.Frontend.Hash()] = loadbalancer.SVC{
  2386  					Frontend: p.Frontend,
  2387  					Backends: p.Backends,
  2388  					Type:     p.Type,
  2389  				}
  2390  			}
  2391  			svcUpsertManagerCalls++
  2392  			return false, 0, nil
  2393  		},
  2394  		OnDeleteService: func(fe loadbalancer.L3n4Addr) (b bool, e error) {
  2395  			switch {
  2396  			// 1st update endpoints
  2397  			case svcDeleteManagerCalls < len(del1stWanted):
  2398  				del1st[fe.Hash()] = struct{}{}
  2399  			// 2nd update endpoints
  2400  			case svcDeleteManagerCalls < len(del1stWanted)+len(del2ndWanted):
  2401  				del2nd[fe.Hash()] = struct{}{}
  2402  			}
  2403  			svcDeleteManagerCalls++
  2404  			return true, nil
  2405  		},
  2406  	}
  2407  
  2408  	db, nodeAddrs := newDB(t)
  2409  	k8sSvcCache := k8s.NewServiceCache(db, nodeAddrs)
  2410  	svcWatcher := &K8sServiceWatcher{
  2411  		k8sSvcCache: k8sSvcCache,
  2412  		svcManager:  svcManager,
  2413  	}
  2414  
  2415  	go svcWatcher.k8sServiceHandler()
  2416  	swg := lock.NewStoppableWaitGroup()
  2417  
  2418  	k8sSvcCache.UpdateService(svc1stApply, swg)
  2419  	k8sSvcCache.UpdateEndpoints(k8s.ParseEndpoints(ep1stApply), swg)
  2420  	// Running a 2nd update should also trigger a new upsert service
  2421  	k8sSvcCache.UpdateEndpoints(k8s.ParseEndpoints(ep2ndApply), swg)
  2422  	// Running a 3rd update should also not trigger anything because the
  2423  	// endpoints are the same
  2424  	k8sSvcCache.UpdateEndpoints(k8s.ParseEndpoints(ep2ndApply), swg)
  2425  
  2426  	k8sSvcCache.UpdateService(svc2ndApply, swg)
  2427  
  2428  	k8sSvcCache.DeleteService(svc1stApply, swg)
  2429  
  2430  	swg.Stop()
  2431  	swg.Wait()
  2432  	require.Equal(t, len(upsert1stWanted)+len(upsert2ndWanted)+len(upsert3rdWanted), svcUpsertManagerCalls)
  2433  	require.Equal(t, len(del1stWanted)+len(del2ndWanted), svcDeleteManagerCalls)
  2434  
  2435  	require.EqualValues(t, upsert1stWanted, upsert1st)
  2436  	require.EqualValues(t, upsert2ndWanted, upsert2nd)
  2437  	require.EqualValues(t, upsert3rdWanted, upsert3rd)
  2438  	require.EqualValues(t, del1stWanted, del1st)
  2439  	require.EqualValues(t, del2ndWanted, del2nd)
  2440  }