k8s.io/kubernetes@v1.29.3/pkg/registry/core/service/ipallocator/controller/repair_test.go (about)

     1  /*
     2  Copyright 2015 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package controller
    18  
    19  import (
    20  	"fmt"
    21  	"net"
    22  	"strings"
    23  	"testing"
    24  
    25  	corev1 "k8s.io/api/core/v1"
    26  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    27  	"k8s.io/client-go/kubernetes/fake"
    28  	"k8s.io/component-base/metrics/testutil"
    29  	api "k8s.io/kubernetes/pkg/apis/core"
    30  	"k8s.io/kubernetes/pkg/registry/core/service/ipallocator"
    31  	netutils "k8s.io/utils/net"
    32  )
    33  
    34  type mockRangeRegistry struct {
    35  	getCalled bool
    36  	item      *api.RangeAllocation
    37  	err       error
    38  
    39  	updateCalled bool
    40  	updated      *api.RangeAllocation
    41  	updateErr    error
    42  }
    43  
    44  func (r *mockRangeRegistry) Get() (*api.RangeAllocation, error) {
    45  	r.getCalled = true
    46  	return r.item, r.err
    47  }
    48  
    49  func (r *mockRangeRegistry) CreateOrUpdate(alloc *api.RangeAllocation) error {
    50  	r.updateCalled = true
    51  	r.updated = alloc
    52  	return r.updateErr
    53  }
    54  
    55  func TestRepair(t *testing.T) {
    56  	fakeClient := fake.NewSimpleClientset()
    57  	ipregistry := &mockRangeRegistry{
    58  		item: &api.RangeAllocation{Range: "192.168.1.0/24"},
    59  	}
    60  	_, cidr, _ := netutils.ParseCIDRSloppy(ipregistry.item.Range)
    61  	r := NewRepair(0, fakeClient.CoreV1(), fakeClient.EventsV1(), cidr, ipregistry, nil, nil)
    62  
    63  	if err := r.runOnce(); err != nil {
    64  		t.Fatal(err)
    65  	}
    66  	if !ipregistry.updateCalled || ipregistry.updated == nil || ipregistry.updated.Range != cidr.String() || ipregistry.updated != ipregistry.item {
    67  		t.Errorf("unexpected ipregistry: %#v", ipregistry)
    68  	}
    69  
    70  	ipregistry = &mockRangeRegistry{
    71  		item:      &api.RangeAllocation{Range: "192.168.1.0/24"},
    72  		updateErr: fmt.Errorf("test error"),
    73  	}
    74  	r = NewRepair(0, fakeClient.CoreV1(), fakeClient.EventsV1(), cidr, ipregistry, nil, nil)
    75  	if err := r.runOnce(); !strings.Contains(err.Error(), ": test error") {
    76  		t.Fatal(err)
    77  	}
    78  }
    79  
    80  func TestRepairLeak(t *testing.T) {
    81  	clearMetrics()
    82  
    83  	_, cidr, _ := netutils.ParseCIDRSloppy("192.168.1.0/24")
    84  	previous, err := ipallocator.NewInMemory(cidr)
    85  	if err != nil {
    86  		t.Fatal(err)
    87  	}
    88  	previous.Allocate(netutils.ParseIPSloppy("192.168.1.10"))
    89  
    90  	var dst api.RangeAllocation
    91  	err = previous.Snapshot(&dst)
    92  	if err != nil {
    93  		t.Fatal(err)
    94  	}
    95  
    96  	fakeClient := fake.NewSimpleClientset()
    97  	ipregistry := &mockRangeRegistry{
    98  		item: &api.RangeAllocation{
    99  			ObjectMeta: metav1.ObjectMeta{
   100  				ResourceVersion: "1",
   101  			},
   102  			Range: dst.Range,
   103  			Data:  dst.Data,
   104  		},
   105  	}
   106  
   107  	r := NewRepair(0, fakeClient.CoreV1(), fakeClient.EventsV1(), cidr, ipregistry, nil, nil)
   108  	// Run through the "leak detection holdoff" loops.
   109  	for i := 0; i < (numRepairsBeforeLeakCleanup - 1); i++ {
   110  		if err := r.runOnce(); err != nil {
   111  			t.Fatal(err)
   112  		}
   113  		after, err := ipallocator.NewFromSnapshot(ipregistry.updated)
   114  		if err != nil {
   115  			t.Fatal(err)
   116  		}
   117  		if !after.Has(netutils.ParseIPSloppy("192.168.1.10")) {
   118  			t.Errorf("expected ipallocator to still have leaked IP")
   119  		}
   120  	}
   121  	// Run one more time to actually remove the leak.
   122  	if err := r.runOnce(); err != nil {
   123  		t.Fatal(err)
   124  	}
   125  	after, err := ipallocator.NewFromSnapshot(ipregistry.updated)
   126  	if err != nil {
   127  		t.Fatal(err)
   128  	}
   129  	if after.Has(netutils.ParseIPSloppy("192.168.1.10")) {
   130  		t.Errorf("expected ipallocator to not have leaked IP")
   131  	}
   132  	em := testMetrics{
   133  		leak:       1,
   134  		repair:     0,
   135  		outOfRange: 0,
   136  		duplicate:  0,
   137  		unknown:    0,
   138  		invalid:    0,
   139  		full:       0,
   140  	}
   141  	expectMetrics(t, em)
   142  }
   143  
   144  func TestRepairWithExisting(t *testing.T) {
   145  	clearMetrics()
   146  
   147  	_, cidr, _ := netutils.ParseCIDRSloppy("192.168.1.0/24")
   148  	previous, err := ipallocator.NewInMemory(cidr)
   149  	if err != nil {
   150  		t.Fatal(err)
   151  	}
   152  
   153  	var dst api.RangeAllocation
   154  	err = previous.Snapshot(&dst)
   155  	if err != nil {
   156  		t.Fatal(err)
   157  	}
   158  
   159  	fakeClient := fake.NewSimpleClientset(
   160  		&corev1.Service{
   161  			ObjectMeta: metav1.ObjectMeta{Namespace: "one", Name: "one"},
   162  			Spec: corev1.ServiceSpec{
   163  				ClusterIP:  "192.168.1.1",
   164  				ClusterIPs: []string{"192.168.1.1"},
   165  				IPFamilies: []corev1.IPFamily{corev1.IPv4Protocol},
   166  			},
   167  		},
   168  		&corev1.Service{
   169  			ObjectMeta: metav1.ObjectMeta{Namespace: "two", Name: "two"},
   170  			Spec: corev1.ServiceSpec{
   171  				ClusterIP:  "192.168.1.100",
   172  				ClusterIPs: []string{"192.168.1.100"},
   173  				IPFamilies: []corev1.IPFamily{corev1.IPv4Protocol},
   174  			},
   175  		},
   176  		&corev1.Service{ // outside CIDR, will be dropped
   177  			ObjectMeta: metav1.ObjectMeta{Namespace: "three", Name: "three"},
   178  			Spec: corev1.ServiceSpec{
   179  				ClusterIP:  "192.168.0.1",
   180  				ClusterIPs: []string{"192.168.0.1"},
   181  				IPFamilies: []corev1.IPFamily{corev1.IPv4Protocol},
   182  			},
   183  		},
   184  		&corev1.Service{ // empty, ignored
   185  			ObjectMeta: metav1.ObjectMeta{Namespace: "four", Name: "four"},
   186  			Spec: corev1.ServiceSpec{
   187  				ClusterIP:  "",
   188  				ClusterIPs: []string{""},
   189  			},
   190  		},
   191  		&corev1.Service{ // duplicate, dropped
   192  			ObjectMeta: metav1.ObjectMeta{Namespace: "five", Name: "five"},
   193  			Spec: corev1.ServiceSpec{
   194  				ClusterIP:  "192.168.1.1",
   195  				ClusterIPs: []string{"192.168.1.1"},
   196  				IPFamilies: []corev1.IPFamily{corev1.IPv4Protocol},
   197  			},
   198  		},
   199  		&corev1.Service{ // headless
   200  			ObjectMeta: metav1.ObjectMeta{Namespace: "six", Name: "six"},
   201  			Spec: corev1.ServiceSpec{
   202  				ClusterIP:  "None",
   203  				ClusterIPs: []string{"None"},
   204  			},
   205  		},
   206  	)
   207  
   208  	ipregistry := &mockRangeRegistry{
   209  		item: &api.RangeAllocation{
   210  			ObjectMeta: metav1.ObjectMeta{
   211  				ResourceVersion: "1",
   212  			},
   213  			Range: dst.Range,
   214  			Data:  dst.Data,
   215  		},
   216  	}
   217  	r := NewRepair(0, fakeClient.CoreV1(), fakeClient.EventsV1(), cidr, ipregistry, nil, nil)
   218  	if err := r.runOnce(); err != nil {
   219  		t.Fatal(err)
   220  	}
   221  	after, err := ipallocator.NewFromSnapshot(ipregistry.updated)
   222  	if err != nil {
   223  		t.Fatal(err)
   224  	}
   225  	if !after.Has(netutils.ParseIPSloppy("192.168.1.1")) || !after.Has(netutils.ParseIPSloppy("192.168.1.100")) {
   226  		t.Errorf("unexpected ipallocator state: %#v", after)
   227  	}
   228  	if free := after.Free(); free != 252 {
   229  		t.Errorf("unexpected ipallocator state: %d free (expected 252)", free)
   230  	}
   231  	em := testMetrics{
   232  		leak:       0,
   233  		repair:     2,
   234  		outOfRange: 1,
   235  		duplicate:  1,
   236  		unknown:    0,
   237  		invalid:    0,
   238  		full:       0,
   239  	}
   240  	expectMetrics(t, em)
   241  }
   242  
   243  func makeRangeRegistry(t *testing.T, cidrRange string) *mockRangeRegistry {
   244  	_, cidr, _ := netutils.ParseCIDRSloppy(cidrRange)
   245  	previous, err := ipallocator.NewInMemory(cidr)
   246  	if err != nil {
   247  		t.Fatal(err)
   248  	}
   249  
   250  	var dst api.RangeAllocation
   251  	err = previous.Snapshot(&dst)
   252  	if err != nil {
   253  		t.Fatal(err)
   254  	}
   255  
   256  	return &mockRangeRegistry{
   257  		item: &api.RangeAllocation{
   258  			ObjectMeta: metav1.ObjectMeta{
   259  				ResourceVersion: "1",
   260  			},
   261  			Range: dst.Range,
   262  			Data:  dst.Data,
   263  		},
   264  	}
   265  }
   266  
   267  func makeFakeClientSet() *fake.Clientset {
   268  	return fake.NewSimpleClientset()
   269  }
   270  func makeIPNet(cidr string) *net.IPNet {
   271  	_, net, _ := netutils.ParseCIDRSloppy(cidr)
   272  	return net
   273  }
   274  func TestShouldWorkOnSecondary(t *testing.T) {
   275  	testCases := []struct {
   276  		name             string
   277  		expectedFamilies []corev1.IPFamily
   278  		primaryNet       *net.IPNet
   279  		secondaryNet     *net.IPNet
   280  	}{
   281  		{
   282  			name:             "primary only (v4)",
   283  			expectedFamilies: []corev1.IPFamily{corev1.IPv4Protocol},
   284  			primaryNet:       makeIPNet("10.0.0.0/16"),
   285  			secondaryNet:     nil,
   286  		},
   287  		{
   288  			name:             "primary only (v6)",
   289  			expectedFamilies: []corev1.IPFamily{corev1.IPv6Protocol},
   290  			primaryNet:       makeIPNet("2000::/120"),
   291  			secondaryNet:     nil,
   292  		},
   293  		{
   294  			name:             "primary and secondary provided (v4,v6)",
   295  			expectedFamilies: []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol},
   296  			primaryNet:       makeIPNet("10.0.0.0/16"),
   297  			secondaryNet:     makeIPNet("2000::/120"),
   298  		},
   299  		{
   300  			name:             "primary and secondary provided (v6,v4)",
   301  			expectedFamilies: []corev1.IPFamily{corev1.IPv6Protocol, corev1.IPv4Protocol},
   302  			primaryNet:       makeIPNet("2000::/120"),
   303  			secondaryNet:     makeIPNet("10.0.0.0/16"),
   304  		},
   305  	}
   306  	for _, tc := range testCases {
   307  		t.Run(tc.name, func(t *testing.T) {
   308  
   309  			fakeClient := makeFakeClientSet()
   310  			primaryRegistry := makeRangeRegistry(t, tc.primaryNet.String())
   311  			var secondaryRegistry *mockRangeRegistry
   312  
   313  			if tc.secondaryNet != nil {
   314  				secondaryRegistry = makeRangeRegistry(t, tc.secondaryNet.String())
   315  			}
   316  
   317  			repair := NewRepair(0, fakeClient.CoreV1(), fakeClient.EventsV1(), tc.primaryNet, primaryRegistry, tc.secondaryNet, secondaryRegistry)
   318  			if len(repair.allocatorByFamily) != len(tc.expectedFamilies) {
   319  				t.Fatalf("expected to have allocator by family count:%v got %v", len(tc.expectedFamilies), len(repair.allocatorByFamily))
   320  			}
   321  
   322  			seen := make(map[corev1.IPFamily]bool)
   323  			for _, family := range tc.expectedFamilies {
   324  				familySeen := true
   325  
   326  				if _, ok := repair.allocatorByFamily[family]; !ok {
   327  					familySeen = familySeen && ok
   328  				}
   329  
   330  				if _, ok := repair.networkByFamily[family]; !ok {
   331  					familySeen = familySeen && ok
   332  				}
   333  
   334  				if _, ok := repair.leaksByFamily[family]; !ok {
   335  					familySeen = familySeen && ok
   336  				}
   337  
   338  				seen[family] = familySeen
   339  			}
   340  
   341  			for family, seen := range seen {
   342  				if !seen {
   343  					t.Fatalf("expected repair look to have family %v, but it was not visible on either (or all) network, allocator, leaks", family)
   344  				}
   345  			}
   346  		})
   347  	}
   348  }
   349  
   350  func TestRepairDualStack(t *testing.T) {
   351  	clearMetrics()
   352  
   353  	fakeClient := fake.NewSimpleClientset()
   354  	ipregistry := &mockRangeRegistry{
   355  		item: &api.RangeAllocation{Range: "192.168.1.0/24"},
   356  	}
   357  	secondaryIPRegistry := &mockRangeRegistry{
   358  		item: &api.RangeAllocation{Range: "2000::/108"},
   359  	}
   360  
   361  	_, cidr, _ := netutils.ParseCIDRSloppy(ipregistry.item.Range)
   362  	_, secondaryCIDR, _ := netutils.ParseCIDRSloppy(secondaryIPRegistry.item.Range)
   363  	r := NewRepair(0, fakeClient.CoreV1(), fakeClient.EventsV1(), cidr, ipregistry, secondaryCIDR, secondaryIPRegistry)
   364  
   365  	if err := r.runOnce(); err != nil {
   366  		t.Fatal(err)
   367  	}
   368  	if !ipregistry.updateCalled || ipregistry.updated == nil || ipregistry.updated.Range != cidr.String() || ipregistry.updated != ipregistry.item {
   369  		t.Errorf("unexpected ipregistry: %#v", ipregistry)
   370  	}
   371  	if !secondaryIPRegistry.updateCalled || secondaryIPRegistry.updated == nil || secondaryIPRegistry.updated.Range != secondaryCIDR.String() || secondaryIPRegistry.updated != secondaryIPRegistry.item {
   372  		t.Errorf("unexpected ipregistry: %#v", ipregistry)
   373  	}
   374  
   375  	repairErrors, err := testutil.GetCounterMetricValue(clusterIPRepairReconcileErrors)
   376  	if err != nil {
   377  		t.Errorf("failed to get %s value, err: %v", clusterIPRepairReconcileErrors.Name, err)
   378  	}
   379  	if repairErrors != 0 {
   380  		t.Fatalf("0 error expected, got %v", repairErrors)
   381  	}
   382  
   383  	ipregistry = &mockRangeRegistry{
   384  		item:      &api.RangeAllocation{Range: "192.168.1.0/24"},
   385  		updateErr: fmt.Errorf("test error"),
   386  	}
   387  	secondaryIPRegistry = &mockRangeRegistry{
   388  		item:      &api.RangeAllocation{Range: "2000::/108"},
   389  		updateErr: fmt.Errorf("test error"),
   390  	}
   391  
   392  	r = NewRepair(0, fakeClient.CoreV1(), fakeClient.EventsV1(), cidr, ipregistry, secondaryCIDR, secondaryIPRegistry)
   393  	if err := r.runOnce(); !strings.Contains(err.Error(), ": test error") {
   394  		t.Fatal(err)
   395  	}
   396  	repairErrors, err = testutil.GetCounterMetricValue(clusterIPRepairReconcileErrors)
   397  	if err != nil {
   398  		t.Errorf("failed to get %s value, err: %v", clusterIPRepairReconcileErrors.Name, err)
   399  	}
   400  	if repairErrors != 1 {
   401  		t.Fatalf("1 error expected, got %v", repairErrors)
   402  	}
   403  }
   404  
   405  func TestRepairLeakDualStack(t *testing.T) {
   406  	clearMetrics()
   407  	_, cidr, _ := netutils.ParseCIDRSloppy("192.168.1.0/24")
   408  	previous, err := ipallocator.NewInMemory(cidr)
   409  	if err != nil {
   410  		t.Fatal(err)
   411  	}
   412  
   413  	previous.Allocate(netutils.ParseIPSloppy("192.168.1.10"))
   414  
   415  	_, secondaryCIDR, _ := netutils.ParseCIDRSloppy("2000::/108")
   416  	secondaryPrevious, err := ipallocator.NewInMemory(secondaryCIDR)
   417  	if err != nil {
   418  		t.Fatal(err)
   419  	}
   420  	secondaryPrevious.Allocate(netutils.ParseIPSloppy("2000::1"))
   421  
   422  	var dst api.RangeAllocation
   423  	err = previous.Snapshot(&dst)
   424  	if err != nil {
   425  		t.Fatal(err)
   426  	}
   427  
   428  	var secondaryDST api.RangeAllocation
   429  	err = secondaryPrevious.Snapshot(&secondaryDST)
   430  	if err != nil {
   431  		t.Fatal(err)
   432  	}
   433  
   434  	fakeClient := fake.NewSimpleClientset()
   435  
   436  	ipregistry := &mockRangeRegistry{
   437  		item: &api.RangeAllocation{
   438  			ObjectMeta: metav1.ObjectMeta{
   439  				ResourceVersion: "1",
   440  			},
   441  			Range: dst.Range,
   442  			Data:  dst.Data,
   443  		},
   444  	}
   445  	secondaryIPRegistry := &mockRangeRegistry{
   446  		item: &api.RangeAllocation{
   447  			ObjectMeta: metav1.ObjectMeta{
   448  				ResourceVersion: "1",
   449  			},
   450  			Range: secondaryDST.Range,
   451  			Data:  secondaryDST.Data,
   452  		},
   453  	}
   454  
   455  	r := NewRepair(0, fakeClient.CoreV1(), fakeClient.EventsV1(), cidr, ipregistry, secondaryCIDR, secondaryIPRegistry)
   456  	// Run through the "leak detection holdoff" loops.
   457  	for i := 0; i < (numRepairsBeforeLeakCleanup - 1); i++ {
   458  		if err := r.runOnce(); err != nil {
   459  			t.Fatal(err)
   460  		}
   461  		after, err := ipallocator.NewFromSnapshot(ipregistry.updated)
   462  		if err != nil {
   463  			t.Fatal(err)
   464  		}
   465  		if !after.Has(netutils.ParseIPSloppy("192.168.1.10")) {
   466  			t.Errorf("expected ipallocator to still have leaked IP")
   467  		}
   468  		secondaryAfter, err := ipallocator.NewFromSnapshot(secondaryIPRegistry.updated)
   469  		if err != nil {
   470  			t.Fatal(err)
   471  		}
   472  		if !secondaryAfter.Has(netutils.ParseIPSloppy("2000::1")) {
   473  			t.Errorf("expected ipallocator to still have leaked IP")
   474  		}
   475  	}
   476  	// Run one more time to actually remove the leak.
   477  	if err := r.runOnce(); err != nil {
   478  		t.Fatal(err)
   479  	}
   480  
   481  	after, err := ipallocator.NewFromSnapshot(ipregistry.updated)
   482  	if err != nil {
   483  		t.Fatal(err)
   484  	}
   485  	if after.Has(netutils.ParseIPSloppy("192.168.1.10")) {
   486  		t.Errorf("expected ipallocator to not have leaked IP")
   487  	}
   488  	secondaryAfter, err := ipallocator.NewFromSnapshot(secondaryIPRegistry.updated)
   489  	if err != nil {
   490  		t.Fatal(err)
   491  	}
   492  	if secondaryAfter.Has(netutils.ParseIPSloppy("2000::1")) {
   493  		t.Errorf("expected ipallocator to not have leaked IP")
   494  	}
   495  
   496  	em := testMetrics{
   497  		leak:       2,
   498  		repair:     0,
   499  		outOfRange: 0,
   500  		duplicate:  0,
   501  		unknown:    0,
   502  		invalid:    0,
   503  		full:       0,
   504  	}
   505  	expectMetrics(t, em)
   506  
   507  }
   508  
   509  func TestRepairWithExistingDualStack(t *testing.T) {
   510  	clearMetrics()
   511  	// because anything (other than allocator) depends
   512  	// on families assigned to service (not the value of IPFamilyPolicy)
   513  	// we can saftly create tests that has ipFamilyPolicy:nil
   514  	// this will work every where except alloc & validation
   515  
   516  	_, cidr, _ := netutils.ParseCIDRSloppy("192.168.1.0/24")
   517  	previous, err := ipallocator.NewInMemory(cidr)
   518  	if err != nil {
   519  		t.Fatal(err)
   520  	}
   521  
   522  	_, secondaryCIDR, _ := netutils.ParseCIDRSloppy("2000::/108")
   523  	secondaryPrevious, err := ipallocator.NewInMemory(secondaryCIDR)
   524  	if err != nil {
   525  		t.Fatal(err)
   526  	}
   527  
   528  	var dst api.RangeAllocation
   529  	err = previous.Snapshot(&dst)
   530  	if err != nil {
   531  		t.Fatal(err)
   532  	}
   533  
   534  	var secondaryDST api.RangeAllocation
   535  	err = secondaryPrevious.Snapshot(&secondaryDST)
   536  	if err != nil {
   537  		t.Fatal(err)
   538  	}
   539  
   540  	fakeClient := fake.NewSimpleClientset(
   541  		&corev1.Service{
   542  			ObjectMeta: metav1.ObjectMeta{Namespace: "x1", Name: "one-v4-v6"},
   543  			Spec: corev1.ServiceSpec{
   544  				ClusterIP:  "192.168.1.1",
   545  				ClusterIPs: []string{"192.168.1.1", "2000::1"},
   546  				IPFamilies: []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol},
   547  			},
   548  		},
   549  		&corev1.Service{
   550  			ObjectMeta: metav1.ObjectMeta{Namespace: "x2", Name: "one-v6-v4"},
   551  			Spec: corev1.ServiceSpec{
   552  				ClusterIP:  "2000::1",
   553  				ClusterIPs: []string{"2000::1", "192.168.1.100"},
   554  				IPFamilies: []corev1.IPFamily{corev1.IPv6Protocol, corev1.IPv4Protocol},
   555  			},
   556  		},
   557  		&corev1.Service{
   558  			ObjectMeta: metav1.ObjectMeta{Namespace: "x3", Name: "two-6"},
   559  			Spec: corev1.ServiceSpec{
   560  				ClusterIP:  "2000::2",
   561  				ClusterIPs: []string{"2000::2"},
   562  				IPFamilies: []corev1.IPFamily{corev1.IPv6Protocol},
   563  			},
   564  		},
   565  		&corev1.Service{
   566  			ObjectMeta: metav1.ObjectMeta{Namespace: "x4", Name: "two-4"},
   567  			Spec: corev1.ServiceSpec{
   568  				ClusterIP:  "192.168.1.90",
   569  				ClusterIPs: []string{"192.168.1.90"},
   570  				IPFamilies: []corev1.IPFamily{corev1.IPv4Protocol},
   571  			},
   572  		},
   573  		// outside CIDR, will be dropped
   574  		&corev1.Service{
   575  			ObjectMeta: metav1.ObjectMeta{Namespace: "x5", Name: "out-v4"},
   576  			Spec: corev1.ServiceSpec{
   577  				ClusterIP:  "192.168.0.1",
   578  				ClusterIPs: []string{"192.168.0.1"},
   579  				IPFamilies: []corev1.IPFamily{corev1.IPv4Protocol},
   580  			},
   581  		},
   582  		&corev1.Service{ // outside CIDR, will be dropped
   583  			ObjectMeta: metav1.ObjectMeta{Namespace: "x6", Name: "out-v6"},
   584  			Spec: corev1.ServiceSpec{
   585  				ClusterIP:  "3000::1",
   586  				ClusterIPs: []string{"3000::1"},
   587  				IPFamilies: []corev1.IPFamily{corev1.IPv6Protocol},
   588  			},
   589  		},
   590  		&corev1.Service{
   591  			ObjectMeta: metav1.ObjectMeta{Namespace: "x6", Name: "out-v4-v6"},
   592  			Spec: corev1.ServiceSpec{
   593  				ClusterIP:  "192.168.0.1",
   594  				ClusterIPs: []string{"192.168.0.1", "3000::1"},
   595  				IPFamilies: []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol},
   596  			},
   597  		},
   598  		&corev1.Service{
   599  			ObjectMeta: metav1.ObjectMeta{Namespace: "x6", Name: "out-v6-v4"},
   600  			Spec: corev1.ServiceSpec{
   601  				ClusterIP:  "3000::1",
   602  				ClusterIPs: []string{"3000::1", "192.168.0.1"},
   603  				IPFamilies: []corev1.IPFamily{corev1.IPv6Protocol, corev1.IPv4Protocol},
   604  			},
   605  		},
   606  
   607  		&corev1.Service{ // empty, ignored
   608  			ObjectMeta: metav1.ObjectMeta{Namespace: "x7", Name: "out-empty"},
   609  			Spec:       corev1.ServiceSpec{ClusterIP: ""},
   610  		},
   611  		&corev1.Service{ // duplicate, dropped
   612  			ObjectMeta: metav1.ObjectMeta{Namespace: "x8", Name: "duplicate"},
   613  			Spec: corev1.ServiceSpec{
   614  				ClusterIP:  "192.168.1.1",
   615  				ClusterIPs: []string{"192.168.1.1"},
   616  				IPFamilies: []corev1.IPFamily{corev1.IPv4Protocol},
   617  			},
   618  		},
   619  		&corev1.Service{ // duplicate, dropped
   620  			ObjectMeta: metav1.ObjectMeta{Namespace: "x9", Name: "duplicate-v6"},
   621  			Spec: corev1.ServiceSpec{
   622  				ClusterIP:  "2000::2",
   623  				ClusterIPs: []string{"2000::2"},
   624  				IPFamilies: []corev1.IPFamily{corev1.IPv6Protocol},
   625  			},
   626  		},
   627  
   628  		&corev1.Service{ // headless
   629  			ObjectMeta: metav1.ObjectMeta{Namespace: "x10", Name: "headless"},
   630  			Spec:       corev1.ServiceSpec{ClusterIP: "None"},
   631  		},
   632  	)
   633  
   634  	ipregistry := &mockRangeRegistry{
   635  		item: &api.RangeAllocation{
   636  			ObjectMeta: metav1.ObjectMeta{
   637  				ResourceVersion: "1",
   638  			},
   639  			Range: dst.Range,
   640  			Data:  dst.Data,
   641  		},
   642  	}
   643  
   644  	secondaryIPRegistry := &mockRangeRegistry{
   645  		item: &api.RangeAllocation{
   646  			ObjectMeta: metav1.ObjectMeta{
   647  				ResourceVersion: "1",
   648  			},
   649  			Range: secondaryDST.Range,
   650  			Data:  secondaryDST.Data,
   651  		},
   652  	}
   653  
   654  	r := NewRepair(0, fakeClient.CoreV1(), fakeClient.EventsV1(), cidr, ipregistry, secondaryCIDR, secondaryIPRegistry)
   655  	if err := r.runOnce(); err != nil {
   656  		t.Fatal(err)
   657  	}
   658  	after, err := ipallocator.NewFromSnapshot(ipregistry.updated)
   659  	if err != nil {
   660  		t.Fatal(err)
   661  	}
   662  
   663  	if !after.Has(netutils.ParseIPSloppy("192.168.1.1")) || !after.Has(netutils.ParseIPSloppy("192.168.1.100")) {
   664  		t.Errorf("unexpected ipallocator state: %#v", after)
   665  	}
   666  	if free := after.Free(); free != 251 {
   667  		t.Errorf("unexpected ipallocator state: %d free (number of free ips is not 251)", free)
   668  	}
   669  
   670  	secondaryAfter, err := ipallocator.NewFromSnapshot(secondaryIPRegistry.updated)
   671  	if err != nil {
   672  		t.Fatal(err)
   673  	}
   674  	if !secondaryAfter.Has(netutils.ParseIPSloppy("2000::1")) || !secondaryAfter.Has(netutils.ParseIPSloppy("2000::2")) {
   675  		t.Errorf("unexpected ipallocator state: %#v", secondaryAfter)
   676  	}
   677  	if free := secondaryAfter.Free(); free != 65533 {
   678  		t.Errorf("unexpected ipallocator state: %d free (number of free ips is not 65532)", free)
   679  	}
   680  	em := testMetrics{
   681  		leak:       0,
   682  		repair:     5,
   683  		outOfRange: 6,
   684  		duplicate:  3,
   685  		unknown:    0,
   686  		invalid:    0,
   687  		full:       0,
   688  	}
   689  	expectMetrics(t, em)
   690  }
   691  
   692  // Metrics helpers
   693  func clearMetrics() {
   694  	clusterIPRepairIPErrors.Reset()
   695  	clusterIPRepairReconcileErrors.Reset()
   696  }
   697  
   698  type testMetrics struct {
   699  	leak       float64
   700  	repair     float64
   701  	outOfRange float64
   702  	full       float64
   703  	duplicate  float64
   704  	invalid    float64
   705  	unknown    float64
   706  }
   707  
   708  func expectMetrics(t *testing.T, em testMetrics) {
   709  	var m testMetrics
   710  	var err error
   711  
   712  	m.leak, err = testutil.GetCounterMetricValue(clusterIPRepairIPErrors.WithLabelValues("leak"))
   713  	if err != nil {
   714  		t.Errorf("failed to get %s value, err: %v", clusterIPRepairIPErrors.Name, err)
   715  	}
   716  	m.repair, err = testutil.GetCounterMetricValue(clusterIPRepairIPErrors.WithLabelValues("repair"))
   717  	if err != nil {
   718  		t.Errorf("failed to get %s value, err: %v", clusterIPRepairIPErrors.Name, err)
   719  	}
   720  	m.outOfRange, err = testutil.GetCounterMetricValue(clusterIPRepairIPErrors.WithLabelValues("outOfRange"))
   721  	if err != nil {
   722  		t.Errorf("failed to get %s value, err: %v", clusterIPRepairIPErrors.Name, err)
   723  	}
   724  	m.duplicate, err = testutil.GetCounterMetricValue(clusterIPRepairIPErrors.WithLabelValues("duplicate"))
   725  	if err != nil {
   726  		t.Errorf("failed to get %s value, err: %v", clusterIPRepairIPErrors.Name, err)
   727  	}
   728  	m.invalid, err = testutil.GetCounterMetricValue(clusterIPRepairIPErrors.WithLabelValues("invalid"))
   729  	if err != nil {
   730  		t.Errorf("failed to get %s value, err: %v", clusterIPRepairIPErrors.Name, err)
   731  	}
   732  	m.full, err = testutil.GetCounterMetricValue(clusterIPRepairIPErrors.WithLabelValues("full"))
   733  	if err != nil {
   734  		t.Errorf("failed to get %s value, err: %v", clusterIPRepairIPErrors.Name, err)
   735  	}
   736  	m.unknown, err = testutil.GetCounterMetricValue(clusterIPRepairIPErrors.WithLabelValues("unknown"))
   737  	if err != nil {
   738  		t.Errorf("failed to get %s value, err: %v", clusterIPRepairIPErrors.Name, err)
   739  	}
   740  	if m != em {
   741  		t.Fatalf("metrics error: expected %v, received %v", em, m)
   742  	}
   743  }