k8s.io/apiserver@v0.31.1/pkg/reconcilers/peer_endpoint_lease_test.go (about)

     1  /*
     2  Copyright 2017 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package reconcilers
    18  
    19  import (
    20  	"reflect"
    21  	"sort"
    22  	"testing"
    23  	"time"
    24  
    25  	"github.com/google/uuid"
    26  	corev1 "k8s.io/api/core/v1"
    27  	"k8s.io/apimachinery/pkg/api/apitesting"
    28  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    29  	"k8s.io/apimachinery/pkg/runtime"
    30  	"k8s.io/apimachinery/pkg/runtime/schema"
    31  	"k8s.io/apimachinery/pkg/runtime/serializer"
    32  	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
    33  	"k8s.io/apiserver/pkg/features"
    34  	"k8s.io/apiserver/pkg/storage"
    35  	etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing"
    36  	"k8s.io/apiserver/pkg/storage/storagebackend/factory"
    37  	utilfeature "k8s.io/apiserver/pkg/util/feature"
    38  	featuregatetesting "k8s.io/component-base/featuregate/testing"
    39  )
    40  
    41  func init() {
    42  	var scheme = runtime.NewScheme()
    43  
    44  	metav1.AddToGroupVersion(scheme, metav1.SchemeGroupVersion)
    45  	utilruntime.Must(corev1.AddToScheme(scheme))
    46  	utilruntime.Must(scheme.SetVersionPriority(corev1.SchemeGroupVersion))
    47  
    48  	codecs = serializer.NewCodecFactory(scheme)
    49  }
    50  
    51  var codecs serializer.CodecFactory
    52  
    53  type serverInfo struct {
    54  	existingIP     string
    55  	id             string
    56  	ports          []corev1.EndpointPort
    57  	newIP          string
    58  	removeLease    bool
    59  	expectEndpoint string
    60  }
    61  
    62  func NewFakePeerEndpointReconciler(t *testing.T, s storage.Interface) peerEndpointLeaseReconciler {
    63  	// use the same base key used by the controlplane, but add a random
    64  	// prefix so we can reuse the etcd instance for subtests independently.
    65  	base := "/" + uuid.New().String() + "/peerserverleases/"
    66  	return peerEndpointLeaseReconciler{serverLeases: &peerEndpointLeases{
    67  		storage:   s,
    68  		destroyFn: func() {},
    69  		baseKey:   base,
    70  		leaseTime: 1 * time.Minute, // avoid the lease to timeout on tests
    71  	}}
    72  }
    73  
    74  func (f *peerEndpointLeaseReconciler) SetKeys(servers []serverInfo) error {
    75  	for _, server := range servers {
    76  		if err := f.UpdateLease(server.id, server.existingIP, server.ports); err != nil {
    77  			return err
    78  		}
    79  	}
    80  	return nil
    81  }
    82  
    83  func TestPeerEndpointLeaseReconciler(t *testing.T) {
    84  	// enable feature flags
    85  	featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.APIServerIdentity, true)
    86  	featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StorageVersionAPI, true)
    87  
    88  	server, sc := etcd3testing.NewUnsecuredEtcd3TestClientServer(t)
    89  	t.Cleanup(func() { server.Terminate(t) })
    90  
    91  	newFunc := func() runtime.Object { return &corev1.Endpoints{} }
    92  	newListFunc := func() runtime.Object { return &corev1.EndpointsList{} }
    93  	sc.Codec = apitesting.TestStorageCodec(codecs, corev1.SchemeGroupVersion)
    94  
    95  	s, dFunc, err := factory.Create(*sc.ForResource(schema.GroupResource{Resource: "endpoints"}), newFunc, newListFunc, "")
    96  	if err != nil {
    97  		t.Fatalf("Error creating storage: %v", err)
    98  	}
    99  	t.Cleanup(dFunc)
   100  
   101  	tests := []struct {
   102  		testName     string
   103  		servers      []serverInfo
   104  		expectLeases []string
   105  	}{
   106  		{
   107  			testName: "existing IP satisfy",
   108  			servers: []serverInfo{{
   109  				existingIP:     "4.3.2.1",
   110  				id:             "server-1",
   111  				ports:          []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}},
   112  				expectEndpoint: "4.3.2.1:8080",
   113  			}, {
   114  				existingIP:     "1.2.3.4",
   115  				id:             "server-2",
   116  				ports:          []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}},
   117  				expectEndpoint: "1.2.3.4:8080",
   118  			}},
   119  			expectLeases: []string{"4.3.2.1", "1.2.3.4"},
   120  		},
   121  		{
   122  			testName: "existing IP + new IP = should return the new IP",
   123  			servers: []serverInfo{{
   124  				existingIP:     "4.3.2.2",
   125  				id:             "server-1",
   126  				ports:          []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}},
   127  				newIP:          "4.3.2.1",
   128  				expectEndpoint: "4.3.2.1:8080",
   129  			}, {
   130  				existingIP:     "1.2.3.4",
   131  				id:             "server-2",
   132  				ports:          []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}},
   133  				newIP:          "1.1.1.1",
   134  				expectEndpoint: "1.1.1.1:8080",
   135  			}},
   136  			expectLeases: []string{"4.3.2.1", "1.1.1.1"},
   137  		},
   138  		{
   139  			testName: "no existing IP, should return new IP",
   140  			servers: []serverInfo{{
   141  				id:             "server-1",
   142  				ports:          []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}},
   143  				newIP:          "1.2.3.4",
   144  				expectEndpoint: "1.2.3.4:8080",
   145  			}},
   146  			expectLeases: []string{"1.2.3.4"},
   147  		},
   148  	}
   149  	for _, test := range tests {
   150  		t.Run(test.testName, func(t *testing.T) {
   151  			fakeReconciler := NewFakePeerEndpointReconciler(t, s)
   152  			err := fakeReconciler.SetKeys(test.servers)
   153  			if err != nil {
   154  				t.Errorf("unexpected error creating keys: %v", err)
   155  			}
   156  
   157  			for _, server := range test.servers {
   158  				if server.newIP != "" {
   159  					err = fakeReconciler.UpdateLease(server.id, server.newIP, server.ports)
   160  					if err != nil {
   161  						t.Errorf("unexpected error reconciling: %v", err)
   162  					}
   163  				}
   164  			}
   165  
   166  			leases, err := fakeReconciler.ListLeases()
   167  			if err != nil {
   168  				t.Errorf("unexpected error: %v", err)
   169  			}
   170  			// sort for comparison
   171  			sort.Strings(leases)
   172  			sort.Strings(test.expectLeases)
   173  			if !reflect.DeepEqual(leases, test.expectLeases) {
   174  				t.Errorf("expected %v got: %v", test.expectLeases, leases)
   175  			}
   176  
   177  			for _, server := range test.servers {
   178  				endpoint, err := fakeReconciler.GetLease(server.id)
   179  				if err != nil {
   180  					t.Errorf("unexpected error: %v", err)
   181  				}
   182  				if endpoint != server.expectEndpoint {
   183  					t.Errorf("expected %v got: %v", server.expectEndpoint, endpoint)
   184  				}
   185  			}
   186  		})
   187  	}
   188  }
   189  
   190  func TestPeerLeaseRemoveEndpoints(t *testing.T) {
   191  	// enable feature flags
   192  	featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.APIServerIdentity, true)
   193  	featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StorageVersionAPI, true)
   194  
   195  	server, sc := etcd3testing.NewUnsecuredEtcd3TestClientServer(t)
   196  	t.Cleanup(func() { server.Terminate(t) })
   197  
   198  	newFunc := func() runtime.Object { return &corev1.Endpoints{} }
   199  	newListFunc := func() runtime.Object { return &corev1.EndpointsList{} }
   200  	sc.Codec = apitesting.TestStorageCodec(codecs, corev1.SchemeGroupVersion)
   201  
   202  	s, dFunc, err := factory.Create(*sc.ForResource(schema.GroupResource{Resource: "pods"}), newFunc, newListFunc, "")
   203  	if err != nil {
   204  		t.Fatalf("Error creating storage: %v", err)
   205  	}
   206  	t.Cleanup(dFunc)
   207  
   208  	stopTests := []struct {
   209  		testName         string
   210  		servers          []serverInfo
   211  		expectLeases     []string
   212  		apiServerStartup bool
   213  	}{
   214  		{
   215  			testName: "successful remove previous endpoints before apiserver starts",
   216  			servers: []serverInfo{
   217  				{
   218  					existingIP:  "1.2.3.4",
   219  					id:          "test-server-1",
   220  					ports:       []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}},
   221  					removeLease: true,
   222  				},
   223  				{
   224  					existingIP: "2.4.6.8",
   225  					id:         "test-server-2",
   226  					ports:      []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}},
   227  				}},
   228  			expectLeases:     []string{"2.4.6.8"},
   229  			apiServerStartup: true,
   230  		},
   231  		{
   232  			testName: "stop reconciling with new IP not in existing ip list",
   233  			servers: []serverInfo{{
   234  				existingIP: "1.2.3.4",
   235  				newIP:      "4.6.8.9",
   236  				id:         "test-server-1",
   237  				ports:      []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}},
   238  			},
   239  				{
   240  					existingIP:  "2.4.6.8",
   241  					id:          "test-server-2",
   242  					ports:       []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}},
   243  					removeLease: true,
   244  				}},
   245  			expectLeases: []string{"1.2.3.4"},
   246  		},
   247  	}
   248  	for _, test := range stopTests {
   249  		t.Run(test.testName, func(t *testing.T) {
   250  			fakeReconciler := NewFakePeerEndpointReconciler(t, s)
   251  			err := fakeReconciler.SetKeys(test.servers)
   252  			if err != nil {
   253  				t.Errorf("unexpected error creating keys: %v", err)
   254  			}
   255  			if !test.apiServerStartup {
   256  				fakeReconciler.StopReconciling()
   257  			}
   258  			for _, server := range test.servers {
   259  				if server.removeLease {
   260  					err = fakeReconciler.RemoveLease(server.id)
   261  					// if the ip is not on the endpoints, it must return an storage error and stop reconciling
   262  					if err != nil {
   263  						t.Errorf("unexpected error reconciling: %v", err)
   264  					}
   265  				}
   266  			}
   267  
   268  			leases, err := fakeReconciler.ListLeases()
   269  			if err != nil {
   270  				t.Errorf("unexpected error: %v", err)
   271  			}
   272  			// sort for comparison
   273  			sort.Strings(leases)
   274  			sort.Strings(test.expectLeases)
   275  			if !reflect.DeepEqual(leases, test.expectLeases) {
   276  				t.Errorf("expected %v got: %v", test.expectLeases, leases)
   277  			}
   278  		})
   279  	}
   280  }