k8s.io/apiserver@v0.31.1/pkg/reconcilers/peer_endpoint_lease_test.go (about) 1 /* 2 Copyright 2017 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package reconcilers 18 19 import ( 20 "reflect" 21 "sort" 22 "testing" 23 "time" 24 25 "github.com/google/uuid" 26 corev1 "k8s.io/api/core/v1" 27 "k8s.io/apimachinery/pkg/api/apitesting" 28 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 29 "k8s.io/apimachinery/pkg/runtime" 30 "k8s.io/apimachinery/pkg/runtime/schema" 31 "k8s.io/apimachinery/pkg/runtime/serializer" 32 utilruntime "k8s.io/apimachinery/pkg/util/runtime" 33 "k8s.io/apiserver/pkg/features" 34 "k8s.io/apiserver/pkg/storage" 35 etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing" 36 "k8s.io/apiserver/pkg/storage/storagebackend/factory" 37 utilfeature "k8s.io/apiserver/pkg/util/feature" 38 featuregatetesting "k8s.io/component-base/featuregate/testing" 39 ) 40 41 func init() { 42 var scheme = runtime.NewScheme() 43 44 metav1.AddToGroupVersion(scheme, metav1.SchemeGroupVersion) 45 utilruntime.Must(corev1.AddToScheme(scheme)) 46 utilruntime.Must(scheme.SetVersionPriority(corev1.SchemeGroupVersion)) 47 48 codecs = serializer.NewCodecFactory(scheme) 49 } 50 51 var codecs serializer.CodecFactory 52 53 type serverInfo struct { 54 existingIP string 55 id string 56 ports []corev1.EndpointPort 57 newIP string 58 removeLease bool 59 expectEndpoint string 60 } 61 62 func NewFakePeerEndpointReconciler(t *testing.T, s storage.Interface) peerEndpointLeaseReconciler { 63 // use the same base key used by the controlplane, but add a random 64 // prefix so we can reuse the etcd instance for subtests independently. 65 base := "/" + uuid.New().String() + "/peerserverleases/" 66 return peerEndpointLeaseReconciler{serverLeases: &peerEndpointLeases{ 67 storage: s, 68 destroyFn: func() {}, 69 baseKey: base, 70 leaseTime: 1 * time.Minute, // avoid the lease to timeout on tests 71 }} 72 } 73 74 func (f *peerEndpointLeaseReconciler) SetKeys(servers []serverInfo) error { 75 for _, server := range servers { 76 if err := f.UpdateLease(server.id, server.existingIP, server.ports); err != nil { 77 return err 78 } 79 } 80 return nil 81 } 82 83 func TestPeerEndpointLeaseReconciler(t *testing.T) { 84 // enable feature flags 85 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.APIServerIdentity, true) 86 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StorageVersionAPI, true) 87 88 server, sc := etcd3testing.NewUnsecuredEtcd3TestClientServer(t) 89 t.Cleanup(func() { server.Terminate(t) }) 90 91 newFunc := func() runtime.Object { return &corev1.Endpoints{} } 92 newListFunc := func() runtime.Object { return &corev1.EndpointsList{} } 93 sc.Codec = apitesting.TestStorageCodec(codecs, corev1.SchemeGroupVersion) 94 95 s, dFunc, err := factory.Create(*sc.ForResource(schema.GroupResource{Resource: "endpoints"}), newFunc, newListFunc, "") 96 if err != nil { 97 t.Fatalf("Error creating storage: %v", err) 98 } 99 t.Cleanup(dFunc) 100 101 tests := []struct { 102 testName string 103 servers []serverInfo 104 expectLeases []string 105 }{ 106 { 107 testName: "existing IP satisfy", 108 servers: []serverInfo{{ 109 existingIP: "4.3.2.1", 110 id: "server-1", 111 ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, 112 expectEndpoint: "4.3.2.1:8080", 113 }, { 114 existingIP: "1.2.3.4", 115 id: "server-2", 116 ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, 117 expectEndpoint: "1.2.3.4:8080", 118 }}, 119 expectLeases: []string{"4.3.2.1", "1.2.3.4"}, 120 }, 121 { 122 testName: "existing IP + new IP = should return the new IP", 123 servers: []serverInfo{{ 124 existingIP: "4.3.2.2", 125 id: "server-1", 126 ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, 127 newIP: "4.3.2.1", 128 expectEndpoint: "4.3.2.1:8080", 129 }, { 130 existingIP: "1.2.3.4", 131 id: "server-2", 132 ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, 133 newIP: "1.1.1.1", 134 expectEndpoint: "1.1.1.1:8080", 135 }}, 136 expectLeases: []string{"4.3.2.1", "1.1.1.1"}, 137 }, 138 { 139 testName: "no existing IP, should return new IP", 140 servers: []serverInfo{{ 141 id: "server-1", 142 ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, 143 newIP: "1.2.3.4", 144 expectEndpoint: "1.2.3.4:8080", 145 }}, 146 expectLeases: []string{"1.2.3.4"}, 147 }, 148 } 149 for _, test := range tests { 150 t.Run(test.testName, func(t *testing.T) { 151 fakeReconciler := NewFakePeerEndpointReconciler(t, s) 152 err := fakeReconciler.SetKeys(test.servers) 153 if err != nil { 154 t.Errorf("unexpected error creating keys: %v", err) 155 } 156 157 for _, server := range test.servers { 158 if server.newIP != "" { 159 err = fakeReconciler.UpdateLease(server.id, server.newIP, server.ports) 160 if err != nil { 161 t.Errorf("unexpected error reconciling: %v", err) 162 } 163 } 164 } 165 166 leases, err := fakeReconciler.ListLeases() 167 if err != nil { 168 t.Errorf("unexpected error: %v", err) 169 } 170 // sort for comparison 171 sort.Strings(leases) 172 sort.Strings(test.expectLeases) 173 if !reflect.DeepEqual(leases, test.expectLeases) { 174 t.Errorf("expected %v got: %v", test.expectLeases, leases) 175 } 176 177 for _, server := range test.servers { 178 endpoint, err := fakeReconciler.GetLease(server.id) 179 if err != nil { 180 t.Errorf("unexpected error: %v", err) 181 } 182 if endpoint != server.expectEndpoint { 183 t.Errorf("expected %v got: %v", server.expectEndpoint, endpoint) 184 } 185 } 186 }) 187 } 188 } 189 190 func TestPeerLeaseRemoveEndpoints(t *testing.T) { 191 // enable feature flags 192 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.APIServerIdentity, true) 193 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StorageVersionAPI, true) 194 195 server, sc := etcd3testing.NewUnsecuredEtcd3TestClientServer(t) 196 t.Cleanup(func() { server.Terminate(t) }) 197 198 newFunc := func() runtime.Object { return &corev1.Endpoints{} } 199 newListFunc := func() runtime.Object { return &corev1.EndpointsList{} } 200 sc.Codec = apitesting.TestStorageCodec(codecs, corev1.SchemeGroupVersion) 201 202 s, dFunc, err := factory.Create(*sc.ForResource(schema.GroupResource{Resource: "pods"}), newFunc, newListFunc, "") 203 if err != nil { 204 t.Fatalf("Error creating storage: %v", err) 205 } 206 t.Cleanup(dFunc) 207 208 stopTests := []struct { 209 testName string 210 servers []serverInfo 211 expectLeases []string 212 apiServerStartup bool 213 }{ 214 { 215 testName: "successful remove previous endpoints before apiserver starts", 216 servers: []serverInfo{ 217 { 218 existingIP: "1.2.3.4", 219 id: "test-server-1", 220 ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, 221 removeLease: true, 222 }, 223 { 224 existingIP: "2.4.6.8", 225 id: "test-server-2", 226 ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, 227 }}, 228 expectLeases: []string{"2.4.6.8"}, 229 apiServerStartup: true, 230 }, 231 { 232 testName: "stop reconciling with new IP not in existing ip list", 233 servers: []serverInfo{{ 234 existingIP: "1.2.3.4", 235 newIP: "4.6.8.9", 236 id: "test-server-1", 237 ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, 238 }, 239 { 240 existingIP: "2.4.6.8", 241 id: "test-server-2", 242 ports: []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}}, 243 removeLease: true, 244 }}, 245 expectLeases: []string{"1.2.3.4"}, 246 }, 247 } 248 for _, test := range stopTests { 249 t.Run(test.testName, func(t *testing.T) { 250 fakeReconciler := NewFakePeerEndpointReconciler(t, s) 251 err := fakeReconciler.SetKeys(test.servers) 252 if err != nil { 253 t.Errorf("unexpected error creating keys: %v", err) 254 } 255 if !test.apiServerStartup { 256 fakeReconciler.StopReconciling() 257 } 258 for _, server := range test.servers { 259 if server.removeLease { 260 err = fakeReconciler.RemoveLease(server.id) 261 // if the ip is not on the endpoints, it must return an storage error and stop reconciling 262 if err != nil { 263 t.Errorf("unexpected error reconciling: %v", err) 264 } 265 } 266 } 267 268 leases, err := fakeReconciler.ListLeases() 269 if err != nil { 270 t.Errorf("unexpected error: %v", err) 271 } 272 // sort for comparison 273 sort.Strings(leases) 274 sort.Strings(test.expectLeases) 275 if !reflect.DeepEqual(leases, test.expectLeases) { 276 t.Errorf("expected %v got: %v", test.expectLeases, leases) 277 } 278 }) 279 } 280 }