k8s.io/kubernetes@v1.29.3/test/integration/network/services_test.go (about) 1 /* 2 Copyright 2021 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package network 18 19 import ( 20 "context" 21 "encoding/json" 22 "fmt" 23 "testing" 24 "time" 25 26 v1 "k8s.io/api/core/v1" 27 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 "k8s.io/apimachinery/pkg/types" 29 "k8s.io/apimachinery/pkg/util/intstr" 30 "k8s.io/apimachinery/pkg/util/strategicpatch" 31 "k8s.io/apimachinery/pkg/util/wait" 32 "k8s.io/kubernetes/cmd/kube-apiserver/app/options" 33 "k8s.io/kubernetes/pkg/controlplane" 34 35 "k8s.io/kubernetes/test/integration/framework" 36 "k8s.io/kubernetes/test/utils/ktesting" 37 ) 38 39 // TestServicesFinalizersRepairLoop tests that Services participate in the object 40 // deletion when using finalizers, and that the Services Repair controller doesn't, 41 // mistakenly, repair the ClusterIP assigned to the Service that is being deleted. 42 // https://issues.k8s.io/87603 43 func TestServicesFinalizersRepairLoop(t *testing.T) { 44 serviceCIDR := "10.0.0.0/16" 45 clusterIP := "10.0.0.20" 46 interval := 5 * time.Second 47 48 _, ctx := ktesting.NewTestContext(t) 49 ctx, cancel := context.WithCancel(ctx) 50 defer cancel() 51 52 client, _, tearDownFn := framework.StartTestServer(ctx, t, framework.TestServerSetup{ 53 ModifyServerRunOptions: func(opts *options.ServerRunOptions) { 54 opts.ServiceClusterIPRanges = serviceCIDR 55 }, 56 ModifyServerConfig: func(cfg *controlplane.Config) { 57 cfg.ExtraConfig.RepairServicesInterval = interval 58 }, 59 }) 60 defer tearDownFn() 61 62 // verify client is working 63 if err := wait.PollImmediate(5*time.Second, 2*time.Minute, func() (bool, error) { 64 _, err := client.CoreV1().Endpoints(metav1.NamespaceDefault).Get(ctx, "kubernetes", metav1.GetOptions{}) 65 if err != nil { 66 t.Logf("error fetching endpoints: %v", err) 67 return false, nil 68 } 69 return true, nil 70 }); err != nil { 71 t.Errorf("server without enabled endpoints failed to register: %v", err) 72 } 73 74 // Create a NodePort service with one finalizer 75 svcNodePort := v1.Service{ 76 ObjectMeta: metav1.ObjectMeta{ 77 Name: "svc", 78 Finalizers: []string{"foo.bar/some-finalizer"}, 79 }, 80 Spec: v1.ServiceSpec{ 81 ClusterIP: clusterIP, 82 Ports: []v1.ServicePort{{ 83 Port: 8443, 84 NodePort: 30443, 85 TargetPort: intstr.FromInt32(8443), 86 Protocol: v1.ProtocolTCP, 87 }}, 88 Type: v1.ServiceTypeNodePort, 89 }, 90 } 91 92 // Create service 93 if _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(ctx, &svcNodePort, metav1.CreateOptions{}); err != nil { 94 t.Errorf("unexpected error creating service: %v", err) 95 } 96 t.Logf("Created service: %s", svcNodePort.Name) 97 98 // Check the service has been created correctly 99 svc, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(ctx, svcNodePort.Name, metav1.GetOptions{}) 100 if err != nil || svc.Spec.ClusterIP != clusterIP { 101 t.Errorf("created service is not correct: %v", err) 102 } 103 t.Logf("Service created successfully: %v", svc) 104 105 // Delete service 106 if err := client.CoreV1().Services(metav1.NamespaceDefault).Delete(ctx, svcNodePort.Name, metav1.DeleteOptions{}); err != nil { 107 t.Errorf("unexpected error deleting service: %v", err) 108 } 109 t.Logf("Deleted service: %s", svcNodePort.Name) 110 111 // wait for the repair loop to recover the deleted resources 112 time.Sleep(interval + 1) 113 114 // Check that the service was not deleted and the IP is already allocated 115 svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(ctx, svcNodePort.Name, metav1.GetOptions{}) 116 if err != nil || svc.Spec.ClusterIP != clusterIP { 117 t.Errorf("created service is not correct: %v", err) 118 } 119 t.Logf("Service after Delete: %v", svc) 120 121 // Remove the finalizer 122 if _, err = client.CoreV1().Services(metav1.NamespaceDefault).Patch(ctx, svcNodePort.Name, types.JSONPatchType, []byte(`[{"op":"remove","path":"/metadata/finalizers"}]`), metav1.PatchOptions{}); err != nil { 123 t.Errorf("unexpected error removing finalizer: %v", err) 124 } 125 t.Logf("Removed service finalizer: %s", svcNodePort.Name) 126 127 // Check that the service was deleted 128 _, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(ctx, svcNodePort.Name, metav1.GetOptions{}) 129 if err == nil { 130 t.Errorf("service was not delete: %v", err) 131 } 132 133 // Try to create service again 134 if _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(ctx, &svcNodePort, metav1.CreateOptions{}); err != nil { 135 t.Errorf("unexpected error creating service: %v", err) 136 } 137 t.Logf("Created service: %s", svcNodePort.Name) 138 } 139 140 func TestServicesFinalizersPatchStatus(t *testing.T) { 141 serviceCIDR := "10.0.0.0/16" 142 clusterIP := "10.0.0.21" 143 nodePort := 30443 144 _, ctx := ktesting.NewTestContext(t) 145 ctx, cancel := context.WithCancel(ctx) 146 defer cancel() 147 148 client, _, tearDownFn := framework.StartTestServer(ctx, t, framework.TestServerSetup{ 149 ModifyServerRunOptions: func(opts *options.ServerRunOptions) { 150 opts.ServiceClusterIPRanges = serviceCIDR 151 }, 152 }) 153 defer tearDownFn() 154 155 for _, testcase := range []string{"spec", "status"} { 156 t.Run(testcase, func(t *testing.T) { 157 // Create a NodePort service with one finalizer 158 svcNodePort := v1.Service{ 159 ObjectMeta: metav1.ObjectMeta{ 160 Name: "svc" + testcase, 161 Finalizers: []string{"foo.bar/some-finalizer"}, 162 }, 163 Spec: v1.ServiceSpec{ 164 ClusterIP: clusterIP, 165 Ports: []v1.ServicePort{{ 166 Port: 8443, 167 NodePort: int32(nodePort), 168 TargetPort: intstr.FromInt32(8443), 169 Protocol: v1.ProtocolTCP, 170 }}, 171 Type: v1.ServiceTypeNodePort, 172 }, 173 } 174 175 ns := framework.CreateNamespaceOrDie(client, "test-service-finalizers-"+testcase, t) 176 defer framework.DeleteNamespaceOrDie(client, ns, t) 177 178 // Create service 179 if _, err := client.CoreV1().Services(ns.Name).Create(ctx, &svcNodePort, metav1.CreateOptions{}); err != nil { 180 t.Fatalf("unexpected error creating service: %v", err) 181 } 182 t.Logf("Created service: %s", svcNodePort.Name) 183 184 // Check the service has been created correctly 185 svc, err := client.CoreV1().Services(ns.Name).Get(ctx, svcNodePort.Name, metav1.GetOptions{}) 186 if err != nil || svc.Spec.ClusterIP != clusterIP { 187 t.Fatalf("created service is not correct: %v", err) 188 } 189 t.Logf("Service created successfully: %+v", svc) 190 191 // Delete service 192 if err := client.CoreV1().Services(ns.Name).Delete(ctx, svcNodePort.Name, metav1.DeleteOptions{}); err != nil { 193 t.Fatalf("unexpected error deleting service: %v", err) 194 } 195 t.Logf("Deleted service: %s", svcNodePort.Name) 196 197 // Check that the service was not deleted and the IP is already allocated 198 svc, err = client.CoreV1().Services(ns.Name).Get(ctx, svcNodePort.Name, metav1.GetOptions{}) 199 if err != nil || 200 svc.Spec.ClusterIP != clusterIP || 201 int(svc.Spec.Ports[0].NodePort) != nodePort || 202 svc.DeletionTimestamp == nil || 203 len(svc.ObjectMeta.Finalizers) != 1 { 204 t.Fatalf("Service expected to be deleting and with the same values: %v", err) 205 } 206 t.Logf("Service after Delete: %+v", svc) 207 208 // Remove the finalizer 209 updated := svc.DeepCopy() 210 updated.ObjectMeta.Finalizers = []string{} 211 patchBytes, err := getPatchBytes(svc, updated) 212 if err != nil { 213 t.Fatalf("unexpected error getting patch bytes: %v", err) 214 } 215 216 if testcase == "spec" { 217 if _, err = client.CoreV1().Services(ns.Name).Patch(ctx, svcNodePort.Name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}); err != nil { 218 t.Fatalf("unexpected error removing finalizer: %v", err) 219 } 220 } else { 221 if _, err = client.CoreV1().Services(ns.Name).Patch(ctx, svcNodePort.Name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}, "status"); err != nil { 222 t.Fatalf("unexpected error removing finalizer: %v", err) 223 } 224 } 225 t.Logf("Removed service finalizer: %s", svcNodePort.Name) 226 227 // Check that the service was deleted 228 _, err = client.CoreV1().Services(ns.Name).Get(ctx, svcNodePort.Name, metav1.GetOptions{}) 229 if err == nil { 230 t.Fatalf("service was not delete: %v", err) 231 } 232 233 // Try to create service again without the finalizer to check the ClusterIP and NodePort are deallocated 234 svc = svcNodePort.DeepCopy() 235 svc.Finalizers = []string{} 236 if _, err := client.CoreV1().Services(ns.Name).Create(ctx, svc, metav1.CreateOptions{}); err != nil { 237 t.Fatalf("unexpected error creating service: %v", err) 238 } 239 // Delete service 240 if err := client.CoreV1().Services(ns.Name).Delete(ctx, svc.Name, metav1.DeleteOptions{}); err != nil { 241 t.Fatalf("unexpected error deleting service: %v", err) 242 } 243 }) 244 } 245 } 246 247 // Regresion test for https://issues.k8s.io/115316 248 func TestServiceCIDR28bits(t *testing.T) { 249 serviceCIDR := "10.0.0.0/28" 250 251 _, ctx := ktesting.NewTestContext(t) 252 ctx, cancel := context.WithCancel(ctx) 253 defer cancel() 254 255 client, _, tearDownFn := framework.StartTestServer(ctx, t, framework.TestServerSetup{ 256 ModifyServerRunOptions: func(opts *options.ServerRunOptions) { 257 opts.ServiceClusterIPRanges = serviceCIDR 258 }, 259 }) 260 defer tearDownFn() 261 262 // Wait until the default "kubernetes" service is created. 263 if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { 264 _, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(ctx, "kubernetes", metav1.GetOptions{}) 265 if err != nil { 266 return false, err 267 } 268 return true, nil 269 }); err != nil { 270 t.Fatalf("creating kubernetes service timed out") 271 } 272 273 ns := framework.CreateNamespaceOrDie(client, "test-regression", t) 274 defer framework.DeleteNamespaceOrDie(client, ns, t) 275 276 service := &v1.Service{ 277 ObjectMeta: metav1.ObjectMeta{ 278 Name: "test-1234", 279 }, 280 Spec: v1.ServiceSpec{ 281 Type: v1.ServiceTypeClusterIP, 282 Ports: []v1.ServicePort{{ 283 Port: int32(80), 284 }}, 285 Selector: map[string]string{ 286 "foo": "bar", 287 }, 288 }, 289 } 290 291 _, err := client.CoreV1().Services(ns.Name).Create(ctx, service, metav1.CreateOptions{}) 292 if err != nil { 293 t.Fatalf("Error creating test service: %v", err) 294 } 295 } 296 297 func getPatchBytes(oldSvc, newSvc *v1.Service) ([]byte, error) { 298 oldData, err := json.Marshal(oldSvc) 299 if err != nil { 300 return nil, fmt.Errorf("failed to Marshal oldData for svc %s/%s: %v", oldSvc.Namespace, oldSvc.Name, err) 301 } 302 303 newData, err := json.Marshal(newSvc) 304 if err != nil { 305 return nil, fmt.Errorf("failed to Marshal newData for svc %s/%s: %v", newSvc.Namespace, newSvc.Name, err) 306 } 307 308 patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Service{}) 309 if err != nil { 310 return nil, fmt.Errorf("failed to CreateTwoWayMergePatch for svc %s/%s: %v", oldSvc.Namespace, oldSvc.Name, err) 311 } 312 return patchBytes, nil 313 314 }