k8s.io/kubernetes@v1.29.3/test/integration/servicecidr/servicecidr_test.go (about) 1 /* 2 Copyright 2023 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package servicecidr 18 19 import ( 20 "context" 21 "fmt" 22 "net/netip" 23 "strings" 24 "testing" 25 "time" 26 27 v1 "k8s.io/api/core/v1" 28 networkingv1alpha1 "k8s.io/api/networking/v1alpha1" 29 apierrors "k8s.io/apimachinery/pkg/api/errors" 30 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 31 "k8s.io/apimachinery/pkg/util/wait" 32 utilfeature "k8s.io/apiserver/pkg/util/feature" 33 "k8s.io/client-go/informers" 34 "k8s.io/client-go/kubernetes" 35 featuregatetesting "k8s.io/component-base/featuregate/testing" 36 kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" 37 "k8s.io/kubernetes/pkg/controller/servicecidrs" 38 "k8s.io/kubernetes/pkg/features" 39 "k8s.io/kubernetes/test/integration/framework" 40 ) 41 42 func TestServiceAllocNewServiceCIDR(t *testing.T) { 43 defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.MultiCIDRServiceAllocator, true)() 44 45 etcdOptions := framework.SharedEtcd() 46 apiServerOptions := kubeapiservertesting.NewDefaultTestServerOptions() 47 s := kubeapiservertesting.StartTestServerOrDie(t, 48 apiServerOptions, 49 []string{ 50 "--runtime-config=networking.k8s.io/v1alpha1=true", 51 "--service-cluster-ip-range=192.168.0.0/29", 52 "--advertise-address=10.1.1.1", 53 "--disable-admission-plugins=ServiceAccount", 54 }, 55 etcdOptions) 56 defer s.TearDownFn() 57 58 client, err := kubernetes.NewForConfig(s.ClientConfig) 59 if err != nil { 60 t.Fatalf("Unexpected error: %v", err) 61 } 62 // ServiceCIDR controller 63 ctx, cancel := context.WithCancel(context.Background()) 64 defer cancel() 65 resyncPeriod := 12 * time.Hour 66 informerFactory := informers.NewSharedInformerFactory(client, resyncPeriod) 67 go servicecidrs.NewController( 68 informerFactory.Networking().V1alpha1().ServiceCIDRs(), 69 informerFactory.Networking().V1alpha1().IPAddresses(), 70 client, 71 ).Run(ctx, 5) 72 informerFactory.Start(ctx.Done()) 73 74 // /29 = 6 services, kubernetes.default takes the first address 75 // make 5 more services to take up all IPs 76 for i := 0; i < 5; i++ { 77 if _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(context.Background(), makeService(fmt.Sprintf("service-%d", i)), metav1.CreateOptions{}); err != nil { 78 t.Fatal(err) 79 } 80 } 81 82 // Make another service. It will fail because we're out of cluster IPs 83 if _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(context.Background(), makeService("fail"), metav1.CreateOptions{}); err != nil { 84 if !strings.Contains(err.Error(), "range is full") { 85 t.Fatalf("unexpected error text: %v", err) 86 } 87 } else { 88 svcs, err := client.CoreV1().Services(metav1.NamespaceAll).List(context.Background(), metav1.ListOptions{}) 89 if err != nil { 90 t.Fatalf("unexpected error getting the services: %v", err) 91 } 92 allIPs := []string{} 93 for _, s := range svcs.Items { 94 allIPs = append(allIPs, s.Spec.ClusterIP) 95 } 96 t.Fatalf("unexpected creation success. The following IPs exist: %#v. It should only be possible to allocate 6 IP addresses in this cluster.\n\n%#v", allIPs, svcs) 97 } 98 99 // Add a new service CIDR to be able to create new IPs. 100 cidr := makeServiceCIDR("test2", "10.168.0.0/24", "") 101 if _, err := client.NetworkingV1alpha1().ServiceCIDRs().Create(context.Background(), cidr, metav1.CreateOptions{}); err != nil { 102 t.Fatalf("got unexpected error: %v", err) 103 } 104 // wait ServiceCIDR is ready 105 if err := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, time.Minute, false, func(ctx context.Context) (bool, error) { 106 cidr, err := client.NetworkingV1alpha1().ServiceCIDRs().Get(context.TODO(), cidr.Name, metav1.GetOptions{}) 107 if err != nil { 108 return false, err 109 } 110 return isServiceCIDRReady(cidr), nil 111 }); err != nil { 112 t.Fatalf("waiting for default service cidr ready condition set to false: %v", err) 113 } 114 // This time creating more Services should work 115 for i := 10; i < 150; i++ { 116 if _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(context.Background(), makeService(fmt.Sprintf("service-%d", i)), metav1.CreateOptions{}); err != nil { 117 t.Fatal(err) 118 } 119 } 120 } 121 122 // A ServiceCIDR can be deleted if there are no orphan IPs or if the existing IPs are contained in other ServiceCIDR 123 // that is not being deleted. 124 // The test starts the apiserver with the range "192.168.0.0/29" 125 // Create Services to fill the range 126 // Creates a new ServiceCIDR cidr1 with the same range as the one defined in the apiserver 127 // Deletes cidr1 object will work since its range is covered by the default ServiceCIDR created by the apiserver flags 128 // Creates a new cidr2 with a different range than cidr1 129 // Creates a new service so it picks an IPAddress on this range because "192.168.0.0/29" is full at this point 130 // Creates a new cidr3 that contains cidr2 131 // Deletes cidr2 since it is covered by cidr3 132 // Tries to delete cidr3 but is blocked since there is an IPAddress 133 // Deletes the Service with the IPAddress blocking the deletion 134 // cidr3 must not exist at this point 135 func TestServiceCIDRDeletion(t *testing.T) { 136 defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.MultiCIDRServiceAllocator, true)() 137 cidr1 := "192.168.0.0/29" // same as the default 138 cidr2 := "10.0.0.0/24" // new range 139 cidr3 := "10.0.0.0/16" // contains cidr2 140 141 etcdOptions := framework.SharedEtcd() 142 apiServerOptions := kubeapiservertesting.NewDefaultTestServerOptions() 143 s := kubeapiservertesting.StartTestServerOrDie(t, 144 apiServerOptions, 145 []string{ 146 "--runtime-config=networking.k8s.io/v1alpha1=true", 147 "--service-cluster-ip-range=" + cidr1, 148 "--advertise-address=172.16.1.1", 149 "--disable-admission-plugins=ServiceAccount", 150 }, 151 etcdOptions) 152 defer s.TearDownFn() 153 154 client, err := kubernetes.NewForConfig(s.ClientConfig) 155 if err != nil { 156 t.Fatalf("Unexpected error: %v", err) 157 } 158 159 ns := framework.CreateNamespaceOrDie(client, "test-service-cidr-deletion", t) 160 defer framework.DeleteNamespaceOrDie(client, ns, t) 161 162 // ServiceCIDR controller 163 ctx, cancel := context.WithCancel(context.Background()) 164 defer cancel() 165 resyncPeriod := 12 * time.Hour 166 informerFactory := informers.NewSharedInformerFactory(client, resyncPeriod) 167 go servicecidrs.NewController( 168 informerFactory.Networking().V1alpha1().ServiceCIDRs(), 169 informerFactory.Networking().V1alpha1().IPAddresses(), 170 client, 171 ).Run(ctx, 5) 172 informerFactory.Start(ctx.Done()) 173 174 // /29 = 6 services, kubernetes.default takes the first address 175 // make 5 more services to take up all IPs 176 for i := 0; i < 5; i++ { 177 if _, err := client.CoreV1().Services(ns.Name).Create(context.Background(), makeService(fmt.Sprintf("service-%d", i)), metav1.CreateOptions{}); err != nil { 178 t.Fatal(err) 179 } 180 } 181 // create a new ServiceCIDRs that overlaps the default one 182 _, err = client.NetworkingV1alpha1().ServiceCIDRs().Create(ctx, makeServiceCIDR("cidr1", cidr1, ""), metav1.CreateOptions{}) 183 if err != nil { 184 t.Fatal((err)) 185 } 186 // Wait until is ready. 187 if err := wait.PollUntilContextTimeout(context.Background(), 250*time.Millisecond, 30*time.Second, false, func(ctx context.Context) (bool, error) { 188 cidr, err := client.NetworkingV1alpha1().ServiceCIDRs().Get(ctx, "cidr1", metav1.GetOptions{}) 189 if err != nil { 190 return false, nil 191 } 192 return isServiceCIDRReady(cidr), nil 193 }); err != nil { 194 t.Fatalf("cidr1 is not ready") 195 } 196 // we should be able to delete the ServiceCIDR despite it contains IP addresses as it overlaps with the default ServiceCIDR 197 err = client.NetworkingV1alpha1().ServiceCIDRs().Delete(ctx, "cidr1", metav1.DeleteOptions{}) 198 if err != nil { 199 t.Fatal((err)) 200 } 201 202 if err := wait.PollUntilContextTimeout(context.Background(), 250*time.Millisecond, 30*time.Second, false, func(ctx context.Context) (bool, error) { 203 _, err := client.NetworkingV1alpha1().ServiceCIDRs().Get(ctx, "cidr1", metav1.GetOptions{}) 204 if err != nil && apierrors.IsNotFound(err) { 205 return true, nil 206 } 207 return false, nil 208 }); err != nil { 209 t.Fatalf("cidr1 has not been deleted") 210 } 211 212 // add a new ServiceCIDR with a new range 213 _, err = client.NetworkingV1alpha1().ServiceCIDRs().Create(ctx, makeServiceCIDR("cidr2", cidr2, ""), metav1.CreateOptions{}) 214 if err != nil { 215 t.Fatal((err)) 216 } 217 // wait the allocator process the new ServiceCIDR 218 // Wait until is ready. 219 if err := wait.PollUntilContextTimeout(context.Background(), 250*time.Millisecond, 30*time.Second, false, func(ctx context.Context) (bool, error) { 220 cidr, err := client.NetworkingV1alpha1().ServiceCIDRs().Get(ctx, "cidr2", metav1.GetOptions{}) 221 if err != nil { 222 return false, nil 223 } 224 return isServiceCIDRReady(cidr), nil 225 }); err != nil { 226 t.Fatalf("cidr2 is not ready") 227 } 228 // create a new Service so it will take a new IP address from the new range 229 svc, err := client.CoreV1().Services(ns.Name).Create(context.Background(), makeService("new-cidr-service"), metav1.CreateOptions{}) 230 if err != nil { 231 t.Fatal(err) 232 } 233 234 if !cidrContainsIP(cidr2, svc.Spec.ClusterIP) { 235 t.Fatalf("Service %s expected to have an IP on range %s, got %s", svc.Name, cidr2, svc.Spec.ClusterIP) 236 } 237 238 // add a new ServiceCIDR that overlaps the existing one 239 _, err = client.NetworkingV1alpha1().ServiceCIDRs().Create(ctx, makeServiceCIDR("cidr3", cidr3, ""), metav1.CreateOptions{}) 240 if err != nil { 241 t.Fatal((err)) 242 } 243 // Wait until is ready. 244 if err := wait.PollUntilContextTimeout(context.Background(), 250*time.Millisecond, 30*time.Second, false, func(ctx context.Context) (bool, error) { 245 cidr, err := client.NetworkingV1alpha1().ServiceCIDRs().Get(ctx, "cidr3", metav1.GetOptions{}) 246 if err != nil { 247 return false, nil 248 } 249 return isServiceCIDRReady(cidr), nil 250 }); err != nil { 251 t.Fatalf("cidr3 is not ready") 252 } 253 // we should be able to delete the ServiceCIDR2 despite it contains IP addresses as it is contained on ServiceCIDR3 254 err = client.NetworkingV1alpha1().ServiceCIDRs().Delete(ctx, "cidr2", metav1.DeleteOptions{}) 255 if err != nil { 256 t.Fatal((err)) 257 } 258 259 if err := wait.PollUntilContextTimeout(context.Background(), 250*time.Millisecond, 30*time.Second, false, func(ctx context.Context) (bool, error) { 260 _, err := client.NetworkingV1alpha1().ServiceCIDRs().Get(ctx, "cidr2", metav1.GetOptions{}) 261 if err != nil && apierrors.IsNotFound(err) { 262 return true, nil 263 } 264 return false, nil 265 }); err != nil { 266 t.Fatalf("cidr2 has not been deleted") 267 } 268 269 // serviceCIDR3 will not be able to be deleted until the IPAddress is removed 270 err = client.NetworkingV1alpha1().ServiceCIDRs().Delete(ctx, "cidr3", metav1.DeleteOptions{}) 271 if err != nil { 272 t.Fatal((err)) 273 } 274 // Wait until is not ready. 275 if err := wait.PollUntilContextTimeout(context.Background(), 250*time.Millisecond, 30*time.Second, false, func(ctx context.Context) (bool, error) { 276 cidr, err := client.NetworkingV1alpha1().ServiceCIDRs().Get(ctx, "cidr3", metav1.GetOptions{}) 277 if err != nil { 278 return false, nil 279 } 280 for _, condition := range cidr.Status.Conditions { 281 if condition.Type == networkingv1alpha1.ServiceCIDRConditionReady { 282 return condition.Status == metav1.ConditionStatus(metav1.ConditionFalse), nil 283 } 284 } 285 return false, nil 286 }); err != nil { 287 t.Fatalf("cidr3 is ready") 288 } 289 290 // delete the service blocking the deletion 291 if err := client.CoreV1().Services(ns.Name).Delete(context.Background(), "new-cidr-service", metav1.DeleteOptions{}); err != nil { 292 t.Fatal(err) 293 } 294 295 // cidr3 must not exist 296 if err := wait.PollUntilContextTimeout(context.Background(), 250*time.Millisecond, 30*time.Second, false, func(ctx context.Context) (bool, error) { 297 _, err := client.NetworkingV1alpha1().ServiceCIDRs().Get(ctx, "cidr3", metav1.GetOptions{}) 298 if err != nil && apierrors.IsNotFound(err) { 299 return true, nil 300 } 301 return false, nil 302 }); err != nil { 303 t.Fatalf("cidr3 has not been deleted") 304 } 305 } 306 307 func makeServiceCIDR(name, primary, secondary string) *networkingv1alpha1.ServiceCIDR { 308 serviceCIDR := &networkingv1alpha1.ServiceCIDR{ 309 ObjectMeta: metav1.ObjectMeta{ 310 Name: name, 311 }, 312 Spec: networkingv1alpha1.ServiceCIDRSpec{}, 313 } 314 serviceCIDR.Spec.CIDRs = append(serviceCIDR.Spec.CIDRs, primary) 315 if secondary != "" { 316 serviceCIDR.Spec.CIDRs = append(serviceCIDR.Spec.CIDRs, secondary) 317 } 318 return serviceCIDR 319 } 320 321 func makeService(name string) *v1.Service { 322 return &v1.Service{ 323 ObjectMeta: metav1.ObjectMeta{ 324 Name: name, 325 }, 326 Spec: v1.ServiceSpec{ 327 Type: v1.ServiceTypeClusterIP, 328 Ports: []v1.ServicePort{ 329 {Port: 80}, 330 }, 331 }, 332 } 333 } 334 335 // returns true of the ServiceCIDRConditionReady is true 336 func isServiceCIDRReady(serviceCIDR *networkingv1alpha1.ServiceCIDR) bool { 337 if serviceCIDR == nil { 338 return false 339 } 340 341 for _, condition := range serviceCIDR.Status.Conditions { 342 if condition.Type == networkingv1alpha1.ServiceCIDRConditionReady { 343 return condition.Status == metav1.ConditionStatus(metav1.ConditionTrue) 344 } 345 } 346 347 return false 348 } 349 350 func cidrContainsIP(cidr, ip string) bool { 351 prefix := netip.MustParsePrefix(cidr) 352 address := netip.MustParseAddr(ip) 353 return prefix.Contains(address) 354 }