k8s.io/kubernetes@v1.29.3/test/integration/dualstack/dualstack_endpoints_test.go (about) 1 /* 2 Copyright 2020 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package dualstack 18 19 import ( 20 "context" 21 "fmt" 22 "testing" 23 "time" 24 25 v1 "k8s.io/api/core/v1" 26 discovery "k8s.io/api/discovery/v1" 27 apierrors "k8s.io/apimachinery/pkg/api/errors" 28 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 29 "k8s.io/apimachinery/pkg/util/intstr" 30 "k8s.io/apimachinery/pkg/util/wait" 31 "k8s.io/client-go/informers" 32 "k8s.io/kubernetes/cmd/kube-apiserver/app/options" 33 "k8s.io/kubernetes/pkg/controller/endpoint" 34 "k8s.io/kubernetes/pkg/controller/endpointslice" 35 "k8s.io/kubernetes/test/integration/framework" 36 "k8s.io/kubernetes/test/utils/ktesting" 37 ) 38 39 func TestDualStackEndpoints(t *testing.T) { 40 // Create an IPv4IPv6 dual stack control-plane 41 serviceCIDR := "10.0.0.0/16" 42 secondaryServiceCIDR := "2001:db8:1::/112" 43 labelMap := func() map[string]string { 44 return map[string]string{"foo": "bar"} 45 } 46 47 _, ctx := ktesting.NewTestContext(t) 48 ctx, cancel := context.WithCancel(ctx) 49 defer cancel() 50 51 client, _, tearDownFn := framework.StartTestServer(ctx, t, framework.TestServerSetup{ 52 ModifyServerRunOptions: func(opts *options.ServerRunOptions) { 53 opts.ServiceClusterIPRanges = fmt.Sprintf("%s,%s", serviceCIDR, secondaryServiceCIDR) 54 // Disable ServiceAccount admission plugin as we don't have serviceaccount controller running. 55 opts.Admission.GenericAdmission.DisablePlugins = []string{"ServiceAccount"} 56 }, 57 }) 58 defer tearDownFn() 59 60 // Wait until the default "kubernetes" service is created. 61 if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) { 62 _, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(ctx, "kubernetes", metav1.GetOptions{}) 63 if err != nil && !apierrors.IsNotFound(err) { 64 return false, err 65 } 66 return !apierrors.IsNotFound(err), nil 67 }); err != nil { 68 t.Fatalf("Creating kubernetes service timed out") 69 } 70 71 resyncPeriod := 0 * time.Hour 72 informers := informers.NewSharedInformerFactory(client, resyncPeriod) 73 74 // Create fake node 75 testNode := &v1.Node{ 76 ObjectMeta: metav1.ObjectMeta{ 77 Name: "fakenode", 78 }, 79 Spec: v1.NodeSpec{Unschedulable: false}, 80 Status: v1.NodeStatus{ 81 Conditions: []v1.NodeCondition{ 82 { 83 Type: v1.NodeReady, 84 Status: v1.ConditionTrue, 85 Reason: fmt.Sprintf("schedulable condition"), 86 LastHeartbeatTime: metav1.Time{Time: time.Now()}, 87 }, 88 }, 89 }, 90 } 91 if _, err := client.CoreV1().Nodes().Create(ctx, testNode, metav1.CreateOptions{}); err != nil { 92 t.Fatalf("Failed to create Node %q: %v", testNode.Name, err) 93 } 94 95 epController := endpoint.NewEndpointController( 96 informers.Core().V1().Pods(), 97 informers.Core().V1().Services(), 98 informers.Core().V1().Endpoints(), 99 client, 100 1*time.Second) 101 102 epsController := endpointslice.NewController( 103 ctx, 104 informers.Core().V1().Pods(), 105 informers.Core().V1().Services(), 106 informers.Core().V1().Nodes(), 107 informers.Discovery().V1().EndpointSlices(), 108 int32(100), 109 client, 110 1*time.Second) 111 112 // Start informer and controllers 113 informers.Start(ctx.Done()) 114 // use only one worker to serialize the updates 115 go epController.Run(ctx, 1) 116 go epsController.Run(ctx, 1) 117 118 var testcases = []struct { 119 name string 120 serviceType v1.ServiceType 121 ipFamilies []v1.IPFamily 122 ipFamilyPolicy v1.IPFamilyPolicy 123 }{ 124 { 125 name: "Service IPv4 Only", 126 serviceType: v1.ServiceTypeClusterIP, 127 ipFamilies: []v1.IPFamily{v1.IPv4Protocol}, 128 ipFamilyPolicy: v1.IPFamilyPolicySingleStack, 129 }, 130 { 131 name: "Service IPv6 Only", 132 serviceType: v1.ServiceTypeClusterIP, 133 ipFamilies: []v1.IPFamily{v1.IPv6Protocol}, 134 ipFamilyPolicy: v1.IPFamilyPolicySingleStack, 135 }, 136 { 137 name: "Service IPv6 IPv4 Dual Stack", 138 serviceType: v1.ServiceTypeClusterIP, 139 ipFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol}, 140 ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, 141 }, 142 { 143 name: "Service IPv4 IPv6 Dual Stack", 144 serviceType: v1.ServiceTypeClusterIP, 145 ipFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, 146 ipFamilyPolicy: v1.IPFamilyPolicyRequireDualStack, 147 }, 148 } 149 150 for i, tc := range testcases { 151 t.Run(tc.name, func(t *testing.T) { 152 ns := framework.CreateNamespaceOrDie(client, fmt.Sprintf("test-endpointslice-dualstack-%d", i), t) 153 defer framework.DeleteNamespaceOrDie(client, ns, t) 154 155 // Create a pod with labels 156 pod := &v1.Pod{ 157 ObjectMeta: metav1.ObjectMeta{ 158 Name: "test-pod", 159 Namespace: ns.Name, 160 Labels: labelMap(), 161 }, 162 Spec: v1.PodSpec{ 163 NodeName: "fakenode", 164 Containers: []v1.Container{ 165 { 166 Name: "fake-name", 167 Image: "fakeimage", 168 }, 169 }, 170 }, 171 } 172 173 createdPod, err := client.CoreV1().Pods(ns.Name).Create(ctx, pod, metav1.CreateOptions{}) 174 if err != nil { 175 t.Fatalf("Failed to create pod %s: %v", pod.Name, err) 176 } 177 178 // Set pod IPs 179 podIPbyFamily := map[v1.IPFamily]string{v1.IPv4Protocol: "1.1.1.1", v1.IPv6Protocol: "2001:db2::65"} 180 createdPod.Status = v1.PodStatus{ 181 Phase: v1.PodRunning, 182 PodIPs: []v1.PodIP{{IP: podIPbyFamily[v1.IPv4Protocol]}, {IP: podIPbyFamily[v1.IPv6Protocol]}}, 183 } 184 _, err = client.CoreV1().Pods(ns.Name).UpdateStatus(ctx, createdPod, metav1.UpdateOptions{}) 185 if err != nil { 186 t.Fatalf("Failed to update status of pod %s: %v", pod.Name, err) 187 } 188 189 svc := &v1.Service{ 190 ObjectMeta: metav1.ObjectMeta{ 191 Name: fmt.Sprintf("svc-test-%d", i), // use different services for each test 192 Namespace: ns.Name, 193 Labels: labelMap(), 194 }, 195 Spec: v1.ServiceSpec{ 196 Type: v1.ServiceTypeClusterIP, 197 IPFamilies: tc.ipFamilies, 198 IPFamilyPolicy: &tc.ipFamilyPolicy, 199 Selector: labelMap(), 200 Ports: []v1.ServicePort{ 201 { 202 Name: fmt.Sprintf("port-test-%d", i), 203 Port: 443, 204 TargetPort: intstr.IntOrString{IntVal: 443}, 205 Protocol: "TCP", 206 }, 207 }, 208 }, 209 } 210 211 // create a service 212 _, err = client.CoreV1().Services(ns.Name).Create(ctx, svc, metav1.CreateOptions{}) 213 if err != nil { 214 t.Fatalf("Error creating service: %v", err) 215 } 216 217 // wait until endpoints are created 218 // legacy endpoints are not dual stack 219 // and use the address of the first IP family 220 if err := wait.PollImmediate(1*time.Second, wait.ForeverTestTimeout, func() (bool, error) { 221 e, err := client.CoreV1().Endpoints(ns.Name).Get(ctx, svc.Name, metav1.GetOptions{}) 222 if err != nil { 223 t.Logf("Error fetching endpoints: %v", err) 224 return false, nil 225 } 226 // check if the endpoint addresses match the pod IP of the first IPFamily of the service 227 // since this is an integration test PodIPs are not "ready" 228 if len(e.Subsets) > 0 && len(e.Subsets[0].NotReadyAddresses) > 0 { 229 if e.Subsets[0].NotReadyAddresses[0].IP == podIPbyFamily[tc.ipFamilies[0]] { 230 return true, nil 231 } 232 t.Logf("Endpoint address %s does not match PodIP %s ", e.Subsets[0].Addresses[0].IP, podIPbyFamily[tc.ipFamilies[0]]) 233 } 234 t.Logf("Endpoint does not contain addresses: %s", e.Name) 235 return false, nil 236 }); err != nil { 237 t.Fatalf("Endpoints not found: %v", err) 238 } 239 240 // wait until the endpoint slices are created 241 err = wait.PollImmediate(1*time.Second, wait.ForeverTestTimeout, func() (bool, error) { 242 lSelector := discovery.LabelServiceName + "=" + svc.Name 243 esList, err := client.DiscoveryV1().EndpointSlices(ns.Name).List(ctx, metav1.ListOptions{LabelSelector: lSelector}) 244 if err != nil { 245 t.Logf("Error listing EndpointSlices: %v", err) 246 return false, nil 247 } 248 // there must be an endpoint slice per ipFamily 249 if len(esList.Items) != len(tc.ipFamilies) { 250 t.Logf("Waiting for EndpointSlice to be created %v", esList) 251 return false, nil 252 } 253 // there must be an endpoint address per each IP family 254 for _, ipFamily := range tc.ipFamilies { 255 found := false 256 for _, slice := range esList.Items { 257 // check if the endpoint addresses match the pod IPs 258 if len(slice.Endpoints) > 0 && len(slice.Endpoints[0].Addresses) > 0 { 259 if string(ipFamily) == string(slice.AddressType) && 260 slice.Endpoints[0].Addresses[0] == podIPbyFamily[ipFamily] { 261 found = true 262 break 263 } 264 } 265 t.Logf("Waiting endpoint slice to contain addresses") 266 } 267 if !found { 268 t.Logf("Endpoint slices does not contain PodIP %s", podIPbyFamily[ipFamily]) 269 return false, nil 270 } 271 } 272 return true, nil 273 }) 274 if err != nil { 275 t.Fatalf("Error waiting for endpoint slices: %v", err) 276 } 277 }) 278 } 279 }