k8s.io/kubernetes@v1.29.3/test/e2e/apimachinery/aggregator.go (about) 1 /* 2 Copyright 2017 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package apimachinery 18 19 import ( 20 "context" 21 "crypto/rand" 22 "encoding/json" 23 "fmt" 24 "math/big" 25 "net" 26 "strings" 27 "time" 28 29 appsv1 "k8s.io/api/apps/v1" 30 v1 "k8s.io/api/core/v1" 31 rbacv1 "k8s.io/api/rbac/v1" 32 apierrors "k8s.io/apimachinery/pkg/api/errors" 33 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 34 "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 35 "k8s.io/apimachinery/pkg/labels" 36 "k8s.io/apimachinery/pkg/runtime/schema" 37 "k8s.io/apimachinery/pkg/types" 38 "k8s.io/apimachinery/pkg/util/intstr" 39 "k8s.io/apimachinery/pkg/util/wait" 40 "k8s.io/client-go/discovery" 41 clientset "k8s.io/client-go/kubernetes" 42 "k8s.io/client-go/util/retry" 43 apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" 44 aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" 45 rbacv1helpers "k8s.io/kubernetes/pkg/apis/rbac/v1" 46 "k8s.io/kubernetes/test/e2e/framework" 47 e2eauth "k8s.io/kubernetes/test/e2e/framework/auth" 48 e2edeployment "k8s.io/kubernetes/test/e2e/framework/deployment" 49 e2epod "k8s.io/kubernetes/test/e2e/framework/pod" 50 "k8s.io/kubernetes/test/utils/format" 51 imageutils "k8s.io/kubernetes/test/utils/image" 52 admissionapi "k8s.io/pod-security-admission/api" 53 samplev1alpha1 "k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1" 54 "k8s.io/utils/pointer" 55 56 "github.com/onsi/ginkgo/v2" 57 "github.com/onsi/gomega" 58 ) 59 60 const ( 61 aggregatorServicePort = 7443 62 63 apiServiceRetryPeriod = 1 * time.Second 64 apiServiceRetryTimeout = 2 * time.Minute 65 66 defaultApiServiceGroupName = samplev1alpha1.GroupName 67 defaultApiServiceVersion = "v1alpha1" 68 ) 69 70 var _ = SIGDescribe("Aggregator", func() { 71 var aggrclient *aggregatorclient.Clientset 72 73 f := framework.NewDefaultFramework("aggregator") 74 f.NamespacePodSecurityLevel = admissionapi.LevelBaseline 75 76 // We want namespace initialization BeforeEach inserted by 77 // NewDefaultFramework to happen before this, so we put this BeforeEach 78 // after NewDefaultFramework. 79 ginkgo.BeforeEach(func() { 80 config, err := framework.LoadConfig() 81 if err != nil { 82 framework.Failf("could not load config: %v", err) 83 } 84 aggrclient, err = aggregatorclient.NewForConfig(config) 85 if err != nil { 86 framework.Failf("could not create aggregator client: %v", err) 87 } 88 apiServiceName := defaultApiServiceVersion + "." + defaultApiServiceGroupName 89 ginkgo.DeferCleanup(cleanupSampleAPIServer, f.ClientSet, aggrclient, generateSampleAPIServerObjectNames(f.Namespace.Name), apiServiceName) 90 }) 91 92 /* 93 Release: v1.17, v1.21, v1.27 94 Testname: aggregator-supports-the-sample-apiserver 95 Description: Ensure that the sample-apiserver code from 1.17 and compiled against 1.17 96 will work on the current Aggregator/API-Server. 97 */ 98 framework.ConformanceIt("Should be able to support the 1.17 Sample API Server using the current Aggregator", func(ctx context.Context) { 99 // Testing a 1.17 version of the sample-apiserver 100 TestSampleAPIServer(ctx, f, aggrclient, imageutils.GetE2EImage(imageutils.APIServer), defaultApiServiceGroupName, defaultApiServiceVersion) 101 }) 102 }) 103 104 func cleanupSampleAPIServer(ctx context.Context, client clientset.Interface, aggrclient *aggregatorclient.Clientset, n sampleAPIServerObjectNames, apiServiceName string) { 105 // delete the APIService first to avoid causing discovery errors 106 _ = aggrclient.ApiregistrationV1().APIServices().Delete(ctx, apiServiceName, metav1.DeleteOptions{}) 107 108 _ = client.AppsV1().Deployments(n.namespace).Delete(ctx, "sample-apiserver-deployment", metav1.DeleteOptions{}) 109 _ = client.CoreV1().Secrets(n.namespace).Delete(ctx, "sample-apiserver-secret", metav1.DeleteOptions{}) 110 _ = client.CoreV1().Services(n.namespace).Delete(ctx, "sample-api", metav1.DeleteOptions{}) 111 _ = client.CoreV1().ServiceAccounts(n.namespace).Delete(ctx, "sample-apiserver", metav1.DeleteOptions{}) 112 _ = client.RbacV1().RoleBindings("kube-system").Delete(ctx, n.roleBinding, metav1.DeleteOptions{}) 113 _ = client.RbacV1().ClusterRoleBindings().Delete(ctx, "wardler:"+n.namespace+":auth-delegator", metav1.DeleteOptions{}) 114 _ = client.RbacV1().ClusterRoles().Delete(ctx, n.clusterRole, metav1.DeleteOptions{}) 115 _ = client.RbacV1().ClusterRoleBindings().Delete(ctx, n.clusterRoleBinding, metav1.DeleteOptions{}) 116 } 117 118 type sampleAPIServerObjectNames struct { 119 namespace string 120 roleBinding string 121 clusterRole string 122 clusterRoleBinding string 123 } 124 125 func generateSampleAPIServerObjectNames(namespace string) sampleAPIServerObjectNames { 126 return sampleAPIServerObjectNames{ 127 namespace: namespace, 128 roleBinding: "wardler-auth-reader-" + namespace, 129 clusterRole: "sample-apiserver-reader-" + namespace, 130 clusterRoleBinding: "wardler:" + namespace + "sample-apiserver-reader-" + namespace, 131 } 132 } 133 134 func SetUpSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclient *aggregatorclient.Clientset, image string, n sampleAPIServerObjectNames, apiServiceGroupName, apiServiceVersion string) { 135 ginkgo.By("Registering the sample API server.") 136 client := f.ClientSet 137 restClient := client.Discovery().RESTClient() 138 certCtx := setupServerCert(n.namespace, "sample-api") 139 apiServiceName := apiServiceVersion + "." + apiServiceGroupName 140 141 // kubectl create -f namespace.yaml 142 // NOTE: aggregated apis should generally be set up in their own namespace. As the test framework is setting up a new namespace, we are just using that. 143 144 // kubectl create -f secret.yaml 145 secretName := "sample-apiserver-secret" 146 secret := &v1.Secret{ 147 ObjectMeta: metav1.ObjectMeta{ 148 Name: secretName, 149 }, 150 Type: v1.SecretTypeOpaque, 151 Data: map[string][]byte{ 152 "tls.crt": certCtx.cert, 153 "tls.key": certCtx.key, 154 }, 155 } 156 _, err := client.CoreV1().Secrets(n.namespace).Create(ctx, secret, metav1.CreateOptions{}) 157 framework.ExpectNoError(err, "creating secret %s in.namespace %s", secretName, n.namespace) 158 159 if e2eauth.IsRBACEnabled(ctx, client.RbacV1()) { 160 // kubectl create -f clusterrole.yaml 161 _, err = client.RbacV1().ClusterRoles().Create(ctx, &rbacv1.ClusterRole{ 162 163 ObjectMeta: metav1.ObjectMeta{Name: n.clusterRole}, 164 Rules: []rbacv1.PolicyRule{ 165 rbacv1helpers.NewRule("get", "list", "watch").Groups("").Resources("namespaces").RuleOrDie(), 166 rbacv1helpers.NewRule("get", "list", "watch").Groups("admissionregistration.k8s.io").Resources("*").RuleOrDie(), 167 }, 168 }, metav1.CreateOptions{}) 169 framework.ExpectNoError(err, "creating cluster role %s", n.clusterRole) 170 171 _, err = client.RbacV1().ClusterRoleBindings().Create(ctx, &rbacv1.ClusterRoleBinding{ 172 ObjectMeta: metav1.ObjectMeta{ 173 Name: n.clusterRoleBinding, 174 }, 175 RoleRef: rbacv1.RoleRef{ 176 APIGroup: "rbac.authorization.k8s.io", 177 Kind: "ClusterRole", 178 Name: n.clusterRole, 179 }, 180 Subjects: []rbacv1.Subject{ 181 { 182 APIGroup: "", 183 Kind: "ServiceAccount", 184 Name: "default", 185 Namespace: n.namespace, 186 }, 187 }, 188 }, metav1.CreateOptions{}) 189 framework.ExpectNoError(err, "creating cluster role binding %s", n.clusterRoleBinding) 190 191 // kubectl create -f authDelegator.yaml 192 _, err = client.RbacV1().ClusterRoleBindings().Create(ctx, &rbacv1.ClusterRoleBinding{ 193 ObjectMeta: metav1.ObjectMeta{ 194 Name: "wardler:" + n.namespace + ":auth-delegator", 195 }, 196 RoleRef: rbacv1.RoleRef{ 197 APIGroup: "rbac.authorization.k8s.io", 198 Kind: "ClusterRole", 199 Name: "system:auth-delegator", 200 }, 201 Subjects: []rbacv1.Subject{ 202 { 203 APIGroup: "", 204 Kind: "ServiceAccount", 205 Name: "default", 206 Namespace: n.namespace, 207 }, 208 }, 209 }, metav1.CreateOptions{}) 210 framework.ExpectNoError(err, "creating cluster role binding %s", "wardler:"+n.namespace+":auth-delegator") 211 } 212 213 // kubectl create -f deploy.yaml 214 deploymentName := "sample-apiserver-deployment" 215 etcdImage := imageutils.GetE2EImage(imageutils.Etcd) 216 podLabels := map[string]string{"app": "sample-apiserver", "apiserver": "true"} 217 replicas := int32(1) 218 etcdLocalhostAddress := "127.0.0.1" 219 if framework.TestContext.ClusterIsIPv6() { 220 etcdLocalhostAddress = "::1" 221 } 222 etcdURL := fmt.Sprintf("http://%s", net.JoinHostPort(etcdLocalhostAddress, "2379")) 223 224 mounts := []v1.VolumeMount{ 225 { 226 Name: "apiserver-certs", 227 ReadOnly: true, 228 MountPath: "/apiserver.local.config/certificates", 229 }, 230 } 231 volumes := []v1.Volume{ 232 { 233 Name: "apiserver-certs", 234 VolumeSource: v1.VolumeSource{ 235 Secret: &v1.SecretVolumeSource{SecretName: secretName}, 236 }, 237 }, 238 } 239 containers := []v1.Container{ 240 { 241 Name: "sample-apiserver", 242 VolumeMounts: mounts, 243 Args: []string{ 244 fmt.Sprintf("--etcd-servers=%s", etcdURL), 245 "--tls-cert-file=/apiserver.local.config/certificates/tls.crt", 246 "--tls-private-key-file=/apiserver.local.config/certificates/tls.key", 247 "--audit-log-path=-", 248 "--audit-log-maxage=0", 249 "--audit-log-maxbackup=0", 250 }, 251 Image: image, 252 ReadinessProbe: &v1.Probe{ 253 ProbeHandler: v1.ProbeHandler{ 254 HTTPGet: &v1.HTTPGetAction{ 255 Scheme: v1.URISchemeHTTPS, 256 Port: intstr.FromInt32(443), 257 Path: "/readyz", 258 }, 259 }, 260 InitialDelaySeconds: 20, 261 PeriodSeconds: 1, 262 SuccessThreshold: 1, 263 FailureThreshold: 3, 264 }, 265 }, 266 { 267 Name: "etcd", 268 Image: etcdImage, 269 Command: []string{ 270 "/usr/local/bin/etcd", 271 "--listen-client-urls", 272 etcdURL, 273 "--advertise-client-urls", 274 etcdURL, 275 }, 276 }, 277 } 278 d := e2edeployment.NewDeployment(deploymentName, replicas, podLabels, "", "", appsv1.RollingUpdateDeploymentStrategyType) 279 d.Spec.Template.Spec.Containers = containers 280 d.Spec.Template.Spec.Volumes = volumes 281 282 deployment, err := client.AppsV1().Deployments(n.namespace).Create(ctx, d, metav1.CreateOptions{}) 283 framework.ExpectNoError(err, "creating deployment %s in namespace %s", deploymentName, n.namespace) 284 285 err = e2edeployment.WaitForDeploymentRevisionAndImage(client, n.namespace, deploymentName, "1", image) 286 framework.ExpectNoError(err, "waiting for the deployment of image %s in %s in %s to complete", image, deploymentName, n.namespace) 287 288 err = e2edeployment.WaitForDeploymentRevisionAndImage(client, n.namespace, deploymentName, "1", etcdImage) 289 framework.ExpectNoError(err, "waiting for the deployment of image %s in %s in %s to complete", etcdImage, deploymentName, n.namespace) 290 291 // kubectl create -f service.yaml 292 serviceLabels := map[string]string{"apiserver": "true"} 293 service := &v1.Service{ 294 ObjectMeta: metav1.ObjectMeta{ 295 Namespace: n.namespace, 296 Name: "sample-api", 297 Labels: map[string]string{"test": "aggregator"}, 298 }, 299 Spec: v1.ServiceSpec{ 300 Selector: serviceLabels, 301 Ports: []v1.ServicePort{ 302 { 303 Protocol: v1.ProtocolTCP, 304 Port: aggregatorServicePort, 305 TargetPort: intstr.FromInt32(443), 306 }, 307 }, 308 }, 309 } 310 _, err = client.CoreV1().Services(n.namespace).Create(ctx, service, metav1.CreateOptions{}) 311 framework.ExpectNoError(err, "creating service %s in namespace %s", "sample-api", n.namespace) 312 313 // kubectl create -f serviceAccount.yaml 314 sa := &v1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "sample-apiserver"}} 315 _, err = client.CoreV1().ServiceAccounts(n.namespace).Create(ctx, sa, metav1.CreateOptions{}) 316 framework.ExpectNoError(err, "creating service account %s in namespace %s", "sample-apiserver", n.namespace) 317 318 if e2eauth.IsRBACEnabled(ctx, client.RbacV1()) { 319 // kubectl create -f auth-reader.yaml 320 _, err = client.RbacV1().RoleBindings("kube-system").Create(ctx, &rbacv1.RoleBinding{ 321 ObjectMeta: metav1.ObjectMeta{ 322 Name: n.roleBinding, 323 Annotations: map[string]string{ 324 rbacv1.AutoUpdateAnnotationKey: "true", 325 }, 326 }, 327 RoleRef: rbacv1.RoleRef{ 328 APIGroup: "", 329 Kind: "Role", 330 Name: "extension-apiserver-authentication-reader", 331 }, 332 Subjects: []rbacv1.Subject{ 333 { 334 Kind: "ServiceAccount", 335 Name: "default", 336 Namespace: n.namespace, 337 }, 338 }, 339 }, metav1.CreateOptions{}) 340 framework.ExpectNoError(err, "creating role binding %s in namespace %s", n.roleBinding, "kube-system") 341 } 342 343 // Wait for the extension apiserver to be up and healthy 344 // kubectl get deployments -n <aggregated-api-namespace> && status == Running 345 // NOTE: aggregated apis should generally be set up in their own namespace (<aggregated-api-namespace>). As the test framework 346 // is setting up a new namespace, we are just using that. 347 err = e2edeployment.WaitForDeploymentComplete(client, deployment) 348 framework.ExpectNoError(err, "deploying extension apiserver in namespace %s", n.namespace) 349 350 // kubectl create -f apiservice.yaml 351 _, err = aggrclient.ApiregistrationV1().APIServices().Create(ctx, &apiregistrationv1.APIService{ 352 ObjectMeta: metav1.ObjectMeta{Name: apiServiceName}, 353 Spec: apiregistrationv1.APIServiceSpec{ 354 Service: &apiregistrationv1.ServiceReference{ 355 Namespace: n.namespace, 356 Name: "sample-api", 357 Port: pointer.Int32(aggregatorServicePort), 358 }, 359 Group: apiServiceGroupName, 360 Version: apiServiceVersion, 361 CABundle: certCtx.signingCert, 362 GroupPriorityMinimum: 2000, 363 VersionPriority: 200, 364 }, 365 }, metav1.CreateOptions{}) 366 framework.ExpectNoError(err, "creating apiservice %s", apiServiceName) 367 368 var ( 369 currentAPIService *apiregistrationv1.APIService 370 currentPods *v1.PodList 371 ) 372 373 err = pollTimed(ctx, 100*time.Millisecond, 60*time.Second, func(ctx context.Context) (bool, error) { 374 375 currentAPIService, _ = aggrclient.ApiregistrationV1().APIServices().Get(ctx, apiServiceName, metav1.GetOptions{}) 376 currentPods, _ = client.CoreV1().Pods(n.namespace).List(ctx, metav1.ListOptions{}) 377 378 request := restClient.Get().AbsPath("/apis/" + apiServiceGroupName + "/" + apiServiceVersion + "/namespaces/default/flunders") 379 request.SetHeader("Accept", "application/json") 380 _, err := request.DoRaw(ctx) 381 if err != nil { 382 status, ok := err.(*apierrors.StatusError) 383 if !ok { 384 return false, err 385 } 386 if status.Status().Code == 403 || status.Status().Code == 503 { 387 return false, nil 388 } 389 if status.Status().Code == 404 && strings.HasPrefix(err.Error(), "the server could not find the requested resource") { 390 return false, nil 391 } 392 return false, err 393 } 394 return true, nil 395 }, "Waited %s for the sample-apiserver to be ready to handle requests.") 396 if err != nil { 397 currentAPIServiceJSON, _ := json.Marshal(currentAPIService) 398 framework.Logf("current APIService: %s", string(currentAPIServiceJSON)) 399 400 currentPodsJSON, _ := json.Marshal(currentPods) 401 framework.Logf("current pods: %s", string(currentPodsJSON)) 402 403 if currentPods != nil { 404 for _, pod := range currentPods.Items { 405 for _, container := range pod.Spec.Containers { 406 logs, err := e2epod.GetPodLogs(ctx, client, n.namespace, pod.Name, container.Name) 407 framework.Logf("logs of %s/%s (error: %v): %s", pod.Name, container.Name, err, logs) 408 } 409 } 410 } 411 } 412 framework.ExpectNoError(err, "gave up waiting for apiservice wardle to come up successfully") 413 } 414 415 // TestSampleAPIServer is a basic test if the sample-apiserver code from 1.10 and compiled against 1.10 416 // will work on the current Aggregator/API-Server. 417 func TestSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclient *aggregatorclient.Clientset, image, apiServiceGroupName, apiServiceVersion string) { 418 n := generateSampleAPIServerObjectNames(f.Namespace.Name) 419 SetUpSampleAPIServer(ctx, f, aggrclient, image, n, apiServiceGroupName, apiServiceVersion) 420 client := f.ClientSet 421 restClient := client.Discovery().RESTClient() 422 423 flunderName := generateFlunderName("rest-flunder") 424 apiServiceName := apiServiceVersion + "." + apiServiceGroupName 425 426 // kubectl create -f flunders-1.yaml -v 9 427 // curl -k -v -XPOST https://localhost/apis/wardle.example.com/v1alpha1/namespaces/default/flunders 428 // Request Body: {"apiVersion":"wardle.example.com/v1alpha1","kind":"Flunder","metadata":{"labels":{"sample-label":"true"},"name":"test-flunder","namespace":"default"}} 429 flunder := `{"apiVersion":"` + apiServiceGroupName + `/` + apiServiceVersion + `","kind":"Flunder","metadata":{"labels":{"sample-label":"true"},"name":"` + flunderName + `","namespace":"default"}}` 430 result := restClient.Post().AbsPath("/apis/"+apiServiceGroupName+"/"+apiServiceVersion+"/namespaces/default/flunders").Body([]byte(flunder)).SetHeader("Accept", "application/json").Do(ctx) 431 framework.ExpectNoError(result.Error(), "creating a new flunders resource") 432 var statusCode int 433 result.StatusCode(&statusCode) 434 if statusCode != 201 { 435 framework.Failf("Flunders client creation response was status %d, not 201", statusCode) 436 } 437 u := &unstructured.Unstructured{} 438 if err := result.Into(u); err != nil { 439 framework.ExpectNoError(err, "reading created response") 440 } 441 442 gomega.Expect(u.GetAPIVersion()).To(gomega.Equal(apiServiceGroupName + "/" + apiServiceVersion)) 443 gomega.Expect(u.GetKind()).To(gomega.Equal("Flunder")) 444 gomega.Expect(u.GetName()).To(gomega.Equal(flunderName)) 445 446 pods, err := client.CoreV1().Pods(n.namespace).List(ctx, metav1.ListOptions{}) 447 framework.ExpectNoError(err, "getting pods for flunders service") 448 449 // kubectl get flunders -v 9 450 // curl -k -v -XGET https://localhost/apis/wardle.example.com/v1alpha1/namespaces/default/flunders 451 contents, err := restClient.Get().AbsPath("/apis/"+apiServiceGroupName+"/"+apiServiceVersion+"/namespaces/default/flunders").SetHeader("Accept", "application/json").DoRaw(ctx) 452 framework.ExpectNoError(err, "attempting to get a newly created flunders resource") 453 var flundersList samplev1alpha1.FlunderList 454 err = json.Unmarshal(contents, &flundersList) 455 validateErrorWithDebugInfo(ctx, f, err, pods, "Error in unmarshalling %T response from server %s", contents, "/apis/"+apiServiceGroupName+"/"+apiServiceVersion) 456 if len(flundersList.Items) != 1 { 457 framework.Failf("failed to get back the correct flunders list %v", flundersList) 458 } 459 460 // kubectl delete flunder test-flunder -v 9 461 // curl -k -v -XDELETE https://35.193.112.40/apis/wardle.example.com/v1alpha1/namespaces/default/flunders/test-flunder 462 _, err = restClient.Delete().AbsPath("/apis/" + apiServiceGroupName + "/" + apiServiceVersion + "/namespaces/default/flunders/" + flunderName).DoRaw(ctx) 463 validateErrorWithDebugInfo(ctx, f, err, pods, "attempting to delete a newly created flunders(%v) resource", flundersList.Items) 464 465 // kubectl get flunders -v 9 466 // curl -k -v -XGET https://localhost/apis/wardle.example.com/v1alpha1/namespaces/default/flunders 467 contents, err = restClient.Get().AbsPath("/apis/"+apiServiceGroupName+"/"+apiServiceVersion+"/namespaces/default/flunders").SetHeader("Accept", "application/json").DoRaw(ctx) 468 framework.ExpectNoError(err, "confirming delete of a newly created flunders resource") 469 err = json.Unmarshal(contents, &flundersList) 470 validateErrorWithDebugInfo(ctx, f, err, pods, "Error in unmarshalling %T response from server %s", contents, "/apis/"+apiServiceGroupName+"/"+apiServiceVersion) 471 if len(flundersList.Items) != 0 { 472 framework.Failf("failed to get back the correct deleted flunders list %v", flundersList) 473 } 474 475 flunderName = generateFlunderName("dynamic-flunder") 476 477 // Rerun the Create/List/Delete tests using the Dynamic client. 478 resources, discoveryErr := client.Discovery().ServerPreferredNamespacedResources() 479 groupVersionResources, err := discovery.GroupVersionResources(resources) 480 framework.ExpectNoError(err, "getting group version resources for dynamic client") 481 gvr := schema.GroupVersionResource{Group: apiServiceGroupName, Version: apiServiceVersion, Resource: "flunders"} 482 _, ok := groupVersionResources[gvr] 483 if !ok { 484 framework.Failf("could not find group version resource for dynamic client and wardle/flunders (discovery error: %v, discovery results: %#v)", discoveryErr, groupVersionResources) 485 } 486 dynamicClient := f.DynamicClient.Resource(gvr).Namespace(n.namespace) 487 488 // kubectl create -f flunders-1.yaml 489 // Request Body: {"apiVersion":"wardle.example.com/v1alpha1","kind":"Flunder","metadata":{"labels":{"sample-label":"true"},"name":"test-flunder","namespace":"default"}} 490 testFlunder := samplev1alpha1.Flunder{ 491 TypeMeta: metav1.TypeMeta{ 492 Kind: "Flunder", 493 APIVersion: apiServiceGroupName + "/" + apiServiceVersion, 494 }, 495 ObjectMeta: metav1.ObjectMeta{Name: flunderName}, 496 Spec: samplev1alpha1.FlunderSpec{}, 497 } 498 jsonFlunder, err := json.Marshal(testFlunder) 499 framework.ExpectNoError(err, "marshalling test-flunder for create using dynamic client") 500 unstruct := &unstructured.Unstructured{} 501 err = unstruct.UnmarshalJSON(jsonFlunder) 502 framework.ExpectNoError(err, "unmarshalling test-flunder as unstructured for create using dynamic client") 503 _, err = dynamicClient.Create(ctx, unstruct, metav1.CreateOptions{}) 504 framework.ExpectNoError(err, "listing flunders using dynamic client") 505 506 // kubectl get flunders 507 unstructuredList, err := dynamicClient.List(ctx, metav1.ListOptions{}) 508 framework.ExpectNoError(err, "listing flunders using dynamic client") 509 if len(unstructuredList.Items) != 1 { 510 framework.Failf("failed to get back the correct flunders list %v from the dynamic client", unstructuredList) 511 } 512 513 ginkgo.By("Read Status for " + apiServiceName) 514 statusContent, err := restClient.Get(). 515 AbsPath("/apis/apiregistration.k8s.io/v1/apiservices/"+apiServiceName+"/status"). 516 SetHeader("Accept", "application/json").DoRaw(ctx) 517 framework.ExpectNoError(err, "No response for .../apiservices/"+apiServiceName+"/status. Error: %v", err) 518 519 var jr *apiregistrationv1.APIService 520 err = json.Unmarshal([]byte(statusContent), &jr) 521 framework.ExpectNoError(err, "Failed to process statusContent: %v | err: %v ", string(statusContent), err) 522 gomega.Expect(jr.Status.Conditions[0].Message).To(gomega.Equal("all checks passed"), "The Message returned was %v", jr.Status.Conditions[0].Message) 523 524 ginkgo.By("kubectl patch apiservice " + apiServiceName + " -p '{\"spec\":{\"versionPriority\": 400}}'") 525 patchContent, err := restClient.Patch(types.MergePatchType). 526 AbsPath("/apis/apiregistration.k8s.io/v1/apiservices/"+apiServiceName). 527 SetHeader("Accept", "application/json"). 528 Body([]byte(`{"spec":{"versionPriority": 400}}`)).DoRaw(ctx) 529 530 framework.ExpectNoError(err, "Patch failed for .../apiservices/"+apiServiceName+". Error: %v", err) 531 err = json.Unmarshal([]byte(patchContent), &jr) 532 framework.ExpectNoError(err, "Failed to process patchContent: %v | err: %v ", string(patchContent), err) 533 gomega.Expect(jr.Spec.VersionPriority).To(gomega.Equal(int32(400)), "The VersionPriority returned was %d", jr.Spec.VersionPriority) 534 535 ginkgo.By("List APIServices") 536 listApiservices, err := restClient.Get(). 537 AbsPath("/apis/apiregistration.k8s.io/v1/apiservices"). 538 SetHeader("Accept", "application/json").DoRaw(ctx) 539 540 framework.ExpectNoError(err, "No response for /apis/apiregistration.k8s.io/v1/apiservices Error: %v", err) 541 542 var list *apiregistrationv1.APIServiceList 543 err = json.Unmarshal([]byte(listApiservices), &list) 544 framework.ExpectNoError(err, "Failed to process APIServiceList: %v | err: %v ", list, err) 545 546 locatedWardle := false 547 for _, item := range list.Items { 548 if item.Name == apiServiceName { 549 framework.Logf("Found " + apiServiceName + " in APIServiceList") 550 locatedWardle = true 551 break 552 } 553 } 554 if !locatedWardle { 555 framework.Failf("Unable to find " + apiServiceName + " in APIServiceList") 556 } 557 558 // As the APIService doesn't have any labels currently set we need to 559 // set one so that we can select it later when we call deleteCollection 560 ginkgo.By("Adding a label to the APIService") 561 apiServiceClient := aggrclient.ApiregistrationV1().APIServices() 562 apiServiceLabel := map[string]string{"e2e-apiservice": "patched"} 563 apiServicePatch, err := json.Marshal(map[string]interface{}{ 564 "metadata": map[string]interface{}{ 565 "labels": apiServiceLabel, 566 }, 567 }) 568 framework.ExpectNoError(err, "failed to Marshal APIService JSON patch") 569 _, err = apiServiceClient.Patch(ctx, apiServiceName, types.StrategicMergePatchType, []byte(apiServicePatch), metav1.PatchOptions{}) 570 framework.ExpectNoError(err, "failed to patch APIService") 571 572 patchedApiService, err := apiServiceClient.Get(ctx, apiServiceName, metav1.GetOptions{}) 573 framework.ExpectNoError(err, "Unable to retrieve api service %s", apiServiceName) 574 framework.Logf("APIService labels: %v", patchedApiService.Labels) 575 576 ginkgo.By("Updating APIService Status") 577 var updatedStatus, wardle *apiregistrationv1.APIService 578 579 err = retry.RetryOnConflict(retry.DefaultRetry, func() error { 580 var statusToUpdate *apiregistrationv1.APIService 581 statusContent, err = restClient.Get(). 582 AbsPath("/apis/apiregistration.k8s.io/v1/apiservices/"+apiServiceName+"/status"). 583 SetHeader("Accept", "application/json").DoRaw(ctx) 584 framework.ExpectNoError(err, "No response for .../apiservices/"+apiServiceName+"/status. Error: %v", err) 585 586 err = json.Unmarshal([]byte(statusContent), &statusToUpdate) 587 framework.ExpectNoError(err, "Failed to process statusContent: %v | err: %v ", string(statusContent), err) 588 589 statusToUpdate.Status.Conditions = append(statusToUpdate.Status.Conditions, apiregistrationv1.APIServiceCondition{ 590 Type: "StatusUpdated", 591 Status: "True", 592 Reason: "E2E", 593 Message: "Set from e2e test", 594 }) 595 596 updatedStatus, err = apiServiceClient.UpdateStatus(ctx, statusToUpdate, metav1.UpdateOptions{}) 597 return err 598 }) 599 framework.ExpectNoError(err, "Failed to update status. %v", err) 600 framework.Logf("updatedStatus.Conditions: %#v", updatedStatus.Status.Conditions) 601 602 ginkgo.By("Confirm that " + apiServiceName + " /status was updated") 603 statusContent, err = restClient.Get(). 604 AbsPath("/apis/apiregistration.k8s.io/v1/apiservices/"+apiServiceName+"/status"). 605 SetHeader("Accept", "application/json").DoRaw(ctx) 606 framework.ExpectNoError(err, "No response for .../apiservices/"+apiServiceName+"/status. Error: %v", err) 607 608 err = json.Unmarshal([]byte(statusContent), &wardle) 609 framework.ExpectNoError(err, "Failed to process statusContent: %v | err: %v ", string(statusContent), err) 610 611 foundUpdatedStatusCondition := false 612 for _, cond := range wardle.Status.Conditions { 613 if cond.Type == "StatusUpdated" && cond.Reason == "E2E" && cond.Message == "Set from e2e test" { 614 framework.Logf("Found APIService %v with Labels: %v & Condition: %v", wardle.ObjectMeta.Name, wardle.Labels, cond) 615 foundUpdatedStatusCondition = true 616 break 617 } else { 618 framework.Logf("Observed APIService %v with Labels: %v & Condition: %v", wardle.ObjectMeta.Name, wardle.Labels, cond) 619 } 620 } 621 if !foundUpdatedStatusCondition { 622 framework.Failf("The updated status condition was not found in:\n%s", format.Object(wardle.Status.Conditions, 1)) 623 } 624 framework.Logf("Found updated status condition for %s", wardle.ObjectMeta.Name) 625 626 ginkgo.By(fmt.Sprintf("Replace APIService %s", apiServiceName)) 627 var updatedApiService *apiregistrationv1.APIService 628 629 err = retry.RetryOnConflict(retry.DefaultRetry, func() error { 630 currentApiService, err := apiServiceClient.Get(ctx, apiServiceName, metav1.GetOptions{}) 631 framework.ExpectNoError(err, "Unable to get APIService %s", apiServiceName) 632 currentApiService.Labels = map[string]string{ 633 apiServiceName: "updated", 634 } 635 updatedApiService, err = apiServiceClient.Update(ctx, currentApiService, metav1.UpdateOptions{}) 636 return err 637 }) 638 framework.ExpectNoError(err) 639 gomega.Expect(updatedApiService.Labels).To(gomega.HaveKeyWithValue(apiServiceName, "updated"), "should have the updated label but have %q", updatedApiService.Labels[apiServiceName]) 640 framework.Logf("Found updated apiService label for %q", apiServiceName) 641 642 // kubectl delete flunder test-flunder 643 ginkgo.By(fmt.Sprintf("Delete flunders resource %q", flunderName)) 644 err = dynamicClient.Delete(ctx, flunderName, metav1.DeleteOptions{}) 645 validateErrorWithDebugInfo(ctx, f, err, pods, "deleting flunders(%v) using dynamic client", unstructuredList.Items) 646 647 // kubectl get flunders 648 unstructuredList, err = dynamicClient.List(ctx, metav1.ListOptions{}) 649 framework.ExpectNoError(err, "listing flunders using dynamic client") 650 if len(unstructuredList.Items) != 0 { 651 framework.Failf("failed to get back the correct deleted flunders list %v from the dynamic client", unstructuredList) 652 } 653 654 ginkgo.By("Recreating test-flunder before removing endpoint via deleteCollection") 655 jsonFlunder, err = json.Marshal(testFlunder) 656 framework.ExpectNoError(err, "marshalling test-flunder for create using dynamic client") 657 unstruct = &unstructured.Unstructured{} 658 err = unstruct.UnmarshalJSON(jsonFlunder) 659 framework.ExpectNoError(err, "unmarshalling test-flunder as unstructured for create using dynamic client") 660 _, err = dynamicClient.Create(ctx, unstruct, metav1.CreateOptions{}) 661 framework.ExpectNoError(err, "listing flunders using dynamic client") 662 663 // kubectl get flunders 664 unstructuredList, err = dynamicClient.List(ctx, metav1.ListOptions{}) 665 framework.ExpectNoError(err, "listing flunders using dynamic client") 666 if len(unstructuredList.Items) != 1 { 667 framework.Failf("failed to get back the correct flunders list %v from the dynamic client", unstructuredList) 668 } 669 670 ginkgo.By("Read " + apiServiceName + " /status before patching it") 671 statusContent, err = restClient.Get(). 672 AbsPath("/apis/apiregistration.k8s.io/v1/apiservices/"+apiServiceName+"/status"). 673 SetHeader("Accept", "application/json").DoRaw(ctx) 674 framework.ExpectNoError(err, "No response for .../apiservices/"+apiServiceName+"/status. Error: %v", err) 675 676 wardle.Reset() 677 err = json.Unmarshal([]byte(statusContent), &wardle) 678 framework.ExpectNoError(err, "Failed to process statusContent: %v | err: %v ", string(statusContent), err) 679 680 ginkgo.By("Patch APIService Status") 681 patch := map[string]interface{}{ 682 "status": map[string]interface{}{ 683 "conditions": append(wardle.Status.Conditions, apiregistrationv1.APIServiceCondition{ 684 Type: "StatusPatched", 685 Status: "True", 686 Reason: "E2E", 687 Message: "Set by e2e test", 688 }), 689 }, 690 } 691 payload, err := json.Marshal(patch) 692 framework.ExpectNoError(err, "Failed to marshal JSON. %v", err) 693 694 _, err = restClient.Patch(types.MergePatchType). 695 AbsPath("/apis/apiregistration.k8s.io/v1/apiservices/"+apiServiceName+"/status"). 696 SetHeader("Accept", "application/json"). 697 Body([]byte(payload)). 698 DoRaw(ctx) 699 framework.ExpectNoError(err, "Patch failed for .../apiservices/"+apiServiceName+"/status. Error: %v", err) 700 701 ginkgo.By("Confirm that " + apiServiceName + " /status was patched") 702 statusContent, err = restClient.Get(). 703 AbsPath("/apis/apiregistration.k8s.io/v1/apiservices/"+apiServiceName+"/status"). 704 SetHeader("Accept", "application/json").DoRaw(ctx) 705 framework.ExpectNoError(err, "No response for .../apiservices/"+apiServiceName+"/status. Error: %v", err) 706 707 wardle.Reset() 708 err = json.Unmarshal([]byte(statusContent), &wardle) 709 framework.ExpectNoError(err, "Failed to process statusContent: %v | err: %v ", string(statusContent), err) 710 711 foundPatchedStatusCondition := false 712 for _, cond := range wardle.Status.Conditions { 713 if cond.Type == "StatusPatched" && cond.Reason == "E2E" && cond.Message == "Set by e2e test" { 714 framework.Logf("Found APIService %v with Labels: %v & Conditions: %v", wardle.ObjectMeta.Name, wardle.Labels, cond) 715 foundPatchedStatusCondition = true 716 break 717 } else { 718 framework.Logf("Observed APIService %v with Labels: %v & Conditions: %v", wardle.ObjectMeta.Name, wardle.Labels, cond) 719 } 720 } 721 if !foundPatchedStatusCondition { 722 framework.Failf("The patched status condition was not found in:\n%s", format.Object(wardle.Status.Conditions, 1)) 723 } 724 framework.Logf("Found patched status condition for %s", wardle.ObjectMeta.Name) 725 726 apiServiceLabelSelector := labels.SelectorFromSet(updatedApiService.Labels).String() 727 ginkgo.By(fmt.Sprintf("APIService deleteCollection with labelSelector: %q", apiServiceLabelSelector)) 728 729 err = aggrclient.ApiregistrationV1().APIServices().DeleteCollection(ctx, 730 metav1.DeleteOptions{}, 731 metav1.ListOptions{LabelSelector: apiServiceLabelSelector}) 732 framework.ExpectNoError(err, "Unable to delete apiservice %s", apiServiceName) 733 734 ginkgo.By("Confirm that the generated APIService has been deleted") 735 err = wait.PollImmediate(apiServiceRetryPeriod, apiServiceRetryTimeout, checkApiServiceListQuantity(ctx, aggrclient, apiServiceLabelSelector, 0)) 736 framework.ExpectNoError(err, "failed to count the required APIServices") 737 framework.Logf("APIService %s has been deleted.", apiServiceName) 738 739 cleanupSampleAPIServer(ctx, client, aggrclient, n, apiServiceName) 740 } 741 742 // pollTimed will call Poll but time how long Poll actually took. 743 // It will then framework.Logf the msg with the duration of the Poll. 744 // It is assumed that msg will contain one %s for the elapsed time. 745 func pollTimed(ctx context.Context, interval, timeout time.Duration, condition wait.ConditionWithContextFunc, msg string) error { 746 defer func(start time.Time, msg string) { 747 elapsed := time.Since(start) 748 framework.Logf(msg, elapsed) 749 }(time.Now(), msg) 750 return wait.PollWithContext(ctx, interval, timeout, condition) 751 } 752 753 func validateErrorWithDebugInfo(ctx context.Context, f *framework.Framework, err error, pods *v1.PodList, msg string, fields ...interface{}) { 754 if err != nil { 755 namespace := f.Namespace.Name 756 msg := fmt.Sprintf(msg, fields...) 757 msg += fmt.Sprintf(" but received unexpected error:\n%v", err) 758 client := f.ClientSet 759 ep, err := client.CoreV1().Endpoints(namespace).Get(ctx, "sample-api", metav1.GetOptions{}) 760 if err == nil { 761 msg += fmt.Sprintf("\nFound endpoints for sample-api:\n%v", ep) 762 } 763 pds, err := client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{}) 764 if err == nil { 765 msg += fmt.Sprintf("\nFound pods in %s:\n%v", namespace, pds) 766 msg += fmt.Sprintf("\nOriginal pods in %s:\n%v", namespace, pods) 767 } 768 769 framework.Failf(msg) 770 } 771 } 772 773 func generateFlunderName(base string) string { 774 id, err := rand.Int(rand.Reader, big.NewInt(2147483647)) 775 if err != nil { 776 return base 777 } 778 return fmt.Sprintf("%s-%d", base, id) 779 } 780 781 func checkApiServiceListQuantity(ctx context.Context, aggrclient *aggregatorclient.Clientset, label string, quantity int) func() (bool, error) { 782 return func() (bool, error) { 783 var err error 784 785 framework.Logf("Requesting list of APIServices to confirm quantity") 786 787 list, err := aggrclient.ApiregistrationV1().APIServices().List(ctx, metav1.ListOptions{LabelSelector: label}) 788 if err != nil { 789 return false, err 790 } 791 792 if len(list.Items) != quantity { 793 return false, err 794 } 795 framework.Logf("Found %d APIService with label %q", quantity, label) 796 return true, nil 797 } 798 }