k8s.io/kubernetes@v1.29.3/test/e2e/apimachinery/webhook.go (about) 1 /* 2 Copyright 2017 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package apimachinery 18 19 import ( 20 "context" 21 "encoding/json" 22 "fmt" 23 "reflect" 24 "strings" 25 "time" 26 27 "k8s.io/utils/pointer" 28 29 admissionregistrationv1 "k8s.io/api/admissionregistration/v1" 30 appsv1 "k8s.io/api/apps/v1" 31 v1 "k8s.io/api/core/v1" 32 rbacv1 "k8s.io/api/rbac/v1" 33 apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" 34 crdclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" 35 apierrors "k8s.io/apimachinery/pkg/api/errors" 36 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 37 "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 38 "k8s.io/apimachinery/pkg/types" 39 "k8s.io/apimachinery/pkg/util/intstr" 40 "k8s.io/apimachinery/pkg/util/uuid" 41 "k8s.io/apimachinery/pkg/util/wait" 42 "k8s.io/client-go/dynamic" 43 clientset "k8s.io/client-go/kubernetes" 44 "k8s.io/client-go/util/retry" 45 "k8s.io/kubernetes/test/e2e/framework" 46 e2edeployment "k8s.io/kubernetes/test/e2e/framework/deployment" 47 e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl" 48 e2epod "k8s.io/kubernetes/test/e2e/framework/pod" 49 "k8s.io/kubernetes/test/utils/crd" 50 imageutils "k8s.io/kubernetes/test/utils/image" 51 admissionapi "k8s.io/pod-security-admission/api" 52 53 "github.com/onsi/ginkgo/v2" 54 "github.com/onsi/gomega" 55 56 // ensure libs have a chance to initialize 57 _ "github.com/stretchr/testify/assert" 58 ) 59 60 const ( 61 secretName = "sample-webhook-secret" 62 deploymentName = "sample-webhook-deployment" 63 serviceName = "e2e-test-webhook" 64 roleBindingName = "webhook-auth-reader" 65 66 skipNamespaceLabelKey = "skip-webhook-admission" 67 skipNamespaceLabelValue = "yes" 68 skipNamespaceBaseName = "exempted-namespace" 69 disallowedPodName = "disallowed-pod" 70 toBeAttachedPodName = "to-be-attached-pod" 71 hangingPodName = "hanging-pod" 72 disallowedConfigMapName = "disallowed-configmap" 73 allowedConfigMapName = "allowed-configmap" 74 failNamespaceLabelKey = "fail-closed-webhook" 75 failNamespaceLabelValue = "yes" 76 failNamespaceBaseName = "fail-closed-namespace" 77 addedLabelKey = "added-label" 78 addedLabelValue = "yes" 79 ) 80 81 var _ = SIGDescribe("AdmissionWebhook [Privileged:ClusterAdmin]", func() { 82 var certCtx *certContext 83 f := framework.NewDefaultFramework("webhook") 84 f.NamespacePodSecurityLevel = admissionapi.LevelBaseline 85 servicePort := int32(8443) 86 containerPort := int32(8444) 87 88 var client clientset.Interface 89 var namespaceName string 90 var markersNamespaceName string 91 92 ginkgo.BeforeEach(func(ctx context.Context) { 93 client = f.ClientSet 94 namespaceName = f.Namespace.Name 95 96 // Make sure the namespace created for the test is labeled to be selected by the webhooks 97 labelNamespace(ctx, f, f.Namespace.Name) 98 markersNamespaceName = createWebhookConfigurationReadyNamespace(ctx, f) 99 100 ginkgo.By("Setting up server cert") 101 certCtx = setupServerCert(namespaceName, serviceName) 102 createAuthReaderRoleBinding(ctx, f, namespaceName) 103 104 deployWebhookAndService(ctx, f, imageutils.GetE2EImage(imageutils.Agnhost), certCtx, servicePort, containerPort) 105 }) 106 107 ginkgo.AfterEach(func(ctx context.Context) { 108 cleanWebhookTest(ctx, client, namespaceName) 109 }) 110 111 /* 112 Release: v1.16 113 Testname: Admission webhook, discovery document 114 Description: The admissionregistration.k8s.io API group MUST exists in the /apis discovery document. 115 The admissionregistration.k8s.io/v1 API group/version MUST exists in the /apis discovery document. 116 The mutatingwebhookconfigurations and validatingwebhookconfigurations resources MUST exist in the 117 /apis/admissionregistration.k8s.io/v1 discovery document. 118 */ 119 framework.ConformanceIt("should include webhook resources in discovery documents", func(ctx context.Context) { 120 { 121 ginkgo.By("fetching the /apis discovery document") 122 apiGroupList := &metav1.APIGroupList{} 123 err := client.Discovery().RESTClient().Get().AbsPath("/apis").Do(ctx).Into(apiGroupList) 124 framework.ExpectNoError(err, "fetching /apis") 125 126 ginkgo.By("finding the admissionregistration.k8s.io API group in the /apis discovery document") 127 var group *metav1.APIGroup 128 for _, g := range apiGroupList.Groups { 129 if g.Name == admissionregistrationv1.GroupName { 130 group = &g 131 break 132 } 133 } 134 framework.ExpectNotEqual(group, nil, "admissionregistration.k8s.io API group not found in /apis discovery document") 135 136 ginkgo.By("finding the admissionregistration.k8s.io/v1 API group/version in the /apis discovery document") 137 var version *metav1.GroupVersionForDiscovery 138 for _, v := range group.Versions { 139 if v.Version == admissionregistrationv1.SchemeGroupVersion.Version { 140 version = &v 141 break 142 } 143 } 144 framework.ExpectNotEqual(version, nil, "admissionregistration.k8s.io/v1 API group version not found in /apis discovery document") 145 } 146 147 { 148 ginkgo.By("fetching the /apis/admissionregistration.k8s.io discovery document") 149 group := &metav1.APIGroup{} 150 err := client.Discovery().RESTClient().Get().AbsPath("/apis/admissionregistration.k8s.io").Do(ctx).Into(group) 151 framework.ExpectNoError(err, "fetching /apis/admissionregistration.k8s.io") 152 gomega.Expect(group.Name).To(gomega.Equal(admissionregistrationv1.GroupName), "verifying API group name in /apis/admissionregistration.k8s.io discovery document") 153 154 ginkgo.By("finding the admissionregistration.k8s.io/v1 API group/version in the /apis/admissionregistration.k8s.io discovery document") 155 var version *metav1.GroupVersionForDiscovery 156 for _, v := range group.Versions { 157 if v.Version == admissionregistrationv1.SchemeGroupVersion.Version { 158 version = &v 159 break 160 } 161 } 162 framework.ExpectNotEqual(version, nil, "admissionregistration.k8s.io/v1 API group version not found in /apis/admissionregistration.k8s.io discovery document") 163 } 164 165 { 166 ginkgo.By("fetching the /apis/admissionregistration.k8s.io/v1 discovery document") 167 apiResourceList := &metav1.APIResourceList{} 168 err := client.Discovery().RESTClient().Get().AbsPath("/apis/admissionregistration.k8s.io/v1").Do(ctx).Into(apiResourceList) 169 framework.ExpectNoError(err, "fetching /apis/admissionregistration.k8s.io/v1") 170 gomega.Expect(apiResourceList.GroupVersion).To(gomega.Equal(admissionregistrationv1.SchemeGroupVersion.String()), "verifying API group/version in /apis/admissionregistration.k8s.io/v1 discovery document") 171 172 ginkgo.By("finding mutatingwebhookconfigurations and validatingwebhookconfigurations resources in the /apis/admissionregistration.k8s.io/v1 discovery document") 173 var ( 174 mutatingWebhookResource *metav1.APIResource 175 validatingWebhookResource *metav1.APIResource 176 ) 177 for i := range apiResourceList.APIResources { 178 if apiResourceList.APIResources[i].Name == "mutatingwebhookconfigurations" { 179 mutatingWebhookResource = &apiResourceList.APIResources[i] 180 } 181 if apiResourceList.APIResources[i].Name == "validatingwebhookconfigurations" { 182 validatingWebhookResource = &apiResourceList.APIResources[i] 183 } 184 } 185 framework.ExpectNotEqual(mutatingWebhookResource, nil, "mutatingwebhookconfigurations resource not found in /apis/admissionregistration.k8s.io/v1 discovery document") 186 framework.ExpectNotEqual(validatingWebhookResource, nil, "validatingwebhookconfigurations resource not found in /apis/admissionregistration.k8s.io/v1 discovery document") 187 } 188 }) 189 190 /* 191 Release: v1.16 192 Testname: Admission webhook, deny create 193 Description: Register an admission webhook configuration that admits pod and configmap. Attempts to create 194 non-compliant pods and configmaps, or update/patch compliant pods and configmaps to be non-compliant MUST 195 be denied. An attempt to create a pod that causes a webhook to hang MUST result in a webhook timeout error, 196 and the pod creation MUST be denied. An attempt to create a non-compliant configmap in a whitelisted 197 namespace based on the webhook namespace selector MUST be allowed. 198 */ 199 framework.ConformanceIt("should be able to deny pod and configmap creation", func(ctx context.Context) { 200 registerWebhook(ctx, f, markersNamespaceName, f.UniqueName, certCtx, servicePort) 201 testWebhook(ctx, f) 202 }) 203 204 /* 205 Release: v1.16 206 Testname: Admission webhook, deny attach 207 Description: Register an admission webhook configuration that denies connecting to a pod's attach sub-resource. 208 Attempts to attach MUST be denied. 209 */ 210 framework.ConformanceIt("should be able to deny attaching pod", func(ctx context.Context) { 211 registerWebhookForAttachingPod(ctx, f, markersNamespaceName, f.UniqueName, certCtx, servicePort) 212 testAttachingPodWebhook(ctx, f) 213 }) 214 215 /* 216 Release: v1.16 217 Testname: Admission webhook, deny custom resource create and delete 218 Description: Register an admission webhook configuration that denies creation, update and deletion of 219 custom resources. Attempts to create, update and delete custom resources MUST be denied. 220 */ 221 framework.ConformanceIt("should be able to deny custom resource creation, update and deletion", func(ctx context.Context) { 222 testcrd, err := crd.CreateTestCRD(f) 223 if err != nil { 224 return 225 } 226 ginkgo.DeferCleanup(testcrd.CleanUp) 227 registerWebhookForCustomResource(ctx, f, markersNamespaceName, f.UniqueName, certCtx, testcrd, servicePort) 228 testCustomResourceWebhook(ctx, f, testcrd.Crd, testcrd.DynamicClients["v1"]) 229 testBlockingCustomResourceUpdateDeletion(ctx, f, testcrd.Crd, testcrd.DynamicClients["v1"]) 230 }) 231 232 /* 233 Release: v1.16 234 Testname: Admission webhook, fail closed 235 Description: Register a webhook with a fail closed policy and without CA bundle so that it cannot be called. 236 Attempt operations that require the admission webhook; all MUST be denied. 237 */ 238 framework.ConformanceIt("should unconditionally reject operations on fail closed webhook", func(ctx context.Context) { 239 registerFailClosedWebhook(ctx, f, markersNamespaceName, f.UniqueName, certCtx, servicePort) 240 testFailClosedWebhook(ctx, f) 241 }) 242 243 /* 244 Release: v1.16 245 Testname: Admission webhook, ordered mutation 246 Description: Register a mutating webhook configuration with two webhooks that admit configmaps, one that 247 adds a data key if the configmap already has a specific key, and another that adds a key if the key added by 248 the first webhook is present. Attempt to create a config map; both keys MUST be added to the config map. 249 */ 250 framework.ConformanceIt("should mutate configmap", func(ctx context.Context) { 251 registerMutatingWebhookForConfigMap(ctx, f, markersNamespaceName, f.UniqueName, certCtx, servicePort) 252 testMutatingConfigMapWebhook(ctx, f) 253 }) 254 255 /* 256 Release: v1.16 257 Testname: Admission webhook, mutation with defaulting 258 Description: Register a mutating webhook that adds an InitContainer to pods. Attempt to create a pod; 259 the InitContainer MUST be added the TerminationMessagePolicy MUST be defaulted. 260 */ 261 framework.ConformanceIt("should mutate pod and apply defaults after mutation", func(ctx context.Context) { 262 registerMutatingWebhookForPod(ctx, f, markersNamespaceName, f.UniqueName, certCtx, servicePort) 263 testMutatingPodWebhook(ctx, f) 264 }) 265 266 /* 267 Release: v1.16 268 Testname: Admission webhook, admission control not allowed on webhook configuration objects 269 Description: Register webhooks that mutate and deny deletion of webhook configuration objects. Attempt to create 270 and delete a webhook configuration object; both operations MUST be allowed and the webhook configuration object 271 MUST NOT be mutated the webhooks. 272 */ 273 framework.ConformanceIt("should not be able to mutate or prevent deletion of webhook configuration objects", func(ctx context.Context) { 274 registerValidatingWebhookForWebhookConfigurations(ctx, f, markersNamespaceName, f.UniqueName+"blocking", certCtx, servicePort) 275 registerMutatingWebhookForWebhookConfigurations(ctx, f, markersNamespaceName, f.UniqueName+"blocking", certCtx, servicePort) 276 testWebhooksForWebhookConfigurations(ctx, f, markersNamespaceName, f.UniqueName, certCtx, servicePort) 277 }) 278 279 /* 280 Release: v1.16 281 Testname: Admission webhook, mutate custom resource 282 Description: Register a webhook that mutates a custom resource. Attempt to create custom resource object; 283 the custom resource MUST be mutated. 284 */ 285 framework.ConformanceIt("should mutate custom resource", func(ctx context.Context) { 286 testcrd, err := crd.CreateTestCRD(f) 287 if err != nil { 288 return 289 } 290 ginkgo.DeferCleanup(testcrd.CleanUp) 291 registerMutatingWebhookForCustomResource(ctx, f, markersNamespaceName, f.UniqueName, certCtx, testcrd, servicePort) 292 testMutatingCustomResourceWebhook(ctx, f, testcrd.Crd, testcrd.DynamicClients["v1"], false) 293 }) 294 295 /* 296 Release: v1.16 297 Testname: Admission webhook, deny custom resource definition 298 Description: Register a webhook that denies custom resource definition create. Attempt to create a 299 custom resource definition; the create request MUST be denied. 300 */ 301 framework.ConformanceIt("should deny crd creation", func(ctx context.Context) { 302 registerValidatingWebhookForCRD(ctx, f, markersNamespaceName, f.UniqueName, certCtx, servicePort) 303 304 testCRDDenyWebhook(ctx, f) 305 }) 306 307 /* 308 Release: v1.16 309 Testname: Admission webhook, mutate custom resource with different stored version 310 Description: Register a webhook that mutates custom resources on create and update. Register a custom resource 311 definition using v1 as stored version. Create a custom resource. Patch the custom resource definition to use v2 as 312 the stored version. Attempt to patch the custom resource with a new field and value; the patch MUST be applied 313 successfully. 314 */ 315 framework.ConformanceIt("should mutate custom resource with different stored version", func(ctx context.Context) { 316 testcrd, err := createAdmissionWebhookMultiVersionTestCRDWithV1Storage(f) 317 if err != nil { 318 return 319 } 320 ginkgo.DeferCleanup(testcrd.CleanUp) 321 registerMutatingWebhookForCustomResource(ctx, f, markersNamespaceName, f.UniqueName, certCtx, testcrd, servicePort) 322 testMultiVersionCustomResourceWebhook(ctx, f, testcrd) 323 }) 324 325 /* 326 Release: v1.16 327 Testname: Admission webhook, mutate custom resource with pruning 328 Description: Register mutating webhooks that adds fields to custom objects. Register a custom resource definition 329 with a schema that includes only one of the data keys added by the webhooks. Attempt to a custom resource; 330 the fields included in the schema MUST be present and field not included in the schema MUST NOT be present. 331 */ 332 framework.ConformanceIt("should mutate custom resource with pruning", func(ctx context.Context) { 333 const prune = true 334 testcrd, err := createAdmissionWebhookMultiVersionTestCRDWithV1Storage(f, func(crd *apiextensionsv1.CustomResourceDefinition) { 335 crd.Spec.PreserveUnknownFields = false 336 for i := range crd.Spec.Versions { 337 crd.Spec.Versions[i].Schema = &apiextensionsv1.CustomResourceValidation{ 338 OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ 339 Type: "object", 340 Properties: map[string]apiextensionsv1.JSONSchemaProps{ 341 "data": { 342 Type: "object", 343 Properties: map[string]apiextensionsv1.JSONSchemaProps{ 344 "mutation-start": {Type: "string"}, 345 "mutation-stage-1": {Type: "string"}, 346 // mutation-stage-2 is intentionally missing such that it is pruned 347 }, 348 }, 349 }, 350 }, 351 } 352 } 353 }) 354 if err != nil { 355 return 356 } 357 ginkgo.DeferCleanup(testcrd.CleanUp) 358 registerMutatingWebhookForCustomResource(ctx, f, markersNamespaceName, f.UniqueName, certCtx, testcrd, servicePort) 359 testMutatingCustomResourceWebhook(ctx, f, testcrd.Crd, testcrd.DynamicClients["v1"], prune) 360 }) 361 362 /* 363 Release: v1.16 364 Testname: Admission webhook, honor timeout 365 Description: Using a webhook that waits 5 seconds before admitting objects, configure the webhook with combinations 366 of timeouts and failure policy values. Attempt to create a config map with each combination. Requests MUST 367 timeout if the configured webhook timeout is less than 5 seconds and failure policy is fail. Requests must not timeout if 368 the failure policy is ignore. Requests MUST NOT timeout if configured webhook timeout is 10 seconds (much longer 369 than the webhook wait duration). 370 */ 371 framework.ConformanceIt("should honor timeout", func(ctx context.Context) { 372 policyFail := admissionregistrationv1.Fail 373 policyIgnore := admissionregistrationv1.Ignore 374 375 ginkgo.By("Setting timeout (1s) shorter than webhook latency (5s)") 376 slowWebhookCleanup := registerSlowWebhook(ctx, f, markersNamespaceName, f.UniqueName, certCtx, &policyFail, pointer.Int32(1), servicePort) 377 testSlowWebhookTimeoutFailEarly(ctx, f) 378 slowWebhookCleanup(ctx) 379 380 ginkgo.By("Having no error when timeout is shorter than webhook latency and failure policy is ignore") 381 slowWebhookCleanup = registerSlowWebhook(ctx, f, markersNamespaceName, f.UniqueName, certCtx, &policyIgnore, pointer.Int32(1), servicePort) 382 testSlowWebhookTimeoutNoError(ctx, f) 383 slowWebhookCleanup(ctx) 384 385 ginkgo.By("Having no error when timeout is longer than webhook latency") 386 slowWebhookCleanup = registerSlowWebhook(ctx, f, markersNamespaceName, f.UniqueName, certCtx, &policyFail, pointer.Int32(10), servicePort) 387 testSlowWebhookTimeoutNoError(ctx, f) 388 slowWebhookCleanup(ctx) 389 390 ginkgo.By("Having no error when timeout is empty (defaulted to 10s in v1)") 391 slowWebhookCleanup = registerSlowWebhook(ctx, f, markersNamespaceName, f.UniqueName, certCtx, &policyFail, nil, servicePort) 392 testSlowWebhookTimeoutNoError(ctx, f) 393 slowWebhookCleanup(ctx) 394 }) 395 396 /* 397 Release: v1.16 398 Testname: Admission webhook, update validating webhook 399 Description: Register a validating admission webhook configuration. Update the webhook to not apply to the create 400 operation and attempt to create an object; the webhook MUST NOT deny the create. Patch the webhook to apply to the 401 create operation again and attempt to create an object; the webhook MUST deny the create. 402 */ 403 framework.ConformanceIt("patching/updating a validating webhook should work", func(ctx context.Context) { 404 client := f.ClientSet 405 admissionClient := client.AdmissionregistrationV1() 406 407 ginkgo.By("Creating a validating webhook configuration") 408 hook, err := createValidatingWebhookConfiguration(ctx, f, &admissionregistrationv1.ValidatingWebhookConfiguration{ 409 ObjectMeta: metav1.ObjectMeta{ 410 Name: f.UniqueName, 411 }, 412 Webhooks: []admissionregistrationv1.ValidatingWebhook{ 413 newDenyConfigMapWebhookFixture(f, certCtx, servicePort), 414 newValidatingIsReadyWebhookFixture(f, certCtx, servicePort), 415 }, 416 }) 417 framework.ExpectNoError(err, "Creating validating webhook configuration") 418 defer func() { 419 err := client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(ctx, hook.Name, metav1.DeleteOptions{}) 420 framework.ExpectNoError(err, "Deleting validating webhook configuration") 421 }() 422 423 // ensure backend is ready before proceeding 424 err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName) 425 framework.ExpectNoError(err, "waiting for webhook configuration to be ready") 426 427 ginkgo.By("Creating a configMap that does not comply to the validation webhook rules") 428 err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (bool, error) { 429 cm := namedNonCompliantConfigMap(string(uuid.NewUUID()), f) 430 _, err = client.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{}) 431 if err == nil { 432 err = client.CoreV1().ConfigMaps(f.Namespace.Name).Delete(ctx, cm.Name, metav1.DeleteOptions{}) 433 framework.ExpectNoError(err, "Deleting successfully created configMap") 434 return false, nil 435 } 436 if !strings.Contains(err.Error(), "denied") { 437 return false, err 438 } 439 return true, nil 440 }) 441 442 ginkgo.By("Updating a validating webhook configuration's rules to not include the create operation") 443 err = retry.RetryOnConflict(retry.DefaultRetry, func() error { 444 h, err := admissionClient.ValidatingWebhookConfigurations().Get(ctx, f.UniqueName, metav1.GetOptions{}) 445 framework.ExpectNoError(err, "Getting validating webhook configuration") 446 h.Webhooks[0].Rules[0].Operations = []admissionregistrationv1.OperationType{admissionregistrationv1.Update} 447 _, err = admissionClient.ValidatingWebhookConfigurations().Update(ctx, h, metav1.UpdateOptions{}) 448 return err 449 }) 450 framework.ExpectNoError(err, "Updating validating webhook configuration") 451 452 ginkgo.By("Creating a configMap that does not comply to the validation webhook rules") 453 err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (bool, error) { 454 cm := namedNonCompliantConfigMap(string(uuid.NewUUID()), f) 455 _, err = client.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{}) 456 if err != nil { 457 if !strings.Contains(err.Error(), "denied") { 458 return false, err 459 } 460 return false, nil 461 } 462 err = client.CoreV1().ConfigMaps(f.Namespace.Name).Delete(ctx, cm.Name, metav1.DeleteOptions{}) 463 framework.ExpectNoError(err, "Deleting successfully created configMap") 464 return true, nil 465 }) 466 framework.ExpectNoError(err, "Waiting for configMap in namespace %s to be allowed creation since webhook was updated to not validate create", f.Namespace.Name) 467 468 ginkgo.By("Patching a validating webhook configuration's rules to include the create operation") 469 hook, err = admissionClient.ValidatingWebhookConfigurations().Patch(ctx, f.UniqueName, 470 types.JSONPatchType, 471 []byte(`[{"op": "replace", "path": "/webhooks/0/rules/0/operations", "value": ["CREATE"]}]`), metav1.PatchOptions{}) 472 framework.ExpectNoError(err, "Patching validating webhook configuration") 473 474 ginkgo.By("Creating a configMap that does not comply to the validation webhook rules") 475 err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (bool, error) { 476 cm := namedNonCompliantConfigMap(string(uuid.NewUUID()), f) 477 _, err = client.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{}) 478 if err == nil { 479 err = client.CoreV1().ConfigMaps(f.Namespace.Name).Delete(ctx, cm.Name, metav1.DeleteOptions{}) 480 framework.ExpectNoError(err, "Deleting successfully created configMap") 481 return false, nil 482 } 483 if !strings.Contains(err.Error(), "denied") { 484 return false, err 485 } 486 return true, nil 487 }) 488 framework.ExpectNoError(err, "Waiting for configMap in namespace %s to be denied creation by validating webhook", f.Namespace.Name) 489 }) 490 491 /* 492 Release: v1.16 493 Testname: Admission webhook, update mutating webhook 494 Description: Register a mutating admission webhook configuration. Update the webhook to not apply to the create 495 operation and attempt to create an object; the webhook MUST NOT mutate the object. Patch the webhook to apply to the 496 create operation again and attempt to create an object; the webhook MUST mutate the object. 497 */ 498 framework.ConformanceIt("patching/updating a mutating webhook should work", func(ctx context.Context) { 499 client := f.ClientSet 500 admissionClient := client.AdmissionregistrationV1() 501 502 ginkgo.By("Creating a mutating webhook configuration") 503 hook, err := createMutatingWebhookConfiguration(ctx, f, &admissionregistrationv1.MutatingWebhookConfiguration{ 504 ObjectMeta: metav1.ObjectMeta{ 505 Name: f.UniqueName, 506 }, 507 Webhooks: []admissionregistrationv1.MutatingWebhook{ 508 newMutateConfigMapWebhookFixture(f, certCtx, 1, servicePort), 509 newMutatingIsReadyWebhookFixture(f, certCtx, servicePort), 510 }, 511 }) 512 framework.ExpectNoError(err, "Creating mutating webhook configuration") 513 defer func() { 514 err := client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(ctx, hook.Name, metav1.DeleteOptions{}) 515 framework.ExpectNoError(err, "Deleting mutating webhook configuration") 516 }() 517 518 // ensure backend is ready before proceeding 519 err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName) 520 framework.ExpectNoError(err, "waiting for webhook configuration to be ready") 521 522 hook, err = admissionClient.MutatingWebhookConfigurations().Get(ctx, f.UniqueName, metav1.GetOptions{}) 523 framework.ExpectNoError(err, "Getting mutating webhook configuration") 524 ginkgo.By("Updating a mutating webhook configuration's rules to not include the create operation") 525 hook.Webhooks[0].Rules[0].Operations = []admissionregistrationv1.OperationType{admissionregistrationv1.Update} 526 hook, err = admissionClient.MutatingWebhookConfigurations().Update(ctx, hook, metav1.UpdateOptions{}) 527 framework.ExpectNoError(err, "Updating mutating webhook configuration") 528 529 ginkgo.By("Creating a configMap that should not be mutated") 530 err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (bool, error) { 531 cm := namedToBeMutatedConfigMap(string(uuid.NewUUID()), f) 532 created, err := client.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{}) 533 if err != nil { 534 return false, err 535 } 536 err = client.CoreV1().ConfigMaps(f.Namespace.Name).Delete(ctx, cm.Name, metav1.DeleteOptions{}) 537 framework.ExpectNoError(err, "Deleting successfully created configMap") 538 _, ok := created.Data["mutation-stage-1"] 539 return !ok, nil 540 }) 541 framework.ExpectNoError(err, "Waiting for configMap in namespace %s this is not mutated", f.Namespace.Name) 542 543 ginkgo.By("Patching a mutating webhook configuration's rules to include the create operation") 544 hook, err = admissionClient.MutatingWebhookConfigurations().Patch(ctx, f.UniqueName, 545 types.JSONPatchType, 546 []byte(`[{"op": "replace", "path": "/webhooks/0/rules/0/operations", "value": ["CREATE"]}]`), metav1.PatchOptions{}) 547 framework.ExpectNoError(err, "Patching mutating webhook configuration") 548 549 ginkgo.By("Creating a configMap that should be mutated") 550 err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (bool, error) { 551 cm := namedToBeMutatedConfigMap(string(uuid.NewUUID()), f) 552 created, err := client.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{}) 553 if err != nil { 554 return false, err 555 } 556 err = client.CoreV1().ConfigMaps(f.Namespace.Name).Delete(ctx, cm.Name, metav1.DeleteOptions{}) 557 framework.ExpectNoError(err, "Deleting successfully created configMap") 558 _, ok := created.Data["mutation-stage-1"] 559 return ok, nil 560 }) 561 framework.ExpectNoError(err, "Waiting for configMap in namespace %s to be mutated", f.Namespace.Name) 562 }) 563 564 /* 565 Release: v1.16 566 Testname: Admission webhook, list validating webhooks 567 Description: Create 10 validating webhook configurations, all with a label. Attempt to list the webhook 568 configurations matching the label; all the created webhook configurations MUST be present. Attempt to create an 569 object; the create MUST be denied. Attempt to remove the webhook configurations matching the label with deletecollection; 570 all webhook configurations MUST be deleted. Attempt to create an object; the create MUST NOT be denied. 571 */ 572 framework.ConformanceIt("listing validating webhooks should work", func(ctx context.Context) { 573 testListSize := 10 574 testUUID := string(uuid.NewUUID()) 575 576 for i := 0; i < testListSize; i++ { 577 name := fmt.Sprintf("%s-%d", f.UniqueName, i) 578 _, err := createValidatingWebhookConfiguration(ctx, f, &admissionregistrationv1.ValidatingWebhookConfiguration{ 579 ObjectMeta: metav1.ObjectMeta{ 580 Name: name, 581 Labels: map[string]string{"e2e-list-test-uuid": testUUID}, 582 }, 583 Webhooks: []admissionregistrationv1.ValidatingWebhook{ 584 newDenyConfigMapWebhookFixture(f, certCtx, servicePort), 585 newValidatingIsReadyWebhookFixture(f, certCtx, servicePort), 586 }, 587 }) 588 framework.ExpectNoError(err, "Creating validating webhook configuration") 589 } 590 selectorListOpts := metav1.ListOptions{LabelSelector: "e2e-list-test-uuid=" + testUUID} 591 592 ginkgo.By("Listing all of the created validation webhooks") 593 list, err := client.AdmissionregistrationV1().ValidatingWebhookConfigurations().List(ctx, selectorListOpts) 594 framework.ExpectNoError(err, "Listing validating webhook configurations") 595 gomega.Expect(list.Items).To(gomega.HaveLen(testListSize)) 596 597 // ensure backend is ready before proceeding 598 err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName) 599 framework.ExpectNoError(err, "waiting for webhook configuration to be ready") 600 601 ginkgo.By("Creating a configMap that does not comply to the validation webhook rules") 602 err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (bool, error) { 603 cm := namedNonCompliantConfigMap(string(uuid.NewUUID()), f) 604 _, err = client.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{}) 605 if err == nil { 606 err = client.CoreV1().ConfigMaps(f.Namespace.Name).Delete(ctx, cm.Name, metav1.DeleteOptions{}) 607 framework.ExpectNoError(err, "Deleting successfully created configMap") 608 return false, nil 609 } 610 if !strings.Contains(err.Error(), "denied") { 611 return false, err 612 } 613 return true, nil 614 }) 615 framework.ExpectNoError(err, "Waiting for configMap in namespace %s to be denied creation by validating webhook", f.Namespace.Name) 616 617 ginkgo.By("Deleting the collection of validation webhooks") 618 err = client.AdmissionregistrationV1().ValidatingWebhookConfigurations().DeleteCollection(ctx, metav1.DeleteOptions{}, selectorListOpts) 619 framework.ExpectNoError(err, "Deleting collection of validating webhook configurations") 620 621 ginkgo.By("Creating a configMap that does not comply to the validation webhook rules") 622 err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (bool, error) { 623 cm := namedNonCompliantConfigMap(string(uuid.NewUUID()), f) 624 _, err = client.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{}) 625 if err != nil { 626 if !strings.Contains(err.Error(), "denied") { 627 return false, err 628 } 629 return false, nil 630 } 631 err = client.CoreV1().ConfigMaps(f.Namespace.Name).Delete(ctx, cm.Name, metav1.DeleteOptions{}) 632 framework.ExpectNoError(err, "Deleting successfully created configMap") 633 return true, nil 634 }) 635 framework.ExpectNoError(err, "Waiting for configMap in namespace %s to be allowed creation since there are no webhooks", f.Namespace.Name) 636 }) 637 638 /* 639 Release: v1.16 640 Testname: Admission webhook, list mutating webhooks 641 Description: Create 10 mutating webhook configurations, all with a label. Attempt to list the webhook 642 configurations matching the label; all the created webhook configurations MUST be present. Attempt to create an 643 object; the object MUST be mutated. Attempt to remove the webhook configurations matching the label with deletecollection; 644 all webhook configurations MUST be deleted. Attempt to create an object; the object MUST NOT be mutated. 645 */ 646 framework.ConformanceIt("listing mutating webhooks should work", func(ctx context.Context) { 647 testListSize := 10 648 testUUID := string(uuid.NewUUID()) 649 650 for i := 0; i < testListSize; i++ { 651 name := fmt.Sprintf("%s-%d", f.UniqueName, i) 652 _, err := createMutatingWebhookConfiguration(ctx, f, &admissionregistrationv1.MutatingWebhookConfiguration{ 653 ObjectMeta: metav1.ObjectMeta{ 654 Name: name, 655 Labels: map[string]string{"e2e-list-test-uuid": testUUID}, 656 }, 657 Webhooks: []admissionregistrationv1.MutatingWebhook{ 658 newMutateConfigMapWebhookFixture(f, certCtx, 1, servicePort), 659 newMutatingIsReadyWebhookFixture(f, certCtx, servicePort), 660 }, 661 }) 662 framework.ExpectNoError(err, "Creating mutating webhook configuration") 663 } 664 selectorListOpts := metav1.ListOptions{LabelSelector: "e2e-list-test-uuid=" + testUUID} 665 666 ginkgo.By("Listing all of the created validation webhooks") 667 list, err := client.AdmissionregistrationV1().MutatingWebhookConfigurations().List(ctx, selectorListOpts) 668 framework.ExpectNoError(err, "Listing mutating webhook configurations") 669 gomega.Expect(list.Items).To(gomega.HaveLen(testListSize)) 670 671 // ensure backend is ready before proceeding 672 err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName) 673 framework.ExpectNoError(err, "waiting for webhook configuration to be ready") 674 675 ginkgo.By("Creating a configMap that should be mutated") 676 err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (bool, error) { 677 cm := namedToBeMutatedConfigMap(string(uuid.NewUUID()), f) 678 created, err := client.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{}) 679 if err != nil { 680 return false, err 681 } 682 err = client.CoreV1().ConfigMaps(f.Namespace.Name).Delete(ctx, cm.Name, metav1.DeleteOptions{}) 683 framework.ExpectNoError(err, "Deleting successfully created configMap") 684 _, ok := created.Data["mutation-stage-1"] 685 return ok, nil 686 }) 687 framework.ExpectNoError(err, "Waiting for configMap in namespace %s to be mutated", f.Namespace.Name) 688 689 ginkgo.By("Deleting the collection of validation webhooks") 690 err = client.AdmissionregistrationV1().MutatingWebhookConfigurations().DeleteCollection(ctx, metav1.DeleteOptions{}, selectorListOpts) 691 framework.ExpectNoError(err, "Deleting collection of mutating webhook configurations") 692 693 ginkgo.By("Creating a configMap that should not be mutated") 694 err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (bool, error) { 695 cm := namedToBeMutatedConfigMap(string(uuid.NewUUID()), f) 696 created, err := client.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{}) 697 if err != nil { 698 return false, err 699 } 700 err = client.CoreV1().ConfigMaps(f.Namespace.Name).Delete(ctx, cm.Name, metav1.DeleteOptions{}) 701 framework.ExpectNoError(err, "Deleting successfully created configMap") 702 _, ok := created.Data["mutation-stage-1"] 703 return !ok, nil 704 }) 705 framework.ExpectNoError(err, "Waiting for configMap in namespace %s this is not mutated", f.Namespace.Name) 706 }) 707 708 /* 709 Release: v1.28 710 Testname: Validating Admission webhook, create and update validating webhook configuration with matchConditions 711 Description: Register a validating webhook configuration. Verify that the match conditions field are 712 properly stored in the api-server. Update the validating webhook configuration and retrieve it; the 713 retrieved object must contain the newly update matchConditions fields. 714 */ 715 ginkgo.It("should be able to create and update validating webhook configurations with match conditions", func(ctx context.Context) { 716 initalMatchConditions := []admissionregistrationv1.MatchCondition{ 717 { 718 Name: "expression-1", 719 Expression: "object.metadata.namespace == 'production'", 720 }, 721 } 722 723 ginkgo.By("creating a validating webhook with match conditions") 724 validatingWebhookConfiguration := newValidatingWebhookWithMatchConditions(f, servicePort, certCtx, initalMatchConditions) 725 726 _, err := createValidatingWebhookConfiguration(ctx, f, validatingWebhookConfiguration) 727 framework.ExpectNoError(err) 728 729 ginkgo.By("verifying the validating webhook match conditions") 730 validatingWebhookConfiguration, err = client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(ctx, f.UniqueName, metav1.GetOptions{}) 731 framework.ExpectNoError(err) 732 gomega.Expect(validatingWebhookConfiguration.Webhooks[0].MatchConditions).To(gomega.Equal(initalMatchConditions), "verifying that match conditions are created") 733 defer func() { 734 err := client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(ctx, validatingWebhookConfiguration.Name, metav1.DeleteOptions{}) 735 framework.ExpectNoError(err, "deleting mutating webhook configuration") 736 }() 737 738 ginkgo.By("updating the validating webhook match conditions") 739 updatedMatchConditions := []admissionregistrationv1.MatchCondition{ 740 { 741 Name: "expression-1", 742 Expression: "object.metadata.namespace == 'production'", 743 }, 744 { 745 Name: "expression-2", 746 Expression: "object.metadata.namespace == 'staging'", 747 }, 748 } 749 validatingWebhookConfiguration.Webhooks[0].MatchConditions = updatedMatchConditions 750 _, err = client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Update(ctx, validatingWebhookConfiguration, metav1.UpdateOptions{}) 751 framework.ExpectNoError(err) 752 753 ginkgo.By("verifying the validating webhook match conditions") 754 validatingWebhookConfiguration, err = client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(ctx, f.UniqueName, metav1.GetOptions{}) 755 framework.ExpectNoError(err) 756 gomega.Expect(validatingWebhookConfiguration.Webhooks[0].MatchConditions).To(gomega.Equal(updatedMatchConditions), "verifying that match conditions are updated") 757 }) 758 759 /* 760 Release: v1.28 761 Testname: Mutating Admission webhook, create and update mutating webhook configuration with matchConditions 762 Description: Register a mutating webhook configuration. Verify that the match conditions field are 763 properly stored in the api-server. Update the mutating webhook configuration and retrieve it; the 764 retrieved object must contain the newly update matchConditions fields. 765 */ 766 ginkgo.It("should be able to create and update mutating webhook configurations with match conditions", func(ctx context.Context) { 767 initalMatchConditions := []admissionregistrationv1.MatchCondition{ 768 { 769 Name: "expression-1", 770 Expression: "object.metadata.namespace == 'production'", 771 }, 772 } 773 774 ginkgo.By("creating a mutating webhook with match conditions") 775 mutatingWebhookConfiguration := newMutatingWebhookWithMatchConditions(f, servicePort, certCtx, initalMatchConditions) 776 777 _, err := createMutatingWebhookConfiguration(ctx, f, mutatingWebhookConfiguration) 778 framework.ExpectNoError(err) 779 780 ginkgo.By("verifying the mutating webhook match conditions") 781 mutatingWebhookConfiguration, err = client.AdmissionregistrationV1().MutatingWebhookConfigurations().Get(ctx, f.UniqueName, metav1.GetOptions{}) 782 framework.ExpectNoError(err) 783 gomega.Expect(mutatingWebhookConfiguration.Webhooks[0].MatchConditions).To(gomega.Equal(initalMatchConditions), "verifying that match conditions are created") 784 defer func() { 785 err := client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(ctx, mutatingWebhookConfiguration.Name, metav1.DeleteOptions{}) 786 framework.ExpectNoError(err, "deleting mutating webhook configuration") 787 }() 788 789 ginkgo.By("updating the mutating webhook match conditions") 790 updatedMatchConditions := []admissionregistrationv1.MatchCondition{ 791 { 792 Name: "expression-1", 793 Expression: "object.metadata.namespace == 'production'", 794 }, 795 { 796 Name: "expression-2", 797 Expression: "object.metadata.namespace == 'staging'", 798 }, 799 } 800 mutatingWebhookConfiguration.Webhooks[0].MatchConditions = updatedMatchConditions 801 _, err = client.AdmissionregistrationV1().MutatingWebhookConfigurations().Update(ctx, mutatingWebhookConfiguration, metav1.UpdateOptions{}) 802 framework.ExpectNoError(err) 803 804 ginkgo.By("verifying the mutating webhook match conditions") 805 mutatingWebhookConfiguration, err = client.AdmissionregistrationV1().MutatingWebhookConfigurations().Get(ctx, f.UniqueName, metav1.GetOptions{}) 806 framework.ExpectNoError(err) 807 gomega.Expect(mutatingWebhookConfiguration.Webhooks[0].MatchConditions).To(gomega.Equal(updatedMatchConditions), "verifying that match conditions are updated") 808 }) 809 810 /* 811 Release: v1.28 812 Testname: Validing Admission webhook, reject validating webhook configurations with invalid matchConditions 813 Description: Creates a validating webhook configuration with an invalid CEL expression in it's 814 matchConditions field. The api-server server should reject the create request with a "compilation 815 failed" error message. 816 */ 817 ginkgo.It("should reject validating webhook configurations with invalid match conditions", func(ctx context.Context) { 818 initalMatchConditions := []admissionregistrationv1.MatchCondition{ 819 { 820 Name: "invalid-expression-1", 821 Expression: "... [] bad expression", 822 }, 823 } 824 825 ginkgo.By("creating a validating webhook with match conditions") 826 validatingWebhookConfiguration := newValidatingWebhookWithMatchConditions(f, servicePort, certCtx, initalMatchConditions) 827 828 _, err := createValidatingWebhookConfiguration(ctx, f, validatingWebhookConfiguration) 829 gomega.Expect(err).To(gomega.HaveOccurred(), "create validatingwebhookconfiguration should have been denied by the api-server") 830 expectedErrMsg := "compilation failed" 831 gomega.Expect(strings.Contains(err.Error(), expectedErrMsg)).To(gomega.BeTrue()) 832 }) 833 834 /* 835 Release: v1.28 836 Testname: Mutating Admission webhook, reject mutating webhook configurations with invalid matchConditions 837 Description: Creates a mutating webhook configuration with an invalid CEL expression in it's 838 matchConditions field. The api-server server should reject the create request with a "compilation 839 failed" error message. 840 */ 841 ginkgo.It("should reject mutating webhook configurations with invalid match conditions", func(ctx context.Context) { 842 initalMatchConditions := []admissionregistrationv1.MatchCondition{ 843 { 844 Name: "invalid-expression-1", 845 Expression: "... [] bad expression", 846 }, 847 } 848 849 ginkgo.By("creating a mutating webhook with match conditions") 850 mutatingWebhookConfiguration := newMutatingWebhookWithMatchConditions(f, servicePort, certCtx, initalMatchConditions) 851 852 _, err := createMutatingWebhookConfiguration(ctx, f, mutatingWebhookConfiguration) 853 gomega.Expect(err).To(gomega.HaveOccurred(), "create mutatingwebhookconfiguration should have been denied by the api-server") 854 expectedErrMsg := "compilation failed" 855 gomega.Expect(strings.Contains(err.Error(), expectedErrMsg)).To(gomega.BeTrue()) 856 }) 857 858 /* 859 Release: v1.28 860 Testname: Mutating Admission webhook, mutating webhook excluding object with specific name 861 Description: Create a mutating webhook configuration with matchConditions field that 862 will reject all resources except ones with a specific name 'skip-me'. Create 863 a configMap with the name 'skip-me' and verify that it's mutated. Create a 864 configMap with a different name than 'skip-me' and verify that it's mustated. 865 */ 866 ginkgo.It("should mutate everything except 'skip-me' configmaps", func(ctx context.Context) { 867 skipMeMatchConditions := []admissionregistrationv1.MatchCondition{ 868 { 869 Name: "skip-me", 870 Expression: "object.metadata.name != 'skip-me'", 871 }, 872 } 873 874 ginkgo.By("creating a mutating webhook with match conditions") 875 namespace := f.Namespace.Name 876 877 mutatingWebhook1 := newMutateConfigMapWebhookFixture(f, certCtx, 1, servicePort) 878 mutatingWebhook1.MatchConditions = skipMeMatchConditions 879 created, err := createMutatingWebhookConfiguration(ctx, f, &admissionregistrationv1.MutatingWebhookConfiguration{ 880 ObjectMeta: metav1.ObjectMeta{ 881 Name: f.UniqueName, 882 }, 883 Webhooks: []admissionregistrationv1.MutatingWebhook{ 884 mutatingWebhook1, 885 // Register a webhook that can be probed by marker requests to detect when the configuration is ready. 886 newMutatingIsReadyWebhookFixture(f, certCtx, servicePort), 887 }, 888 }) 889 framework.ExpectNoError(err, "registering mutating webhook config %s with namespace %s", f.UniqueName, namespace) 890 defer func() { 891 err := client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(ctx, created.Name, metav1.DeleteOptions{}) 892 framework.ExpectNoError(err, "deleting mutating webhook configuration") 893 }() 894 895 err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName) 896 framework.ExpectNoError(err, "waiting for webhook configuration to be ready") 897 ginkgo.DeferCleanup(framework.IgnoreNotFound(client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete), f.UniqueName, metav1.DeleteOptions{}) 898 899 // ensure backend is ready before proceeding 900 err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName) 901 framework.ExpectNoError(err, "waiting for webhook configuration to be ready") 902 903 ginkgo.By("create the configmap with a random name") 904 905 cm := namedToBeMutatedConfigMap(string(uuid.NewUUID()), f) 906 mutatedCM, err := client.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{}) 907 framework.ExpectNoError(err, "creating configMap object") 908 909 ginkgo.By("verify the configmap is mutated") 910 expectedConfigMapData := map[string]string{ 911 "mutation-start": "yes", 912 "mutation-stage-1": "yes", 913 } 914 gomega.Expect(reflect.DeepEqual(expectedConfigMapData, mutatedCM.Data)).To(gomega.BeTrue()) 915 916 ginkgo.By("create the configmap with 'skip-me' name") 917 918 cm = namedToBeMutatedConfigMap("skip-me", f) 919 skippedCM, err := client.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, cm, metav1.CreateOptions{}) 920 framework.ExpectNoError(err, "creating configMap object") 921 expectedConfigMapData = map[string]string{ 922 "mutation-start": "yes", 923 } 924 gomega.Expect(reflect.DeepEqual(expectedConfigMapData, skippedCM.Data)).To(gomega.BeTrue()) 925 }) 926 }) 927 928 func newValidatingWebhookWithMatchConditions( 929 f *framework.Framework, 930 servicePort int32, 931 certCtx *certContext, 932 matchConditions []admissionregistrationv1.MatchCondition, 933 ) *admissionregistrationv1.ValidatingWebhookConfiguration { 934 sideEffects := admissionregistrationv1.SideEffectClassNone 935 equivalent := admissionregistrationv1.Equivalent 936 return &admissionregistrationv1.ValidatingWebhookConfiguration{ 937 ObjectMeta: metav1.ObjectMeta{ 938 Name: f.UniqueName, 939 }, 940 Webhooks: []admissionregistrationv1.ValidatingWebhook{ 941 { 942 Name: "validation-webhook-with-match-conditions.k8s.io", 943 Rules: []admissionregistrationv1.RuleWithOperations{{ 944 Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create}, 945 Rule: admissionregistrationv1.Rule{ 946 APIGroups: []string{""}, 947 APIVersions: []string{"v1"}, 948 Resources: []string{"configmaps"}, 949 }, 950 }}, 951 ClientConfig: admissionregistrationv1.WebhookClientConfig{ 952 Service: &admissionregistrationv1.ServiceReference{ 953 Namespace: f.Namespace.Name, 954 Name: serviceName, 955 Path: strPtr("/always-deny"), 956 Port: pointer.Int32(servicePort), 957 }, 958 CABundle: certCtx.signingCert, 959 }, 960 SideEffects: &sideEffects, 961 MatchPolicy: &equivalent, 962 AdmissionReviewVersions: []string{"v1"}, 963 // Scope the webhook to just the markers namespace 964 NamespaceSelector: &metav1.LabelSelector{ 965 MatchLabels: map[string]string{f.UniqueName: "true"}, 966 }, 967 MatchConditions: matchConditions, 968 }, 969 newValidatingIsReadyWebhookFixture(f, certCtx, servicePort), 970 }, 971 } 972 } 973 974 func newMutatingWebhookWithMatchConditions( 975 f *framework.Framework, 976 servicePort int32, 977 certCtx *certContext, 978 matchConditions []admissionregistrationv1.MatchCondition, 979 ) *admissionregistrationv1.MutatingWebhookConfiguration { 980 sideEffects := admissionregistrationv1.SideEffectClassNone 981 return &admissionregistrationv1.MutatingWebhookConfiguration{ 982 ObjectMeta: metav1.ObjectMeta{ 983 Name: f.UniqueName, 984 }, 985 Webhooks: []admissionregistrationv1.MutatingWebhook{ 986 { 987 Name: "adding-configmap-data.k8s.io", 988 Rules: []admissionregistrationv1.RuleWithOperations{{ 989 Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create}, 990 Rule: admissionregistrationv1.Rule{ 991 APIGroups: []string{""}, 992 APIVersions: []string{"v1"}, 993 Resources: []string{"configmaps"}, 994 }, 995 }}, 996 ClientConfig: admissionregistrationv1.WebhookClientConfig{ 997 Service: &admissionregistrationv1.ServiceReference{ 998 Namespace: f.Namespace.Name, 999 Name: serviceName, 1000 Path: strPtr("/mutating-configmaps"), 1001 Port: pointer.Int32(servicePort), 1002 }, 1003 CABundle: certCtx.signingCert, 1004 }, 1005 SideEffects: &sideEffects, 1006 AdmissionReviewVersions: []string{"v1", "v1beta1"}, 1007 // Scope the webhook to just this namespace 1008 NamespaceSelector: &metav1.LabelSelector{ 1009 MatchLabels: map[string]string{f.UniqueName: "true"}, 1010 }, 1011 MatchConditions: matchConditions, 1012 }, 1013 newMutatingIsReadyWebhookFixture(f, certCtx, servicePort), 1014 }, 1015 } 1016 } 1017 1018 func createAuthReaderRoleBinding(ctx context.Context, f *framework.Framework, namespace string) { 1019 ginkgo.By("Create role binding to let webhook read extension-apiserver-authentication") 1020 client := f.ClientSet 1021 // Create the role binding to allow the webhook read the extension-apiserver-authentication configmap 1022 _, err := client.RbacV1().RoleBindings("kube-system").Create(ctx, &rbacv1.RoleBinding{ 1023 ObjectMeta: metav1.ObjectMeta{ 1024 Name: roleBindingName, 1025 Annotations: map[string]string{ 1026 rbacv1.AutoUpdateAnnotationKey: "true", 1027 }, 1028 }, 1029 RoleRef: rbacv1.RoleRef{ 1030 APIGroup: "", 1031 Kind: "Role", 1032 Name: "extension-apiserver-authentication-reader", 1033 }, 1034 1035 Subjects: []rbacv1.Subject{ 1036 { 1037 Kind: "ServiceAccount", 1038 Name: "default", 1039 Namespace: namespace, 1040 }, 1041 }, 1042 }, metav1.CreateOptions{}) 1043 if err != nil && apierrors.IsAlreadyExists(err) { 1044 framework.Logf("role binding %s already exists", roleBindingName) 1045 } else { 1046 framework.ExpectNoError(err, "creating role binding %s:webhook to access configMap", namespace) 1047 } 1048 } 1049 1050 func deployWebhookAndService(ctx context.Context, f *framework.Framework, image string, certCtx *certContext, servicePort int32, containerPort int32) { 1051 ginkgo.By("Deploying the webhook pod") 1052 client := f.ClientSet 1053 1054 // Creating the secret that contains the webhook's cert. 1055 secret := &v1.Secret{ 1056 ObjectMeta: metav1.ObjectMeta{ 1057 Name: secretName, 1058 }, 1059 Type: v1.SecretTypeOpaque, 1060 Data: map[string][]byte{ 1061 "tls.crt": certCtx.cert, 1062 "tls.key": certCtx.key, 1063 }, 1064 } 1065 namespace := f.Namespace.Name 1066 _, err := client.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) 1067 framework.ExpectNoError(err, "creating secret %q in namespace %q", secretName, namespace) 1068 1069 // Create the deployment of the webhook 1070 podLabels := map[string]string{"app": "sample-webhook", "webhook": "true"} 1071 replicas := int32(1) 1072 mounts := []v1.VolumeMount{ 1073 { 1074 Name: "webhook-certs", 1075 ReadOnly: true, 1076 MountPath: "/webhook.local.config/certificates", 1077 }, 1078 } 1079 volumes := []v1.Volume{ 1080 { 1081 Name: "webhook-certs", 1082 VolumeSource: v1.VolumeSource{ 1083 Secret: &v1.SecretVolumeSource{SecretName: secretName}, 1084 }, 1085 }, 1086 } 1087 containers := []v1.Container{ 1088 { 1089 Name: "sample-webhook", 1090 VolumeMounts: mounts, 1091 Args: []string{ 1092 "webhook", 1093 "--tls-cert-file=/webhook.local.config/certificates/tls.crt", 1094 "--tls-private-key-file=/webhook.local.config/certificates/tls.key", 1095 "-v=4", 1096 // Use a non-default port for containers. 1097 fmt.Sprintf("--port=%d", containerPort), 1098 }, 1099 ReadinessProbe: &v1.Probe{ 1100 ProbeHandler: v1.ProbeHandler{ 1101 HTTPGet: &v1.HTTPGetAction{ 1102 Scheme: v1.URISchemeHTTPS, 1103 Port: intstr.FromInt32(containerPort), 1104 Path: "/readyz", 1105 }, 1106 }, 1107 PeriodSeconds: 1, 1108 SuccessThreshold: 1, 1109 FailureThreshold: 30, 1110 }, 1111 Image: image, 1112 Ports: []v1.ContainerPort{{ContainerPort: containerPort}}, 1113 }, 1114 } 1115 d := e2edeployment.NewDeployment(deploymentName, replicas, podLabels, "", "", appsv1.RollingUpdateDeploymentStrategyType) 1116 d.Spec.Template.Spec.Containers = containers 1117 d.Spec.Template.Spec.Volumes = volumes 1118 1119 deployment, err := client.AppsV1().Deployments(namespace).Create(ctx, d, metav1.CreateOptions{}) 1120 framework.ExpectNoError(err, "creating deployment %s in namespace %s", deploymentName, namespace) 1121 ginkgo.By("Wait for the deployment to be ready") 1122 err = e2edeployment.WaitForDeploymentRevisionAndImage(client, namespace, deploymentName, "1", image) 1123 framework.ExpectNoError(err, "waiting for the deployment of image %s in %s in %s to complete", image, deploymentName, namespace) 1124 err = e2edeployment.WaitForDeploymentComplete(client, deployment) 1125 framework.ExpectNoError(err, "waiting for the deployment status valid", image, deploymentName, namespace) 1126 1127 ginkgo.By("Deploying the webhook service") 1128 1129 serviceLabels := map[string]string{"webhook": "true"} 1130 service := &v1.Service{ 1131 ObjectMeta: metav1.ObjectMeta{ 1132 Namespace: namespace, 1133 Name: serviceName, 1134 Labels: map[string]string{"test": "webhook"}, 1135 }, 1136 Spec: v1.ServiceSpec{ 1137 Selector: serviceLabels, 1138 Ports: []v1.ServicePort{ 1139 { 1140 Protocol: v1.ProtocolTCP, 1141 Port: servicePort, 1142 TargetPort: intstr.FromInt32(containerPort), 1143 }, 1144 }, 1145 }, 1146 } 1147 _, err = client.CoreV1().Services(namespace).Create(ctx, service, metav1.CreateOptions{}) 1148 framework.ExpectNoError(err, "creating service %s in namespace %s", serviceName, namespace) 1149 1150 ginkgo.By("Verifying the service has paired with the endpoint") 1151 err = framework.WaitForServiceEndpointsNum(ctx, client, namespace, serviceName, 1, 1*time.Second, 30*time.Second) 1152 framework.ExpectNoError(err, "waiting for service %s/%s have %d endpoint", namespace, serviceName, 1) 1153 } 1154 1155 func strPtr(s string) *string { return &s } 1156 1157 func registerWebhook(ctx context.Context, f *framework.Framework, markersNamespaceName string, configName string, certCtx *certContext, servicePort int32) { 1158 client := f.ClientSet 1159 ginkgo.By("Registering the webhook via the AdmissionRegistration API") 1160 1161 namespace := f.Namespace.Name 1162 // A webhook that cannot talk to server, with fail-open policy 1163 failOpenHook := failingWebhook(namespace, "fail-open.k8s.io", servicePort) 1164 policyIgnore := admissionregistrationv1.Ignore 1165 failOpenHook.FailurePolicy = &policyIgnore 1166 failOpenHook.NamespaceSelector = &metav1.LabelSelector{ 1167 MatchLabels: map[string]string{f.UniqueName: "true"}, 1168 } 1169 1170 _, err := createValidatingWebhookConfiguration(ctx, f, &admissionregistrationv1.ValidatingWebhookConfiguration{ 1171 ObjectMeta: metav1.ObjectMeta{ 1172 Name: configName, 1173 }, 1174 Webhooks: []admissionregistrationv1.ValidatingWebhook{ 1175 newDenyPodWebhookFixture(f, certCtx, servicePort), 1176 newDenyConfigMapWebhookFixture(f, certCtx, servicePort), 1177 // Server cannot talk to this webhook, so it always fails. 1178 // Because this webhook is configured fail-open, request should be admitted after the call fails. 1179 failOpenHook, 1180 1181 // Register a webhook that can be probed by marker requests to detect when the configuration is ready. 1182 newValidatingIsReadyWebhookFixture(f, certCtx, servicePort), 1183 }, 1184 }) 1185 framework.ExpectNoError(err, "registering webhook config %s with namespace %s", configName, namespace) 1186 1187 err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName) 1188 framework.ExpectNoError(err, "waiting for webhook configuration to be ready") 1189 1190 ginkgo.DeferCleanup(framework.IgnoreNotFound(client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete), configName, metav1.DeleteOptions{}) 1191 } 1192 1193 func registerWebhookForAttachingPod(ctx context.Context, f *framework.Framework, markersNamespaceName string, configName string, certCtx *certContext, servicePort int32) { 1194 client := f.ClientSet 1195 ginkgo.By("Registering the webhook via the AdmissionRegistration API") 1196 1197 namespace := f.Namespace.Name 1198 sideEffectsNone := admissionregistrationv1.SideEffectClassNone 1199 1200 _, err := createValidatingWebhookConfiguration(ctx, f, &admissionregistrationv1.ValidatingWebhookConfiguration{ 1201 ObjectMeta: metav1.ObjectMeta{ 1202 Name: configName, 1203 }, 1204 Webhooks: []admissionregistrationv1.ValidatingWebhook{ 1205 { 1206 Name: "deny-attaching-pod.k8s.io", 1207 Rules: []admissionregistrationv1.RuleWithOperations{{ 1208 Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Connect}, 1209 Rule: admissionregistrationv1.Rule{ 1210 APIGroups: []string{""}, 1211 APIVersions: []string{"v1"}, 1212 Resources: []string{"pods/attach"}, 1213 }, 1214 }}, 1215 ClientConfig: admissionregistrationv1.WebhookClientConfig{ 1216 Service: &admissionregistrationv1.ServiceReference{ 1217 Namespace: namespace, 1218 Name: serviceName, 1219 Path: strPtr("/pods/attach"), 1220 Port: pointer.Int32(servicePort), 1221 }, 1222 CABundle: certCtx.signingCert, 1223 }, 1224 SideEffects: &sideEffectsNone, 1225 AdmissionReviewVersions: []string{"v1", "v1beta1"}, 1226 // Scope the webhook to just this namespace 1227 NamespaceSelector: &metav1.LabelSelector{ 1228 MatchLabels: map[string]string{f.UniqueName: "true"}, 1229 }, 1230 }, 1231 // Register a webhook that can be probed by marker requests to detect when the configuration is ready. 1232 newValidatingIsReadyWebhookFixture(f, certCtx, servicePort), 1233 }, 1234 }) 1235 framework.ExpectNoError(err, "registering webhook config %s with namespace %s", configName, namespace) 1236 1237 err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName) 1238 framework.ExpectNoError(err, "waiting for webhook configuration to be ready") 1239 1240 ginkgo.DeferCleanup(framework.IgnoreNotFound(client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete), configName, metav1.DeleteOptions{}) 1241 } 1242 1243 func registerMutatingWebhookForConfigMap(ctx context.Context, f *framework.Framework, markersNamespaceName string, configName string, certCtx *certContext, servicePort int32) { 1244 client := f.ClientSet 1245 ginkgo.By("Registering the mutating configmap webhook via the AdmissionRegistration API") 1246 1247 namespace := f.Namespace.Name 1248 1249 _, err := createMutatingWebhookConfiguration(ctx, f, &admissionregistrationv1.MutatingWebhookConfiguration{ 1250 ObjectMeta: metav1.ObjectMeta{ 1251 Name: configName, 1252 }, 1253 Webhooks: []admissionregistrationv1.MutatingWebhook{ 1254 newMutateConfigMapWebhookFixture(f, certCtx, 1, servicePort), 1255 newMutateConfigMapWebhookFixture(f, certCtx, 2, servicePort), 1256 // Register a webhook that can be probed by marker requests to detect when the configuration is ready. 1257 newMutatingIsReadyWebhookFixture(f, certCtx, servicePort), 1258 }, 1259 }) 1260 framework.ExpectNoError(err, "registering mutating webhook config %s with namespace %s", configName, namespace) 1261 1262 err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName) 1263 framework.ExpectNoError(err, "waiting for webhook configuration to be ready") 1264 ginkgo.DeferCleanup(framework.IgnoreNotFound(client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete), configName, metav1.DeleteOptions{}) 1265 } 1266 1267 func testMutatingConfigMapWebhook(ctx context.Context, f *framework.Framework) { 1268 ginkgo.By("create a configmap that should be updated by the webhook") 1269 client := f.ClientSet 1270 configMap := toBeMutatedConfigMap(f) 1271 mutatedConfigMap, err := client.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, configMap, metav1.CreateOptions{}) 1272 framework.ExpectNoError(err) 1273 expectedConfigMapData := map[string]string{ 1274 "mutation-start": "yes", 1275 "mutation-stage-1": "yes", 1276 "mutation-stage-2": "yes", 1277 } 1278 if !reflect.DeepEqual(expectedConfigMapData, mutatedConfigMap.Data) { 1279 framework.Failf("\nexpected %#v\n, got %#v\n", expectedConfigMapData, mutatedConfigMap.Data) 1280 } 1281 } 1282 1283 func registerMutatingWebhookForPod(ctx context.Context, f *framework.Framework, markersNamespaceName string, configName string, certCtx *certContext, servicePort int32) { 1284 client := f.ClientSet 1285 ginkgo.By("Registering the mutating pod webhook via the AdmissionRegistration API") 1286 1287 namespace := f.Namespace.Name 1288 sideEffectsNone := admissionregistrationv1.SideEffectClassNone 1289 1290 _, err := createMutatingWebhookConfiguration(ctx, f, &admissionregistrationv1.MutatingWebhookConfiguration{ 1291 ObjectMeta: metav1.ObjectMeta{ 1292 Name: configName, 1293 }, 1294 Webhooks: []admissionregistrationv1.MutatingWebhook{ 1295 { 1296 Name: "adding-init-container.k8s.io", 1297 Rules: []admissionregistrationv1.RuleWithOperations{{ 1298 Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create}, 1299 Rule: admissionregistrationv1.Rule{ 1300 APIGroups: []string{""}, 1301 APIVersions: []string{"v1"}, 1302 Resources: []string{"pods"}, 1303 }, 1304 }}, 1305 ClientConfig: admissionregistrationv1.WebhookClientConfig{ 1306 Service: &admissionregistrationv1.ServiceReference{ 1307 Namespace: namespace, 1308 Name: serviceName, 1309 Path: strPtr("/mutating-pods"), 1310 Port: pointer.Int32(servicePort), 1311 }, 1312 CABundle: certCtx.signingCert, 1313 }, 1314 SideEffects: &sideEffectsNone, 1315 AdmissionReviewVersions: []string{"v1", "v1beta1"}, 1316 // Scope the webhook to just this namespace 1317 NamespaceSelector: &metav1.LabelSelector{ 1318 MatchLabels: map[string]string{f.UniqueName: "true"}, 1319 }, 1320 }, 1321 // Register a webhook that can be probed by marker requests to detect when the configuration is ready. 1322 newMutatingIsReadyWebhookFixture(f, certCtx, servicePort), 1323 }, 1324 }) 1325 framework.ExpectNoError(err, "registering mutating webhook config %s with namespace %s", configName, namespace) 1326 1327 err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName) 1328 framework.ExpectNoError(err, "waiting for webhook configuration to be ready") 1329 1330 ginkgo.DeferCleanup(framework.IgnoreNotFound(client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete), configName, metav1.DeleteOptions{}) 1331 } 1332 1333 func testMutatingPodWebhook(ctx context.Context, f *framework.Framework) { 1334 ginkgo.By("create a pod that should be updated by the webhook") 1335 client := f.ClientSet 1336 pod := toBeMutatedPod(f) 1337 mutatedPod, err := client.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{}) 1338 framework.ExpectNoError(err) 1339 if len(mutatedPod.Spec.InitContainers) != 1 { 1340 framework.Failf("expect pod to have 1 init container, got %#v", mutatedPod.Spec.InitContainers) 1341 } 1342 if got, expected := mutatedPod.Spec.InitContainers[0].Name, "webhook-added-init-container"; got != expected { 1343 framework.Failf("expect the init container name to be %q, got %q", expected, got) 1344 } 1345 if got, expected := mutatedPod.Spec.InitContainers[0].TerminationMessagePolicy, v1.TerminationMessageReadFile; got != expected { 1346 framework.Failf("expect the init terminationMessagePolicy to be default to %q, got %q", expected, got) 1347 } 1348 } 1349 1350 func toBeMutatedPod(f *framework.Framework) *v1.Pod { 1351 return &v1.Pod{ 1352 ObjectMeta: metav1.ObjectMeta{ 1353 Name: "webhook-to-be-mutated", 1354 }, 1355 Spec: v1.PodSpec{ 1356 Containers: []v1.Container{ 1357 { 1358 Name: "example", 1359 Image: imageutils.GetPauseImageName(), 1360 }, 1361 }, 1362 }, 1363 } 1364 } 1365 1366 func testWebhook(ctx context.Context, f *framework.Framework) { 1367 ginkgo.By("create a pod that should be denied by the webhook") 1368 client := f.ClientSet 1369 // Creating the pod, the request should be rejected 1370 pod := nonCompliantPod(f) 1371 _, err := client.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{}) 1372 gomega.Expect(err).To(gomega.HaveOccurred(), "create pod %s in namespace %s should have been denied by webhook", pod.Name, f.Namespace.Name) 1373 expectedErrMsg1 := "the pod contains unwanted container name" 1374 if !strings.Contains(err.Error(), expectedErrMsg1) { 1375 framework.Failf("expect error contains %q, got %q", expectedErrMsg1, err.Error()) 1376 } 1377 expectedErrMsg2 := "the pod contains unwanted label" 1378 if !strings.Contains(err.Error(), expectedErrMsg2) { 1379 framework.Failf("expect error contains %q, got %q", expectedErrMsg2, err.Error()) 1380 } 1381 1382 ginkgo.By("create a pod that causes the webhook to hang") 1383 client = f.ClientSet 1384 // Creating the pod, the request should be rejected 1385 pod = hangingPod(f) 1386 _, err = client.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{}) 1387 gomega.Expect(err).To(gomega.HaveOccurred(), "create pod %s in namespace %s should have caused webhook to hang", pod.Name, f.Namespace.Name) 1388 // ensure the error is webhook-related, not client-side 1389 if !strings.Contains(err.Error(), "webhook") { 1390 framework.Failf("expect error %q, got %q", "webhook", err.Error()) 1391 } 1392 // ensure the error is a timeout 1393 if !strings.Contains(err.Error(), "deadline") { 1394 framework.Failf("expect error %q, got %q", "deadline", err.Error()) 1395 } 1396 // ensure the pod was not actually created 1397 if _, err := client.CoreV1().Pods(f.Namespace.Name).Get(ctx, pod.Name, metav1.GetOptions{}); !apierrors.IsNotFound(err) { 1398 framework.Failf("expect notfound error looking for rejected pod, got %v", err) 1399 } 1400 1401 ginkgo.By("create a configmap that should be denied by the webhook") 1402 // Creating the configmap, the request should be rejected 1403 configmap := nonCompliantConfigMap(f) 1404 _, err = client.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, configmap, metav1.CreateOptions{}) 1405 gomega.Expect(err).To(gomega.HaveOccurred(), "create configmap %s in namespace %s should have been denied by the webhook", configmap.Name, f.Namespace.Name) 1406 expectedErrMsg := "the configmap contains unwanted key and value" 1407 if !strings.Contains(err.Error(), expectedErrMsg) { 1408 framework.Failf("expect error contains %q, got %q", expectedErrMsg, err.Error()) 1409 } 1410 1411 ginkgo.By("create a configmap that should be admitted by the webhook") 1412 // Creating the configmap, the request should be admitted 1413 configmap = &v1.ConfigMap{ 1414 ObjectMeta: metav1.ObjectMeta{ 1415 Name: allowedConfigMapName, 1416 }, 1417 Data: map[string]string{ 1418 "admit": "this", 1419 }, 1420 } 1421 _, err = client.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, configmap, metav1.CreateOptions{}) 1422 framework.ExpectNoError(err, "failed to create configmap %s in namespace: %s", configmap.Name, f.Namespace.Name) 1423 1424 ginkgo.By("update (PUT) the admitted configmap to a non-compliant one should be rejected by the webhook") 1425 toNonCompliantFn := func(cm *v1.ConfigMap) { 1426 if cm.Data == nil { 1427 cm.Data = map[string]string{} 1428 } 1429 cm.Data["webhook-e2e-test"] = "webhook-disallow" 1430 } 1431 _, err = updateConfigMap(ctx, client, f.Namespace.Name, allowedConfigMapName, toNonCompliantFn) 1432 gomega.Expect(err).To(gomega.HaveOccurred(), "update (PUT) admitted configmap %s in namespace %s to a non-compliant one should be rejected by webhook", allowedConfigMapName, f.Namespace.Name) 1433 if !strings.Contains(err.Error(), expectedErrMsg) { 1434 framework.Failf("expect error contains %q, got %q", expectedErrMsg, err.Error()) 1435 } 1436 1437 ginkgo.By("update (PATCH) the admitted configmap to a non-compliant one should be rejected by the webhook") 1438 patch := nonCompliantConfigMapPatch() 1439 _, err = client.CoreV1().ConfigMaps(f.Namespace.Name).Patch(ctx, allowedConfigMapName, types.StrategicMergePatchType, []byte(patch), metav1.PatchOptions{}) 1440 gomega.Expect(err).To(gomega.HaveOccurred(), "update admitted configmap %s in namespace %s by strategic merge patch to a non-compliant one should be rejected by webhook. Patch: %+v", allowedConfigMapName, f.Namespace.Name, patch) 1441 if !strings.Contains(err.Error(), expectedErrMsg) { 1442 framework.Failf("expect error contains %q, got %q", expectedErrMsg, err.Error()) 1443 } 1444 1445 ginkgo.By("create a namespace that bypass the webhook") 1446 // skipNamespace will be deleted by framework at the end of the test 1447 skipNamespace, err := f.CreateNamespace(ctx, skipNamespaceBaseName, map[string]string{ 1448 skipNamespaceLabelKey: skipNamespaceLabelValue, 1449 f.UniqueName: "true", 1450 }) 1451 framework.ExpectNoError(err, "creating namespace %q", skipNamespaceBaseName) 1452 skipNamespaceName := skipNamespace.Name 1453 1454 ginkgo.By("create a configmap that violates the webhook policy but is in a whitelisted namespace") 1455 configmap = nonCompliantConfigMap(f) 1456 _, err = client.CoreV1().ConfigMaps(skipNamespaceName).Create(ctx, configmap, metav1.CreateOptions{}) 1457 framework.ExpectNoError(err, "failed to create configmap %s in namespace: %s", configmap.Name, skipNamespaceName) 1458 } 1459 1460 func testAttachingPodWebhook(ctx context.Context, f *framework.Framework) { 1461 ginkgo.By("create a pod") 1462 client := f.ClientSet 1463 pod := toBeAttachedPod(f) 1464 _, err := client.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{}) 1465 framework.ExpectNoError(err, "failed to create pod %s in namespace: %s", pod.Name, f.Namespace.Name) 1466 err = e2epod.WaitForPodNameRunningInNamespace(ctx, client, pod.Name, f.Namespace.Name) 1467 framework.ExpectNoError(err, "error while waiting for pod %s to go to Running phase in namespace: %s", pod.Name, f.Namespace.Name) 1468 1469 ginkgo.By("'kubectl attach' the pod, should be denied by the webhook") 1470 timer := time.NewTimer(30 * time.Second) 1471 defer timer.Stop() 1472 _, err = e2ekubectl.NewKubectlCommand(f.Namespace.Name, "attach", fmt.Sprintf("--namespace=%v", f.Namespace.Name), pod.Name, "-i", "-c=container1").WithTimeout(timer.C).Exec() 1473 gomega.Expect(err).To(gomega.HaveOccurred(), "'kubectl attach' the pod, should be denied by the webhook") 1474 if e, a := "attaching to pod 'to-be-attached-pod' is not allowed", err.Error(); !strings.Contains(a, e) { 1475 framework.Failf("unexpected 'kubectl attach' error message. expected to contain %q, got %q", e, a) 1476 } 1477 } 1478 1479 // failingWebhook returns a webhook with rule of create configmaps, 1480 // but with an invalid client config so that server cannot communicate with it 1481 func failingWebhook(namespace, name string, servicePort int32) admissionregistrationv1.ValidatingWebhook { 1482 sideEffectsNone := admissionregistrationv1.SideEffectClassNone 1483 1484 return admissionregistrationv1.ValidatingWebhook{ 1485 Name: name, 1486 Rules: []admissionregistrationv1.RuleWithOperations{{ 1487 Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create}, 1488 Rule: admissionregistrationv1.Rule{ 1489 APIGroups: []string{""}, 1490 APIVersions: []string{"v1"}, 1491 Resources: []string{"configmaps"}, 1492 }, 1493 }}, 1494 ClientConfig: admissionregistrationv1.WebhookClientConfig{ 1495 Service: &admissionregistrationv1.ServiceReference{ 1496 Namespace: namespace, 1497 Name: serviceName, 1498 Path: strPtr("/configmaps"), 1499 Port: pointer.Int32(servicePort), 1500 }, 1501 // Without CA bundle, the call to webhook always fails 1502 CABundle: nil, 1503 }, 1504 SideEffects: &sideEffectsNone, 1505 AdmissionReviewVersions: []string{"v1", "v1beta1"}, 1506 } 1507 } 1508 1509 func registerFailClosedWebhook(ctx context.Context, f *framework.Framework, markersNamespaceName string, configName string, certCtx *certContext, servicePort int32) { 1510 ginkgo.By("Registering a webhook that server cannot talk to, with fail closed policy, via the AdmissionRegistration API") 1511 1512 namespace := f.Namespace.Name 1513 // A webhook that cannot talk to server, with fail-closed policy 1514 policyFail := admissionregistrationv1.Fail 1515 hook := failingWebhook(namespace, "fail-closed.k8s.io", servicePort) 1516 hook.FailurePolicy = &policyFail 1517 hook.NamespaceSelector = &metav1.LabelSelector{ 1518 MatchLabels: map[string]string{f.UniqueName: "true"}, 1519 MatchExpressions: []metav1.LabelSelectorRequirement{ 1520 { 1521 Key: failNamespaceLabelKey, 1522 Operator: metav1.LabelSelectorOpIn, 1523 Values: []string{failNamespaceLabelValue}, 1524 }, 1525 }, 1526 } 1527 1528 _, err := createValidatingWebhookConfiguration(ctx, f, &admissionregistrationv1.ValidatingWebhookConfiguration{ 1529 ObjectMeta: metav1.ObjectMeta{ 1530 Name: configName, 1531 }, 1532 Webhooks: []admissionregistrationv1.ValidatingWebhook{ 1533 // Server cannot talk to this webhook, so it always fails. 1534 // Because this webhook is configured fail-closed, request should be rejected after the call fails. 1535 hook, 1536 // Register a webhook that can be probed by marker requests to detect when the configuration is ready. 1537 newValidatingIsReadyWebhookFixture(f, certCtx, servicePort), 1538 }, 1539 }) 1540 framework.ExpectNoError(err, "registering webhook config %s with namespace %s", configName, namespace) 1541 1542 err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName) 1543 framework.ExpectNoError(err, "waiting for webhook configuration to be ready") 1544 ginkgo.DeferCleanup(framework.IgnoreNotFound(f.ClientSet.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete), configName, metav1.DeleteOptions{}) 1545 } 1546 1547 func testFailClosedWebhook(ctx context.Context, f *framework.Framework) { 1548 client := f.ClientSet 1549 ginkgo.By("create a namespace for the webhook") 1550 // failNamespace will be deleted by framework at the end of the test 1551 failNamespace, err := f.CreateNamespace(ctx, failNamespaceBaseName, map[string]string{ 1552 failNamespaceLabelKey: failNamespaceLabelValue, 1553 f.UniqueName: "true", 1554 }) 1555 framework.ExpectNoError(err, "creating namespace %q", failNamespaceBaseName) 1556 failNamespaceName := failNamespace.Name 1557 1558 ginkgo.By("create a configmap should be unconditionally rejected by the webhook") 1559 configmap := &v1.ConfigMap{ 1560 ObjectMeta: metav1.ObjectMeta{ 1561 Name: "foo", 1562 }, 1563 } 1564 _, err = client.CoreV1().ConfigMaps(failNamespaceName).Create(ctx, configmap, metav1.CreateOptions{}) 1565 gomega.Expect(err).To(gomega.HaveOccurred(), "create configmap in namespace: %s should be unconditionally rejected by the webhook", failNamespaceName) 1566 if !apierrors.IsInternalError(err) { 1567 framework.Failf("expect an internal error, got %#v", err) 1568 } 1569 } 1570 1571 func registerValidatingWebhookForWebhookConfigurations(ctx context.Context, f *framework.Framework, markersNamespaceName string, configName string, certCtx *certContext, servicePort int32) { 1572 var err error 1573 client := f.ClientSet 1574 ginkgo.By("Registering a validating webhook on ValidatingWebhookConfiguration and MutatingWebhookConfiguration objects, via the AdmissionRegistration API") 1575 1576 namespace := f.Namespace.Name 1577 failurePolicy := admissionregistrationv1.Fail 1578 sideEffectsNone := admissionregistrationv1.SideEffectClassNone 1579 1580 // This webhook denies all requests to Delete validating webhook configuration and 1581 // mutating webhook configuration objects. It should never be called, however, because 1582 // dynamic admission webhooks should not be called on requests involving webhook configuration objects. 1583 _, err = createValidatingWebhookConfiguration(ctx, f, &admissionregistrationv1.ValidatingWebhookConfiguration{ 1584 ObjectMeta: metav1.ObjectMeta{ 1585 Name: configName, 1586 }, 1587 Webhooks: []admissionregistrationv1.ValidatingWebhook{ 1588 { 1589 Name: "deny-webhook-configuration-deletions.k8s.io", 1590 Rules: []admissionregistrationv1.RuleWithOperations{{ 1591 Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Delete}, 1592 Rule: admissionregistrationv1.Rule{ 1593 APIGroups: []string{"admissionregistration.k8s.io"}, 1594 APIVersions: []string{"*"}, 1595 Resources: []string{ 1596 "validatingwebhookconfigurations", 1597 "mutatingwebhookconfigurations", 1598 }, 1599 }, 1600 }}, 1601 ClientConfig: admissionregistrationv1.WebhookClientConfig{ 1602 Service: &admissionregistrationv1.ServiceReference{ 1603 Namespace: namespace, 1604 Name: serviceName, 1605 Path: strPtr("/always-deny"), 1606 Port: pointer.Int32(servicePort), 1607 }, 1608 CABundle: certCtx.signingCert, 1609 }, 1610 SideEffects: &sideEffectsNone, 1611 AdmissionReviewVersions: []string{"v1", "v1beta1"}, 1612 FailurePolicy: &failurePolicy, 1613 // Scope the webhook to just this namespace 1614 NamespaceSelector: &metav1.LabelSelector{ 1615 MatchLabels: map[string]string{f.UniqueName: "true"}, 1616 }, 1617 }, 1618 // Register a webhook that can be probed by marker requests to detect when the configuration is ready. 1619 newValidatingIsReadyWebhookFixture(f, certCtx, servicePort), 1620 }, 1621 }) 1622 framework.ExpectNoError(err, "registering webhook config %s with namespace %s", configName, namespace) 1623 1624 err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName) 1625 framework.ExpectNoError(err, "waiting for webhook configuration to be ready") 1626 ginkgo.DeferCleanup(framework.IgnoreNotFound(client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete), configName, metav1.DeleteOptions{}) 1627 } 1628 1629 func registerMutatingWebhookForWebhookConfigurations(ctx context.Context, f *framework.Framework, markersNamespaceName string, configName string, certCtx *certContext, servicePort int32) { 1630 var err error 1631 client := f.ClientSet 1632 ginkgo.By("Registering a mutating webhook on ValidatingWebhookConfiguration and MutatingWebhookConfiguration objects, via the AdmissionRegistration API") 1633 1634 namespace := f.Namespace.Name 1635 failurePolicy := admissionregistrationv1.Fail 1636 sideEffectsNone := admissionregistrationv1.SideEffectClassNone 1637 1638 // This webhook adds a label to all requests create to validating webhook configuration and 1639 // mutating webhook configuration objects. It should never be called, however, because 1640 // dynamic admission webhooks should not be called on requests involving webhook configuration objects. 1641 _, err = createMutatingWebhookConfiguration(ctx, f, &admissionregistrationv1.MutatingWebhookConfiguration{ 1642 ObjectMeta: metav1.ObjectMeta{ 1643 Name: configName, 1644 }, 1645 Webhooks: []admissionregistrationv1.MutatingWebhook{ 1646 { 1647 Name: "add-label-to-webhook-configurations.k8s.io", 1648 Rules: []admissionregistrationv1.RuleWithOperations{{ 1649 Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create}, 1650 Rule: admissionregistrationv1.Rule{ 1651 APIGroups: []string{"admissionregistration.k8s.io"}, 1652 APIVersions: []string{"*"}, 1653 Resources: []string{ 1654 "validatingwebhookconfigurations", 1655 "mutatingwebhookconfigurations", 1656 }, 1657 }, 1658 }}, 1659 ClientConfig: admissionregistrationv1.WebhookClientConfig{ 1660 Service: &admissionregistrationv1.ServiceReference{ 1661 Namespace: namespace, 1662 Name: serviceName, 1663 Path: strPtr("/add-label"), 1664 Port: pointer.Int32(servicePort), 1665 }, 1666 CABundle: certCtx.signingCert, 1667 }, 1668 SideEffects: &sideEffectsNone, 1669 AdmissionReviewVersions: []string{"v1", "v1beta1"}, 1670 FailurePolicy: &failurePolicy, 1671 // Scope the webhook to just this namespace 1672 NamespaceSelector: &metav1.LabelSelector{ 1673 MatchLabels: map[string]string{f.UniqueName: "true"}, 1674 }, 1675 }, 1676 // Register a webhook that can be probed by marker requests to detect when the configuration is ready. 1677 newMutatingIsReadyWebhookFixture(f, certCtx, servicePort), 1678 }, 1679 }) 1680 framework.ExpectNoError(err, "registering webhook config %s with namespace %s", configName, namespace) 1681 1682 err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName) 1683 framework.ExpectNoError(err, "waiting for webhook configuration to be ready") 1684 ginkgo.DeferCleanup(framework.IgnoreNotFound(client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete), configName, metav1.DeleteOptions{}) 1685 } 1686 1687 // This test assumes that the deletion-rejecting webhook defined in 1688 // registerValidatingWebhookForWebhookConfigurations and the webhook-config-mutating 1689 // webhook defined in registerMutatingWebhookForWebhookConfigurations already exist. 1690 func testWebhooksForWebhookConfigurations(ctx context.Context, f *framework.Framework, markersNamespaceName string, configName string, certCtx *certContext, servicePort int32) { 1691 var err error 1692 client := f.ClientSet 1693 ginkgo.By("Creating a dummy validating-webhook-configuration object") 1694 1695 namespace := f.Namespace.Name 1696 failurePolicy := admissionregistrationv1.Ignore 1697 sideEffectsNone := admissionregistrationv1.SideEffectClassNone 1698 1699 mutatedValidatingWebhookConfiguration, err := createValidatingWebhookConfiguration(ctx, f, &admissionregistrationv1.ValidatingWebhookConfiguration{ 1700 ObjectMeta: metav1.ObjectMeta{ 1701 Name: configName, 1702 }, 1703 Webhooks: []admissionregistrationv1.ValidatingWebhook{ 1704 { 1705 Name: "dummy-validating-webhook.k8s.io", 1706 Rules: []admissionregistrationv1.RuleWithOperations{{ 1707 Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create}, 1708 // This will not match any real resources so this webhook should never be called. 1709 Rule: admissionregistrationv1.Rule{ 1710 APIGroups: []string{""}, 1711 APIVersions: []string{"v1"}, 1712 Resources: []string{"invalid"}, 1713 }, 1714 }}, 1715 ClientConfig: admissionregistrationv1.WebhookClientConfig{ 1716 Service: &admissionregistrationv1.ServiceReference{ 1717 Namespace: namespace, 1718 Name: serviceName, 1719 // This path not recognized by the webhook service, 1720 // so the call to this webhook will always fail, 1721 // but because the failure policy is ignore, it will 1722 // have no effect on admission requests. 1723 Path: strPtr(""), 1724 Port: pointer.Int32(servicePort), 1725 }, 1726 CABundle: nil, 1727 }, 1728 SideEffects: &sideEffectsNone, 1729 AdmissionReviewVersions: []string{"v1", "v1beta1"}, 1730 FailurePolicy: &failurePolicy, 1731 // Scope the webhook to just this namespace 1732 NamespaceSelector: &metav1.LabelSelector{ 1733 MatchLabels: map[string]string{f.UniqueName: "true"}, 1734 }, 1735 }, 1736 // Register a webhook that can be probed by marker requests to detect when the configuration is ready. 1737 newValidatingIsReadyWebhookFixture(f, certCtx, servicePort), 1738 }, 1739 }) 1740 framework.ExpectNoError(err, "registering webhook config %s with namespace %s", configName, namespace) 1741 if mutatedValidatingWebhookConfiguration.ObjectMeta.Labels != nil && mutatedValidatingWebhookConfiguration.ObjectMeta.Labels[addedLabelKey] == addedLabelValue { 1742 framework.Failf("expected %s not to be mutated by mutating webhooks but it was", configName) 1743 } 1744 1745 err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName) 1746 framework.ExpectNoError(err, "waiting for webhook configuration to be ready") 1747 1748 ginkgo.By("Deleting the validating-webhook-configuration, which should be possible to remove") 1749 1750 err = client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(ctx, configName, metav1.DeleteOptions{}) 1751 framework.ExpectNoError(err, "deleting webhook config %s with namespace %s", configName, namespace) 1752 1753 ginkgo.By("Creating a dummy mutating-webhook-configuration object") 1754 1755 mutatedMutatingWebhookConfiguration, err := createMutatingWebhookConfiguration(ctx, f, &admissionregistrationv1.MutatingWebhookConfiguration{ 1756 ObjectMeta: metav1.ObjectMeta{ 1757 Name: configName, 1758 }, 1759 Webhooks: []admissionregistrationv1.MutatingWebhook{ 1760 { 1761 Name: "dummy-mutating-webhook.k8s.io", 1762 Rules: []admissionregistrationv1.RuleWithOperations{{ 1763 Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create}, 1764 // This will not match any real resources so this webhook should never be called. 1765 Rule: admissionregistrationv1.Rule{ 1766 APIGroups: []string{""}, 1767 APIVersions: []string{"v1"}, 1768 Resources: []string{"invalid"}, 1769 }, 1770 }}, 1771 ClientConfig: admissionregistrationv1.WebhookClientConfig{ 1772 Service: &admissionregistrationv1.ServiceReference{ 1773 Namespace: namespace, 1774 Name: serviceName, 1775 // This path not recognized by the webhook service, 1776 // so the call to this webhook will always fail, 1777 // but because the failure policy is ignore, it will 1778 // have no effect on admission requests. 1779 Path: strPtr(""), 1780 Port: pointer.Int32(servicePort), 1781 }, 1782 CABundle: nil, 1783 }, 1784 SideEffects: &sideEffectsNone, 1785 AdmissionReviewVersions: []string{"v1", "v1beta1"}, 1786 FailurePolicy: &failurePolicy, 1787 // Scope the webhook to just this namespace 1788 NamespaceSelector: &metav1.LabelSelector{ 1789 MatchLabels: map[string]string{f.UniqueName: "true"}, 1790 }, 1791 }, 1792 // Register a webhook that can be probed by marker requests to detect when the configuration is ready. 1793 newMutatingIsReadyWebhookFixture(f, certCtx, servicePort), 1794 }, 1795 }) 1796 framework.ExpectNoError(err, "registering webhook config %s with namespace %s", configName, namespace) 1797 if mutatedMutatingWebhookConfiguration.ObjectMeta.Labels != nil && mutatedMutatingWebhookConfiguration.ObjectMeta.Labels[addedLabelKey] == addedLabelValue { 1798 framework.Failf("expected %s not to be mutated by mutating webhooks but it was", configName) 1799 } 1800 1801 err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName) 1802 framework.ExpectNoError(err, "waiting for webhook configuration to be ready") 1803 1804 ginkgo.By("Deleting the mutating-webhook-configuration, which should be possible to remove") 1805 1806 err = client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(ctx, configName, metav1.DeleteOptions{}) 1807 framework.ExpectNoError(err, "deleting webhook config %s with namespace %s", configName, namespace) 1808 } 1809 1810 func nonCompliantPod(f *framework.Framework) *v1.Pod { 1811 return &v1.Pod{ 1812 ObjectMeta: metav1.ObjectMeta{ 1813 Name: disallowedPodName, 1814 Labels: map[string]string{ 1815 "webhook-e2e-test": "webhook-disallow", 1816 }, 1817 }, 1818 Spec: v1.PodSpec{ 1819 Containers: []v1.Container{ 1820 { 1821 Name: "webhook-disallow", 1822 Image: imageutils.GetPauseImageName(), 1823 }, 1824 }, 1825 }, 1826 } 1827 } 1828 1829 func hangingPod(f *framework.Framework) *v1.Pod { 1830 return &v1.Pod{ 1831 ObjectMeta: metav1.ObjectMeta{ 1832 Name: hangingPodName, 1833 Labels: map[string]string{ 1834 "webhook-e2e-test": "wait-forever", 1835 }, 1836 }, 1837 Spec: v1.PodSpec{ 1838 Containers: []v1.Container{ 1839 { 1840 Name: "wait-forever", 1841 Image: imageutils.GetPauseImageName(), 1842 }, 1843 }, 1844 }, 1845 } 1846 } 1847 1848 func toBeAttachedPod(f *framework.Framework) *v1.Pod { 1849 return &v1.Pod{ 1850 ObjectMeta: metav1.ObjectMeta{ 1851 Name: toBeAttachedPodName, 1852 }, 1853 Spec: v1.PodSpec{ 1854 Containers: []v1.Container{ 1855 { 1856 Name: "container1", 1857 Image: imageutils.GetPauseImageName(), 1858 }, 1859 }, 1860 }, 1861 } 1862 } 1863 1864 func nonCompliantConfigMap(f *framework.Framework) *v1.ConfigMap { 1865 return namedNonCompliantConfigMap(disallowedConfigMapName, f) 1866 } 1867 1868 func namedNonCompliantConfigMap(name string, f *framework.Framework) *v1.ConfigMap { 1869 return &v1.ConfigMap{ 1870 ObjectMeta: metav1.ObjectMeta{ 1871 Name: name, 1872 }, 1873 Data: map[string]string{ 1874 "webhook-e2e-test": "webhook-disallow", 1875 }, 1876 } 1877 } 1878 1879 func toBeMutatedConfigMap(f *framework.Framework) *v1.ConfigMap { 1880 return namedToBeMutatedConfigMap("to-be-mutated", f) 1881 } 1882 1883 func namedToBeMutatedConfigMap(name string, f *framework.Framework) *v1.ConfigMap { 1884 return &v1.ConfigMap{ 1885 ObjectMeta: metav1.ObjectMeta{ 1886 Name: name, 1887 }, 1888 Data: map[string]string{ 1889 "mutation-start": "yes", 1890 }, 1891 } 1892 } 1893 1894 func nonCompliantConfigMapPatch() string { 1895 return fmt.Sprint(`{"data":{"webhook-e2e-test":"webhook-disallow"}}`) 1896 } 1897 1898 type updateConfigMapFn func(cm *v1.ConfigMap) 1899 1900 func updateConfigMap(ctx context.Context, c clientset.Interface, ns, name string, update updateConfigMapFn) (*v1.ConfigMap, error) { 1901 var cm *v1.ConfigMap 1902 pollErr := wait.PollImmediate(2*time.Second, 1*time.Minute, func() (bool, error) { 1903 var err error 1904 if cm, err = c.CoreV1().ConfigMaps(ns).Get(ctx, name, metav1.GetOptions{}); err != nil { 1905 return false, err 1906 } 1907 update(cm) 1908 if cm, err = c.CoreV1().ConfigMaps(ns).Update(ctx, cm, metav1.UpdateOptions{}); err == nil { 1909 return true, nil 1910 } 1911 // Only retry update on conflict 1912 if !apierrors.IsConflict(err) { 1913 return false, err 1914 } 1915 return false, nil 1916 }) 1917 return cm, pollErr 1918 } 1919 1920 type updateCustomResourceFn func(cm *unstructured.Unstructured) 1921 1922 func updateCustomResource(ctx context.Context, c dynamic.ResourceInterface, ns, name string, update updateCustomResourceFn) (*unstructured.Unstructured, error) { 1923 var cr *unstructured.Unstructured 1924 pollErr := wait.PollImmediate(2*time.Second, 1*time.Minute, func() (bool, error) { 1925 var err error 1926 if cr, err = c.Get(ctx, name, metav1.GetOptions{}); err != nil { 1927 return false, err 1928 } 1929 update(cr) 1930 if cr, err = c.Update(ctx, cr, metav1.UpdateOptions{}); err == nil { 1931 return true, nil 1932 } 1933 // Only retry update on conflict 1934 if !apierrors.IsConflict(err) { 1935 return false, err 1936 } 1937 return false, nil 1938 }) 1939 return cr, pollErr 1940 } 1941 1942 func cleanWebhookTest(ctx context.Context, client clientset.Interface, namespaceName string) { 1943 _ = client.CoreV1().Services(namespaceName).Delete(ctx, serviceName, metav1.DeleteOptions{}) 1944 _ = client.AppsV1().Deployments(namespaceName).Delete(ctx, deploymentName, metav1.DeleteOptions{}) 1945 _ = client.CoreV1().Secrets(namespaceName).Delete(ctx, secretName, metav1.DeleteOptions{}) 1946 _ = client.RbacV1().RoleBindings("kube-system").Delete(ctx, roleBindingName, metav1.DeleteOptions{}) 1947 } 1948 1949 func registerWebhookForCustomResource(ctx context.Context, f *framework.Framework, markersNamespaceName string, configName string, certCtx *certContext, testcrd *crd.TestCrd, servicePort int32) { 1950 client := f.ClientSet 1951 ginkgo.By("Registering the custom resource webhook via the AdmissionRegistration API") 1952 1953 namespace := f.Namespace.Name 1954 sideEffectsNone := admissionregistrationv1.SideEffectClassNone 1955 1956 _, err := createValidatingWebhookConfiguration(ctx, f, &admissionregistrationv1.ValidatingWebhookConfiguration{ 1957 ObjectMeta: metav1.ObjectMeta{ 1958 Name: configName, 1959 }, 1960 Webhooks: []admissionregistrationv1.ValidatingWebhook{ 1961 { 1962 Name: "deny-unwanted-custom-resource-data.k8s.io", 1963 Rules: []admissionregistrationv1.RuleWithOperations{{ 1964 Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create, admissionregistrationv1.Update, admissionregistrationv1.Delete}, 1965 Rule: admissionregistrationv1.Rule{ 1966 APIGroups: []string{testcrd.Crd.Spec.Group}, 1967 APIVersions: servedAPIVersions(testcrd.Crd), 1968 Resources: []string{testcrd.Crd.Spec.Names.Plural}, 1969 }, 1970 }}, 1971 ClientConfig: admissionregistrationv1.WebhookClientConfig{ 1972 Service: &admissionregistrationv1.ServiceReference{ 1973 Namespace: namespace, 1974 Name: serviceName, 1975 Path: strPtr("/custom-resource"), 1976 Port: pointer.Int32(servicePort), 1977 }, 1978 CABundle: certCtx.signingCert, 1979 }, 1980 SideEffects: &sideEffectsNone, 1981 AdmissionReviewVersions: []string{"v1", "v1beta1"}, 1982 // Scope the webhook to just this namespace 1983 NamespaceSelector: &metav1.LabelSelector{ 1984 MatchLabels: map[string]string{f.UniqueName: "true"}, 1985 }, 1986 }, 1987 // Register a webhook that can be probed by marker requests to detect when the configuration is ready. 1988 newValidatingIsReadyWebhookFixture(f, certCtx, servicePort), 1989 }, 1990 }) 1991 framework.ExpectNoError(err, "registering custom resource webhook config %s with namespace %s", configName, namespace) 1992 1993 err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName) 1994 framework.ExpectNoError(err, "waiting for webhook configuration to be ready") 1995 ginkgo.DeferCleanup(framework.IgnoreNotFound(client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete), configName, metav1.DeleteOptions{}) 1996 } 1997 1998 func registerMutatingWebhookForCustomResource(ctx context.Context, f *framework.Framework, markersNamespaceName string, configName string, certCtx *certContext, testcrd *crd.TestCrd, servicePort int32) { 1999 client := f.ClientSet 2000 ginkgo.By(fmt.Sprintf("Registering the mutating webhook for custom resource %s via the AdmissionRegistration API", testcrd.Crd.Name)) 2001 2002 namespace := f.Namespace.Name 2003 sideEffectsNone := admissionregistrationv1.SideEffectClassNone 2004 2005 _, err := createMutatingWebhookConfiguration(ctx, f, &admissionregistrationv1.MutatingWebhookConfiguration{ 2006 ObjectMeta: metav1.ObjectMeta{ 2007 Name: configName, 2008 }, 2009 Webhooks: []admissionregistrationv1.MutatingWebhook{ 2010 { 2011 Name: "mutate-custom-resource-data-stage-1.k8s.io", 2012 Rules: []admissionregistrationv1.RuleWithOperations{{ 2013 Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create, admissionregistrationv1.Update}, 2014 Rule: admissionregistrationv1.Rule{ 2015 APIGroups: []string{testcrd.Crd.Spec.Group}, 2016 APIVersions: servedAPIVersions(testcrd.Crd), 2017 Resources: []string{testcrd.Crd.Spec.Names.Plural}, 2018 }, 2019 }}, 2020 ClientConfig: admissionregistrationv1.WebhookClientConfig{ 2021 Service: &admissionregistrationv1.ServiceReference{ 2022 Namespace: namespace, 2023 Name: serviceName, 2024 Path: strPtr("/mutating-custom-resource"), 2025 Port: pointer.Int32(servicePort), 2026 }, 2027 CABundle: certCtx.signingCert, 2028 }, 2029 SideEffects: &sideEffectsNone, 2030 AdmissionReviewVersions: []string{"v1", "v1beta1"}, 2031 // Scope the webhook to just this namespace 2032 NamespaceSelector: &metav1.LabelSelector{ 2033 MatchLabels: map[string]string{f.UniqueName: "true"}, 2034 }, 2035 }, 2036 { 2037 Name: "mutate-custom-resource-data-stage-2.k8s.io", 2038 Rules: []admissionregistrationv1.RuleWithOperations{{ 2039 Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create}, 2040 Rule: admissionregistrationv1.Rule{ 2041 APIGroups: []string{testcrd.Crd.Spec.Group}, 2042 APIVersions: servedAPIVersions(testcrd.Crd), 2043 Resources: []string{testcrd.Crd.Spec.Names.Plural}, 2044 }, 2045 }}, 2046 ClientConfig: admissionregistrationv1.WebhookClientConfig{ 2047 Service: &admissionregistrationv1.ServiceReference{ 2048 Namespace: namespace, 2049 Name: serviceName, 2050 Path: strPtr("/mutating-custom-resource"), 2051 Port: pointer.Int32(servicePort), 2052 }, 2053 CABundle: certCtx.signingCert, 2054 }, 2055 SideEffects: &sideEffectsNone, 2056 AdmissionReviewVersions: []string{"v1", "v1beta1"}, 2057 // Scope the webhook to just this namespace 2058 NamespaceSelector: &metav1.LabelSelector{ 2059 MatchLabels: map[string]string{f.UniqueName: "true"}, 2060 }, 2061 }, 2062 // Register a webhook that can be probed by marker requests to detect when the configuration is ready. 2063 newMutatingIsReadyWebhookFixture(f, certCtx, servicePort), 2064 }, 2065 }) 2066 framework.ExpectNoError(err, "registering custom resource webhook config %s with namespace %s", configName, namespace) 2067 2068 err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName) 2069 framework.ExpectNoError(err, "waiting for webhook configuration to be ready") 2070 2071 ginkgo.DeferCleanup(framework.IgnoreNotFound(client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete), configName, metav1.DeleteOptions{}) 2072 } 2073 2074 func testCustomResourceWebhook(ctx context.Context, f *framework.Framework, crd *apiextensionsv1.CustomResourceDefinition, customResourceClient dynamic.ResourceInterface) { 2075 ginkgo.By("Creating a custom resource that should be denied by the webhook") 2076 crInstanceName := "cr-instance-1" 2077 crInstance := &unstructured.Unstructured{ 2078 Object: map[string]interface{}{ 2079 "kind": crd.Spec.Names.Kind, 2080 "apiVersion": crd.Spec.Group + "/" + crd.Spec.Versions[0].Name, 2081 "metadata": map[string]interface{}{ 2082 "name": crInstanceName, 2083 "namespace": f.Namespace.Name, 2084 }, 2085 "data": map[string]interface{}{ 2086 "webhook-e2e-test": "webhook-disallow", 2087 }, 2088 }, 2089 } 2090 _, err := customResourceClient.Create(ctx, crInstance, metav1.CreateOptions{}) 2091 gomega.Expect(err).To(gomega.HaveOccurred(), "create custom resource %s in namespace %s should be denied by webhook", crInstanceName, f.Namespace.Name) 2092 expectedErrMsg := "the custom resource contains unwanted data" 2093 if !strings.Contains(err.Error(), expectedErrMsg) { 2094 framework.Failf("expect error contains %q, got %q", expectedErrMsg, err.Error()) 2095 } 2096 } 2097 2098 func testBlockingCustomResourceUpdateDeletion(ctx context.Context, f *framework.Framework, crd *apiextensionsv1.CustomResourceDefinition, customResourceClient dynamic.ResourceInterface) { 2099 ginkgo.By("Creating a custom resource whose deletion would be denied by the webhook") 2100 crInstanceName := "cr-instance-2" 2101 crInstance := &unstructured.Unstructured{ 2102 Object: map[string]interface{}{ 2103 "kind": crd.Spec.Names.Kind, 2104 "apiVersion": crd.Spec.Group + "/" + crd.Spec.Versions[0].Name, 2105 "metadata": map[string]interface{}{ 2106 "name": crInstanceName, 2107 "namespace": f.Namespace.Name, 2108 }, 2109 "data": map[string]interface{}{ 2110 "webhook-e2e-test": "webhook-nondeletable", 2111 }, 2112 }, 2113 } 2114 _, err := customResourceClient.Create(ctx, crInstance, metav1.CreateOptions{}) 2115 framework.ExpectNoError(err, "failed to create custom resource %s in namespace: %s", crInstanceName, f.Namespace.Name) 2116 2117 ginkgo.By("Updating the custom resource with disallowed data should be denied") 2118 toNonCompliantFn := func(cr *unstructured.Unstructured) { 2119 if _, ok := cr.Object["data"]; !ok { 2120 cr.Object["data"] = map[string]interface{}{} 2121 } 2122 data := cr.Object["data"].(map[string]interface{}) 2123 data["webhook-e2e-test"] = "webhook-disallow" 2124 } 2125 _, err = updateCustomResource(ctx, customResourceClient, f.Namespace.Name, crInstanceName, toNonCompliantFn) 2126 gomega.Expect(err).To(gomega.HaveOccurred(), "updating custom resource %s in namespace: %s should be denied", crInstanceName, f.Namespace.Name) 2127 2128 expectedErrMsg := "the custom resource contains unwanted data" 2129 if !strings.Contains(err.Error(), expectedErrMsg) { 2130 framework.Failf("expect error contains %q, got %q", expectedErrMsg, err.Error()) 2131 } 2132 2133 ginkgo.By("Deleting the custom resource should be denied") 2134 err = customResourceClient.Delete(ctx, crInstanceName, metav1.DeleteOptions{}) 2135 gomega.Expect(err).To(gomega.HaveOccurred(), "deleting custom resource %s in namespace: %s should be denied", crInstanceName, f.Namespace.Name) 2136 expectedErrMsg1 := "the custom resource cannot be deleted because it contains unwanted key and value" 2137 if !strings.Contains(err.Error(), expectedErrMsg1) { 2138 framework.Failf("expect error contains %q, got %q", expectedErrMsg1, err.Error()) 2139 } 2140 2141 ginkgo.By("Remove the offending key and value from the custom resource data") 2142 toCompliantFn := func(cr *unstructured.Unstructured) { 2143 if _, ok := cr.Object["data"]; !ok { 2144 cr.Object["data"] = map[string]interface{}{} 2145 } 2146 data := cr.Object["data"].(map[string]interface{}) 2147 data["webhook-e2e-test"] = "webhook-allow" 2148 } 2149 _, err = updateCustomResource(ctx, customResourceClient, f.Namespace.Name, crInstanceName, toCompliantFn) 2150 framework.ExpectNoError(err, "failed to update custom resource %s in namespace: %s", crInstanceName, f.Namespace.Name) 2151 2152 ginkgo.By("Deleting the updated custom resource should be successful") 2153 err = customResourceClient.Delete(ctx, crInstanceName, metav1.DeleteOptions{}) 2154 framework.ExpectNoError(err, "failed to delete custom resource %s in namespace: %s", crInstanceName, f.Namespace.Name) 2155 2156 } 2157 2158 func testMutatingCustomResourceWebhook(ctx context.Context, f *framework.Framework, crd *apiextensionsv1.CustomResourceDefinition, customResourceClient dynamic.ResourceInterface, prune bool) { 2159 ginkgo.By("Creating a custom resource that should be mutated by the webhook") 2160 crName := "cr-instance-1" 2161 cr := &unstructured.Unstructured{ 2162 Object: map[string]interface{}{ 2163 "kind": crd.Spec.Names.Kind, 2164 "apiVersion": crd.Spec.Group + "/" + crd.Spec.Versions[0].Name, 2165 "metadata": map[string]interface{}{ 2166 "name": crName, 2167 "namespace": f.Namespace.Name, 2168 }, 2169 "data": map[string]interface{}{ 2170 "mutation-start": "yes", 2171 }, 2172 }, 2173 } 2174 mutatedCR, err := customResourceClient.Create(ctx, cr, metav1.CreateOptions{}) 2175 framework.ExpectNoError(err, "failed to create custom resource %s in namespace: %s", crName, f.Namespace.Name) 2176 expectedCRData := map[string]interface{}{ 2177 "mutation-start": "yes", 2178 "mutation-stage-1": "yes", 2179 } 2180 if !prune { 2181 expectedCRData["mutation-stage-2"] = "yes" 2182 } 2183 if !reflect.DeepEqual(expectedCRData, mutatedCR.Object["data"]) { 2184 framework.Failf("\nexpected %#v\n, got %#v\n", expectedCRData, mutatedCR.Object["data"]) 2185 } 2186 } 2187 2188 func testMultiVersionCustomResourceWebhook(ctx context.Context, f *framework.Framework, testcrd *crd.TestCrd) { 2189 customResourceClient := testcrd.DynamicClients["v1"] 2190 ginkgo.By("Creating a custom resource while v1 is storage version") 2191 crName := "cr-instance-1" 2192 cr := &unstructured.Unstructured{ 2193 Object: map[string]interface{}{ 2194 "kind": testcrd.Crd.Spec.Names.Kind, 2195 "apiVersion": testcrd.Crd.Spec.Group + "/" + testcrd.Crd.Spec.Versions[0].Name, 2196 "metadata": map[string]interface{}{ 2197 "name": crName, 2198 "namespace": f.Namespace.Name, 2199 }, 2200 "data": map[string]interface{}{ 2201 "mutation-start": "yes", 2202 }, 2203 }, 2204 } 2205 _, err := customResourceClient.Create(ctx, cr, metav1.CreateOptions{}) 2206 framework.ExpectNoError(err, "failed to create custom resource %s in namespace: %s", crName, f.Namespace.Name) 2207 2208 ginkgo.By("Patching Custom Resource Definition to set v2 as storage") 2209 apiVersionWithV2StoragePatch := `{ 2210 "spec": { 2211 "versions": [ 2212 { 2213 "name": "v1", 2214 "storage": false, 2215 "served": true, 2216 "schema": { 2217 "openAPIV3Schema": {"x-kubernetes-preserve-unknown-fields": true, "type": "object"} 2218 } 2219 }, 2220 { 2221 "name": "v2", 2222 "storage": true, 2223 "served": true, 2224 "schema": { 2225 "openAPIV3Schema": {"x-kubernetes-preserve-unknown-fields": true, "type": "object"} 2226 } 2227 } 2228 ] 2229 } 2230 }` 2231 _, err = testcrd.APIExtensionClient.ApiextensionsV1().CustomResourceDefinitions().Patch(ctx, testcrd.Crd.Name, types.StrategicMergePatchType, []byte(apiVersionWithV2StoragePatch), metav1.PatchOptions{}) 2232 framework.ExpectNoError(err, "failed to patch custom resource definition %s in namespace: %s", testcrd.Crd.Name, f.Namespace.Name) 2233 2234 ginkgo.By("Patching the custom resource while v2 is storage version") 2235 crDummyPatch := fmt.Sprint(`[{ "op": "add", "path": "/dummy", "value": "test" }]`) 2236 mutatedCR, err := testcrd.DynamicClients["v2"].Patch(ctx, crName, types.JSONPatchType, []byte(crDummyPatch), metav1.PatchOptions{}) 2237 framework.ExpectNoError(err, "failed to patch custom resource %s in namespace: %s", crName, f.Namespace.Name) 2238 expectedCRData := map[string]interface{}{ 2239 "mutation-start": "yes", 2240 "mutation-stage-1": "yes", 2241 "mutation-stage-2": "yes", 2242 } 2243 if !reflect.DeepEqual(expectedCRData, mutatedCR.Object["data"]) { 2244 framework.Failf("\nexpected %#v\n, got %#v\n", expectedCRData, mutatedCR.Object["data"]) 2245 } 2246 if !reflect.DeepEqual("test", mutatedCR.Object["dummy"]) { 2247 framework.Failf("\nexpected %#v\n, got %#v\n", "test", mutatedCR.Object["dummy"]) 2248 } 2249 } 2250 2251 func registerValidatingWebhookForCRD(ctx context.Context, f *framework.Framework, markersNamespaceName string, configName string, certCtx *certContext, servicePort int32) { 2252 client := f.ClientSet 2253 ginkgo.By("Registering the crd webhook via the AdmissionRegistration API") 2254 2255 namespace := f.Namespace.Name 2256 sideEffectsNone := admissionregistrationv1.SideEffectClassNone 2257 2258 // This webhook will deny the creation of CustomResourceDefinitions which have the 2259 // label "webhook-e2e-test":"webhook-disallow" 2260 // NOTE: Because tests are run in parallel and in an unpredictable order, it is critical 2261 // that no other test attempts to create CRD with that label. 2262 _, err := createValidatingWebhookConfiguration(ctx, f, &admissionregistrationv1.ValidatingWebhookConfiguration{ 2263 ObjectMeta: metav1.ObjectMeta{ 2264 Name: configName, 2265 }, 2266 Webhooks: []admissionregistrationv1.ValidatingWebhook{ 2267 { 2268 Name: "deny-crd-with-unwanted-label.k8s.io", 2269 Rules: []admissionregistrationv1.RuleWithOperations{{ 2270 Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create}, 2271 Rule: admissionregistrationv1.Rule{ 2272 APIGroups: []string{"apiextensions.k8s.io"}, 2273 APIVersions: []string{"*"}, 2274 Resources: []string{"customresourcedefinitions"}, 2275 }, 2276 }}, 2277 ClientConfig: admissionregistrationv1.WebhookClientConfig{ 2278 Service: &admissionregistrationv1.ServiceReference{ 2279 Namespace: namespace, 2280 Name: serviceName, 2281 Path: strPtr("/crd"), 2282 Port: pointer.Int32(servicePort), 2283 }, 2284 CABundle: certCtx.signingCert, 2285 }, 2286 SideEffects: &sideEffectsNone, 2287 AdmissionReviewVersions: []string{"v1", "v1beta1"}, 2288 // Scope the webhook to just this test 2289 ObjectSelector: &metav1.LabelSelector{ 2290 MatchLabels: map[string]string{f.UniqueName: "true"}, 2291 }, 2292 }, 2293 // Register a webhook that can be probed by marker requests to detect when the configuration is ready. 2294 newValidatingIsReadyWebhookFixture(f, certCtx, servicePort), 2295 }, 2296 }) 2297 framework.ExpectNoError(err, "registering crd webhook config %s with namespace %s", configName, namespace) 2298 2299 err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName) 2300 framework.ExpectNoError(err, "waiting for webhook configuration to be ready") 2301 ginkgo.DeferCleanup(framework.IgnoreNotFound(client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete), configName, metav1.DeleteOptions{}) 2302 } 2303 2304 func testCRDDenyWebhook(ctx context.Context, f *framework.Framework) { 2305 ginkgo.By("Creating a custom resource definition that should be denied by the webhook") 2306 name := fmt.Sprintf("e2e-test-%s-%s-crd", f.BaseName, "deny") 2307 kind := fmt.Sprintf("E2e-test-%s-%s-crd", f.BaseName, "deny") 2308 group := fmt.Sprintf("%s.example.com", f.BaseName) 2309 apiVersions := []apiextensionsv1.CustomResourceDefinitionVersion{ 2310 { 2311 Name: "v1", 2312 Served: true, 2313 Storage: true, 2314 Schema: &apiextensionsv1.CustomResourceValidation{ 2315 OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ 2316 XPreserveUnknownFields: pointer.BoolPtr(true), 2317 Type: "object", 2318 }, 2319 }, 2320 }, 2321 } 2322 2323 // Creating a custom resource definition for use by assorted tests. 2324 config, err := framework.LoadConfig() 2325 if err != nil { 2326 framework.Failf("failed to load config: %v", err) 2327 return 2328 } 2329 apiExtensionClient, err := crdclientset.NewForConfig(config) 2330 if err != nil { 2331 framework.Failf("failed to initialize apiExtensionClient: %v", err) 2332 return 2333 } 2334 crd := &apiextensionsv1.CustomResourceDefinition{ 2335 ObjectMeta: metav1.ObjectMeta{ 2336 Name: name + "s." + group, 2337 Labels: map[string]string{ 2338 // this label ensures our object is routed to this test's webhook 2339 f.UniqueName: "true", 2340 // this is the label the webhook disallows 2341 "webhook-e2e-test": "webhook-disallow", 2342 }, 2343 }, 2344 Spec: apiextensionsv1.CustomResourceDefinitionSpec{ 2345 Group: group, 2346 Versions: apiVersions, 2347 Names: apiextensionsv1.CustomResourceDefinitionNames{ 2348 Singular: name, 2349 Kind: kind, 2350 ListKind: kind + "List", 2351 Plural: name + "s", 2352 }, 2353 Scope: apiextensionsv1.NamespaceScoped, 2354 }, 2355 } 2356 2357 // create CRD 2358 _, err = apiExtensionClient.ApiextensionsV1().CustomResourceDefinitions().Create(ctx, crd, metav1.CreateOptions{}) 2359 gomega.Expect(err).To(gomega.HaveOccurred(), "create custom resource definition %s should be denied by webhook", crd.Name) 2360 expectedErrMsg := "the crd contains unwanted label" 2361 if !strings.Contains(err.Error(), expectedErrMsg) { 2362 framework.Failf("expect error contains %q, got %q", expectedErrMsg, err.Error()) 2363 } 2364 } 2365 2366 func labelNamespace(ctx context.Context, f *framework.Framework, namespace string) { 2367 client := f.ClientSet 2368 2369 // Add a unique label to the namespace 2370 nsPatch, err := json.Marshal(map[string]interface{}{ 2371 "metadata": map[string]interface{}{ 2372 "labels": map[string]string{f.UniqueName: "true"}, 2373 }, 2374 }) 2375 framework.ExpectNoError(err, "error marshaling namespace %s", namespace) 2376 _, err = client.CoreV1().Namespaces().Patch(ctx, namespace, types.StrategicMergePatchType, nsPatch, metav1.PatchOptions{}) 2377 framework.ExpectNoError(err, "error labeling namespace %s", namespace) 2378 } 2379 2380 func registerSlowWebhook(ctx context.Context, f *framework.Framework, markersNamespaceName string, configName string, certCtx *certContext, policy *admissionregistrationv1.FailurePolicyType, timeout *int32, servicePort int32) func(ctx context.Context) { 2381 client := f.ClientSet 2382 ginkgo.By("Registering slow webhook via the AdmissionRegistration API") 2383 2384 namespace := f.Namespace.Name 2385 sideEffectsNone := admissionregistrationv1.SideEffectClassNone 2386 2387 _, err := createValidatingWebhookConfiguration(ctx, f, &admissionregistrationv1.ValidatingWebhookConfiguration{ 2388 ObjectMeta: metav1.ObjectMeta{ 2389 Name: configName, 2390 }, 2391 Webhooks: []admissionregistrationv1.ValidatingWebhook{ 2392 { 2393 Name: "allow-configmap-with-delay-webhook.k8s.io", 2394 Rules: []admissionregistrationv1.RuleWithOperations{{ 2395 Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create}, 2396 Rule: admissionregistrationv1.Rule{ 2397 APIGroups: []string{""}, 2398 APIVersions: []string{"v1"}, 2399 Resources: []string{"configmaps"}, 2400 }, 2401 }}, 2402 ClientConfig: admissionregistrationv1.WebhookClientConfig{ 2403 Service: &admissionregistrationv1.ServiceReference{ 2404 Namespace: namespace, 2405 Name: serviceName, 2406 Path: strPtr("/always-allow-delay-5s"), 2407 Port: pointer.Int32(servicePort), 2408 }, 2409 CABundle: certCtx.signingCert, 2410 }, 2411 // Scope the webhook to just this namespace 2412 NamespaceSelector: &metav1.LabelSelector{ 2413 MatchLabels: map[string]string{f.UniqueName: "true"}, 2414 }, 2415 FailurePolicy: policy, 2416 TimeoutSeconds: timeout, 2417 SideEffects: &sideEffectsNone, 2418 AdmissionReviewVersions: []string{"v1", "v1beta1"}, 2419 }, 2420 // Register a webhook that can be probed by marker requests to detect when the configuration is ready. 2421 newValidatingIsReadyWebhookFixture(f, certCtx, servicePort), 2422 }, 2423 }) 2424 framework.ExpectNoError(err, "registering slow webhook config %s with namespace %s", configName, namespace) 2425 2426 err = waitWebhookConfigurationReady(ctx, f, markersNamespaceName) 2427 framework.ExpectNoError(err, "waiting for webhook configuration to be ready") 2428 2429 cleanup := func(ctx context.Context) { 2430 err := client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(ctx, configName, metav1.DeleteOptions{}) 2431 if !apierrors.IsNotFound(err) { 2432 framework.ExpectNoError(err) 2433 } 2434 } 2435 2436 // We clean up ourselves if the caller doesn't get to it, but we also 2437 // give the caller a chance to do it in the middle of the test. 2438 ginkgo.DeferCleanup(cleanup) 2439 return cleanup 2440 } 2441 2442 func testSlowWebhookTimeoutFailEarly(ctx context.Context, f *framework.Framework) { 2443 ginkgo.By("Request fails when timeout (1s) is shorter than slow webhook latency (5s)") 2444 client := f.ClientSet 2445 name := "e2e-test-slow-webhook-configmap" 2446 _, err := client.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, &v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: name}}, metav1.CreateOptions{}) 2447 gomega.Expect(err).To(gomega.HaveOccurred(), "create configmap in namespace %s should have timed-out reaching slow webhook", f.Namespace.Name) 2448 // http timeout message: context deadline exceeded 2449 // dial timeout message: dial tcp {address}: i/o timeout 2450 isTimeoutError := strings.Contains(err.Error(), `context deadline exceeded`) || strings.Contains(err.Error(), `timeout`) 2451 isErrorQueryingWebhook := strings.Contains(err.Error(), `/always-allow-delay-5s?timeout=1s`) 2452 if !isTimeoutError || !isErrorQueryingWebhook { 2453 framework.Failf("expect an HTTP/dial timeout error querying the slow webhook, got: %q", err.Error()) 2454 } 2455 } 2456 2457 func testSlowWebhookTimeoutNoError(ctx context.Context, f *framework.Framework) { 2458 client := f.ClientSet 2459 name := "e2e-test-slow-webhook-configmap" 2460 _, err := client.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, &v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: name}}, metav1.CreateOptions{}) 2461 framework.ExpectNoError(err) 2462 err = client.CoreV1().ConfigMaps(f.Namespace.Name).Delete(ctx, name, metav1.DeleteOptions{}) 2463 framework.ExpectNoError(err) 2464 } 2465 2466 // createAdmissionWebhookMultiVersionTestCRDWithV1Storage creates a new CRD specifically 2467 // for the admission webhook calling test. 2468 func createAdmissionWebhookMultiVersionTestCRDWithV1Storage(f *framework.Framework, opts ...crd.Option) (*crd.TestCrd, error) { 2469 group := fmt.Sprintf("%s.example.com", f.BaseName) 2470 return crd.CreateMultiVersionTestCRD(f, group, append([]crd.Option{func(crd *apiextensionsv1.CustomResourceDefinition) { 2471 crd.Spec.Versions = []apiextensionsv1.CustomResourceDefinitionVersion{ 2472 { 2473 Name: "v1", 2474 Served: true, 2475 Storage: true, 2476 Schema: &apiextensionsv1.CustomResourceValidation{ 2477 OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ 2478 XPreserveUnknownFields: pointer.BoolPtr(true), 2479 Type: "object", 2480 }, 2481 }, 2482 }, 2483 { 2484 Name: "v2", 2485 Served: true, 2486 Storage: false, 2487 Schema: &apiextensionsv1.CustomResourceValidation{ 2488 OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ 2489 XPreserveUnknownFields: pointer.BoolPtr(true), 2490 Type: "object", 2491 }, 2492 }, 2493 }, 2494 } 2495 }}, opts...)...) 2496 } 2497 2498 // servedAPIVersions returns the API versions served by the CRD. 2499 func servedAPIVersions(crd *apiextensionsv1.CustomResourceDefinition) []string { 2500 ret := []string{} 2501 for _, v := range crd.Spec.Versions { 2502 if v.Served { 2503 ret = append(ret, v.Name) 2504 } 2505 } 2506 return ret 2507 } 2508 2509 // createValidatingWebhookConfiguration ensures the webhook config scopes object or namespace selection 2510 // to avoid interfering with other tests, then creates the config. 2511 func createValidatingWebhookConfiguration(ctx context.Context, f *framework.Framework, config *admissionregistrationv1.ValidatingWebhookConfiguration) (*admissionregistrationv1.ValidatingWebhookConfiguration, error) { 2512 for _, webhook := range config.Webhooks { 2513 if webhook.NamespaceSelector != nil && webhook.NamespaceSelector.MatchLabels[f.UniqueName] == "true" { 2514 continue 2515 } 2516 if webhook.ObjectSelector != nil && webhook.ObjectSelector.MatchLabels[f.UniqueName] == "true" { 2517 continue 2518 } 2519 framework.Failf(`webhook %s in config %s has no namespace or object selector with %s="true", and can interfere with other tests`, webhook.Name, config.Name, f.UniqueName) 2520 } 2521 return f.ClientSet.AdmissionregistrationV1().ValidatingWebhookConfigurations().Create(ctx, config, metav1.CreateOptions{}) 2522 } 2523 2524 // createMutatingWebhookConfiguration ensures the webhook config scopes object or namespace selection 2525 // to avoid interfering with other tests, then creates the config. 2526 func createMutatingWebhookConfiguration(ctx context.Context, f *framework.Framework, config *admissionregistrationv1.MutatingWebhookConfiguration) (*admissionregistrationv1.MutatingWebhookConfiguration, error) { 2527 for _, webhook := range config.Webhooks { 2528 if webhook.NamespaceSelector != nil && webhook.NamespaceSelector.MatchLabels[f.UniqueName] == "true" { 2529 continue 2530 } 2531 if webhook.ObjectSelector != nil && webhook.ObjectSelector.MatchLabels[f.UniqueName] == "true" { 2532 continue 2533 } 2534 framework.Failf(`webhook %s in config %s has no namespace or object selector with %s="true", and can interfere with other tests`, webhook.Name, config.Name, f.UniqueName) 2535 } 2536 return f.ClientSet.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(ctx, config, metav1.CreateOptions{}) 2537 } 2538 2539 func newDenyPodWebhookFixture(f *framework.Framework, certCtx *certContext, servicePort int32) admissionregistrationv1.ValidatingWebhook { 2540 sideEffectsNone := admissionregistrationv1.SideEffectClassNone 2541 return admissionregistrationv1.ValidatingWebhook{ 2542 Name: "deny-unwanted-pod-container-name-and-label.k8s.io", 2543 Rules: []admissionregistrationv1.RuleWithOperations{{ 2544 Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create}, 2545 Rule: admissionregistrationv1.Rule{ 2546 APIGroups: []string{""}, 2547 APIVersions: []string{"v1"}, 2548 Resources: []string{"pods"}, 2549 }, 2550 }}, 2551 ClientConfig: admissionregistrationv1.WebhookClientConfig{ 2552 Service: &admissionregistrationv1.ServiceReference{ 2553 Namespace: f.Namespace.Name, 2554 Name: serviceName, 2555 Path: strPtr("/pods"), 2556 Port: pointer.Int32(servicePort), 2557 }, 2558 CABundle: certCtx.signingCert, 2559 }, 2560 SideEffects: &sideEffectsNone, 2561 AdmissionReviewVersions: []string{"v1", "v1beta1"}, 2562 // Scope the webhook to just this namespace 2563 NamespaceSelector: &metav1.LabelSelector{ 2564 MatchLabels: map[string]string{f.UniqueName: "true"}, 2565 }, 2566 } 2567 } 2568 2569 func newDenyConfigMapWebhookFixture(f *framework.Framework, certCtx *certContext, servicePort int32) admissionregistrationv1.ValidatingWebhook { 2570 sideEffectsNone := admissionregistrationv1.SideEffectClassNone 2571 return admissionregistrationv1.ValidatingWebhook{ 2572 Name: "deny-unwanted-configmap-data.k8s.io", 2573 Rules: []admissionregistrationv1.RuleWithOperations{{ 2574 Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create, admissionregistrationv1.Update, admissionregistrationv1.Delete}, 2575 Rule: admissionregistrationv1.Rule{ 2576 APIGroups: []string{""}, 2577 APIVersions: []string{"v1"}, 2578 Resources: []string{"configmaps"}, 2579 }, 2580 }}, 2581 // The webhook skips the namespace that has label "skip-webhook-admission":"yes" 2582 NamespaceSelector: &metav1.LabelSelector{ 2583 MatchLabels: map[string]string{f.UniqueName: "true"}, 2584 MatchExpressions: []metav1.LabelSelectorRequirement{ 2585 { 2586 Key: skipNamespaceLabelKey, 2587 Operator: metav1.LabelSelectorOpNotIn, 2588 Values: []string{skipNamespaceLabelValue}, 2589 }, 2590 }, 2591 }, 2592 ClientConfig: admissionregistrationv1.WebhookClientConfig{ 2593 Service: &admissionregistrationv1.ServiceReference{ 2594 Namespace: f.Namespace.Name, 2595 Name: serviceName, 2596 Path: strPtr("/configmaps"), 2597 Port: pointer.Int32(servicePort), 2598 }, 2599 CABundle: certCtx.signingCert, 2600 }, 2601 SideEffects: &sideEffectsNone, 2602 AdmissionReviewVersions: []string{"v1", "v1beta1"}, 2603 } 2604 } 2605 2606 func newMutateConfigMapWebhookFixture(f *framework.Framework, certCtx *certContext, stage int, servicePort int32) admissionregistrationv1.MutatingWebhook { 2607 sideEffectsNone := admissionregistrationv1.SideEffectClassNone 2608 return admissionregistrationv1.MutatingWebhook{ 2609 Name: fmt.Sprintf("adding-configmap-data-stage-%d.k8s.io", stage), 2610 Rules: []admissionregistrationv1.RuleWithOperations{{ 2611 Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create}, 2612 Rule: admissionregistrationv1.Rule{ 2613 APIGroups: []string{""}, 2614 APIVersions: []string{"v1"}, 2615 Resources: []string{"configmaps"}, 2616 }, 2617 }}, 2618 ClientConfig: admissionregistrationv1.WebhookClientConfig{ 2619 Service: &admissionregistrationv1.ServiceReference{ 2620 Namespace: f.Namespace.Name, 2621 Name: serviceName, 2622 Path: strPtr("/mutating-configmaps"), 2623 Port: pointer.Int32(servicePort), 2624 }, 2625 CABundle: certCtx.signingCert, 2626 }, 2627 SideEffects: &sideEffectsNone, 2628 AdmissionReviewVersions: []string{"v1", "v1beta1"}, 2629 // Scope the webhook to just this namespace 2630 NamespaceSelector: &metav1.LabelSelector{ 2631 MatchLabels: map[string]string{f.UniqueName: "true"}, 2632 }, 2633 } 2634 } 2635 2636 // createWebhookConfigurationReadyNamespace creates a separate namespace for webhook configuration ready markers to 2637 // prevent cross-talk with webhook configurations being tested. It returns the name of the created namespace. 2638 func createWebhookConfigurationReadyNamespace(ctx context.Context, f *framework.Framework) string { 2639 baseName := f.BaseName + "-markers" 2640 // the framework will taker care of deleting the namespace 2641 ns, err := f.CreateNamespace(ctx, baseName, map[string]string{ 2642 f.UniqueName + "-markers": "true", 2643 }) 2644 framework.ExpectNoError(err, "creating namespace for webhook configuration ready markers") 2645 return ns.Name 2646 } 2647 2648 // waitWebhookConfigurationReady sends "marker" requests until a webhook configuration is ready. 2649 // A webhook created with newValidatingIsReadyWebhookFixture or newMutatingIsReadyWebhookFixture should first be added to 2650 // the webhook configuration. 2651 func waitWebhookConfigurationReady(ctx context.Context, f *framework.Framework, markersNamespaceName string) error { 2652 cmClient := f.ClientSet.CoreV1().ConfigMaps(markersNamespaceName) 2653 return wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (bool, error) { 2654 marker := &v1.ConfigMap{ 2655 ObjectMeta: metav1.ObjectMeta{ 2656 Name: string(uuid.NewUUID()), 2657 Labels: map[string]string{ 2658 f.UniqueName: "true", 2659 }, 2660 }, 2661 } 2662 _, err := cmClient.Create(ctx, marker, metav1.CreateOptions{}) 2663 if err != nil { 2664 // The always-deny webhook does not provide a reason, so check for the error string we expect 2665 if strings.Contains(err.Error(), "denied") { 2666 return true, nil 2667 } 2668 return false, err 2669 } 2670 // best effort cleanup of markers that are no longer needed 2671 _ = cmClient.Delete(ctx, marker.GetName(), metav1.DeleteOptions{}) 2672 framework.Logf("Waiting for webhook configuration to be ready...") 2673 return false, nil 2674 }) 2675 } 2676 2677 // newValidatingIsReadyWebhookFixture creates a validating webhook that can be added to a webhook configuration and then probed 2678 // with "marker" requests via waitWebhookConfigurationReady to wait for a webhook configuration to be ready. 2679 func newValidatingIsReadyWebhookFixture(f *framework.Framework, certCtx *certContext, servicePort int32) admissionregistrationv1.ValidatingWebhook { 2680 sideEffectsNone := admissionregistrationv1.SideEffectClassNone 2681 failOpen := admissionregistrationv1.Ignore 2682 return admissionregistrationv1.ValidatingWebhook{ 2683 Name: "validating-is-webhook-configuration-ready.k8s.io", 2684 Rules: []admissionregistrationv1.RuleWithOperations{{ 2685 Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create}, 2686 Rule: admissionregistrationv1.Rule{ 2687 APIGroups: []string{""}, 2688 APIVersions: []string{"v1"}, 2689 Resources: []string{"configmaps"}, 2690 }, 2691 }}, 2692 ClientConfig: admissionregistrationv1.WebhookClientConfig{ 2693 Service: &admissionregistrationv1.ServiceReference{ 2694 Namespace: f.Namespace.Name, 2695 Name: serviceName, 2696 Path: strPtr("/always-deny"), 2697 Port: pointer.Int32(servicePort), 2698 }, 2699 CABundle: certCtx.signingCert, 2700 }, 2701 // network failures while the service network routing is being set up should be ignored by the marker 2702 FailurePolicy: &failOpen, 2703 SideEffects: &sideEffectsNone, 2704 AdmissionReviewVersions: []string{"v1", "v1beta1"}, 2705 // Scope the webhook to just the markers namespace 2706 NamespaceSelector: &metav1.LabelSelector{ 2707 MatchLabels: map[string]string{f.UniqueName + "-markers": "true"}, 2708 }, 2709 // appease createValidatingWebhookConfiguration isolation requirements 2710 ObjectSelector: &metav1.LabelSelector{ 2711 MatchLabels: map[string]string{f.UniqueName: "true"}, 2712 }, 2713 } 2714 } 2715 2716 // newMutatingIsReadyWebhookFixture creates a mutating webhook that can be added to a webhook configuration and then probed 2717 // with "marker" requests via waitWebhookConfigurationReady to wait for a webhook configuration to be ready. 2718 func newMutatingIsReadyWebhookFixture(f *framework.Framework, certCtx *certContext, servicePort int32) admissionregistrationv1.MutatingWebhook { 2719 sideEffectsNone := admissionregistrationv1.SideEffectClassNone 2720 failOpen := admissionregistrationv1.Ignore 2721 return admissionregistrationv1.MutatingWebhook{ 2722 Name: "mutating-is-webhook-configuration-ready.k8s.io", 2723 Rules: []admissionregistrationv1.RuleWithOperations{{ 2724 Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create}, 2725 Rule: admissionregistrationv1.Rule{ 2726 APIGroups: []string{""}, 2727 APIVersions: []string{"v1"}, 2728 Resources: []string{"configmaps"}, 2729 }, 2730 }}, 2731 ClientConfig: admissionregistrationv1.WebhookClientConfig{ 2732 Service: &admissionregistrationv1.ServiceReference{ 2733 Namespace: f.Namespace.Name, 2734 Name: serviceName, 2735 Path: strPtr("/always-deny"), 2736 Port: pointer.Int32(servicePort), 2737 }, 2738 CABundle: certCtx.signingCert, 2739 }, 2740 // network failures while the service network routing is being set up should be ignored by the marker 2741 FailurePolicy: &failOpen, 2742 SideEffects: &sideEffectsNone, 2743 AdmissionReviewVersions: []string{"v1", "v1beta1"}, 2744 // Scope the webhook to just the markers namespace 2745 NamespaceSelector: &metav1.LabelSelector{ 2746 MatchLabels: map[string]string{f.UniqueName + "-markers": "true"}, 2747 }, 2748 // appease createMutatingWebhookConfiguration isolation requirements 2749 ObjectSelector: &metav1.LabelSelector{ 2750 MatchLabels: map[string]string{f.UniqueName: "true"}, 2751 }, 2752 } 2753 }