k8s.io/kubernetes@v1.29.3/test/e2e/apimachinery/custom_resource_definition.go (about) 1 /* 2 Copyright 2016 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package apimachinery 18 19 import ( 20 "context" 21 "fmt" 22 "time" 23 24 "github.com/google/go-cmp/cmp" 25 "github.com/onsi/ginkgo/v2" 26 "github.com/onsi/gomega" 27 28 v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" 29 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" 30 "k8s.io/apiextensions-apiserver/test/integration/fixtures" 31 "k8s.io/apimachinery/pkg/api/equality" 32 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 33 "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 34 "k8s.io/apimachinery/pkg/runtime" 35 "k8s.io/apimachinery/pkg/runtime/schema" 36 "k8s.io/apimachinery/pkg/types" 37 "k8s.io/apimachinery/pkg/util/uuid" 38 "k8s.io/apimachinery/pkg/util/wait" 39 "k8s.io/apiserver/pkg/storage/names" 40 "k8s.io/client-go/dynamic" 41 "k8s.io/client-go/util/retry" 42 "k8s.io/kubernetes/test/e2e/framework" 43 admissionapi "k8s.io/pod-security-admission/api" 44 ) 45 46 var _ = SIGDescribe("CustomResourceDefinition resources [Privileged:ClusterAdmin]", func() { 47 48 f := framework.NewDefaultFramework("custom-resource-definition") 49 f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged 50 51 ginkgo.Context("Simple CustomResourceDefinition", func() { 52 /* 53 Release: v1.9 54 Testname: Custom Resource Definition, create 55 Description: Create a API extension client and define a random custom resource definition. 56 Create the custom resource definition and then delete it. The creation and deletion MUST 57 be successful. 58 */ 59 framework.ConformanceIt("creating/deleting custom resource definition objects works", func(ctx context.Context) { 60 61 config, err := framework.LoadConfig() 62 framework.ExpectNoError(err, "loading config") 63 apiExtensionClient, err := clientset.NewForConfig(config) 64 framework.ExpectNoError(err, "initializing apiExtensionClient") 65 66 randomDefinition := fixtures.NewRandomNameV1CustomResourceDefinition(v1.ClusterScoped) 67 68 // Create CRD and waits for the resource to be recognized and available. 69 randomDefinition, err = fixtures.CreateNewV1CustomResourceDefinitionWatchUnsafe(randomDefinition, apiExtensionClient) 70 framework.ExpectNoError(err, "creating CustomResourceDefinition") 71 72 defer func() { 73 err = fixtures.DeleteV1CustomResourceDefinition(randomDefinition, apiExtensionClient) 74 framework.ExpectNoError(err, "deleting CustomResourceDefinition") 75 }() 76 }) 77 78 /* 79 Release: v1.16 80 Testname: Custom Resource Definition, list 81 Description: Create a API extension client, define 10 labeled custom resource definitions and list them using 82 a label selector; the list result MUST contain only the labeled custom resource definitions. Delete the labeled 83 custom resource definitions via delete collection; the delete MUST be successful and MUST delete only the 84 labeled custom resource definitions. 85 */ 86 framework.ConformanceIt("listing custom resource definition objects works", func(ctx context.Context) { 87 testListSize := 10 88 config, err := framework.LoadConfig() 89 framework.ExpectNoError(err, "loading config") 90 apiExtensionClient, err := clientset.NewForConfig(config) 91 framework.ExpectNoError(err, "initializing apiExtensionClient") 92 93 // Label the CRDs we create so we can list only them even though they are cluster scoped 94 testUUID := string(uuid.NewUUID()) 95 96 // Create CRD and wait for the resource to be recognized and available. 97 crds := make([]*v1.CustomResourceDefinition, testListSize) 98 for i := 0; i < testListSize; i++ { 99 crd := fixtures.NewRandomNameV1CustomResourceDefinition(v1.ClusterScoped) 100 crd.Labels = map[string]string{"e2e-list-test-uuid": testUUID} 101 crd, err = fixtures.CreateNewV1CustomResourceDefinitionWatchUnsafe(crd, apiExtensionClient) 102 framework.ExpectNoError(err, "creating CustomResourceDefinition") 103 crds[i] = crd 104 } 105 106 // Create a crd w/o the label to ensure the label selector matching works correctly 107 crd := fixtures.NewRandomNameV1CustomResourceDefinition(v1.ClusterScoped) 108 crd, err = fixtures.CreateNewV1CustomResourceDefinitionWatchUnsafe(crd, apiExtensionClient) 109 framework.ExpectNoError(err, "creating CustomResourceDefinition") 110 defer func() { 111 err = fixtures.DeleteV1CustomResourceDefinition(crd, apiExtensionClient) 112 framework.ExpectNoError(err, "deleting CustomResourceDefinition") 113 }() 114 115 selectorListOpts := metav1.ListOptions{LabelSelector: "e2e-list-test-uuid=" + testUUID} 116 list, err := apiExtensionClient.ApiextensionsV1().CustomResourceDefinitions().List(ctx, selectorListOpts) 117 framework.ExpectNoError(err, "listing CustomResourceDefinitions") 118 gomega.Expect(list.Items).To(gomega.HaveLen(testListSize)) 119 for _, actual := range list.Items { 120 var expected *v1.CustomResourceDefinition 121 for _, e := range crds { 122 if e.Name == actual.Name && e.Namespace == actual.Namespace { 123 expected = e 124 } 125 } 126 framework.ExpectNotEqual(expected, nil) 127 if !equality.Semantic.DeepEqual(actual.Spec, expected.Spec) { 128 framework.Failf("Expected CustomResourceDefinition in list with name %s to match crd created with same name, but got different specs:\n%s", 129 actual.Name, cmp.Diff(expected.Spec, actual.Spec)) 130 } 131 } 132 133 // Use delete collection to remove the CRDs 134 err = fixtures.DeleteV1CustomResourceDefinitions(selectorListOpts, apiExtensionClient) 135 framework.ExpectNoError(err, "deleting CustomResourceDefinitions") 136 _, err = apiExtensionClient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, crd.Name, metav1.GetOptions{}) 137 framework.ExpectNoError(err, "getting remaining CustomResourceDefinition") 138 }) 139 140 /* 141 Release: v1.16 142 Testname: Custom Resource Definition, status sub-resource 143 Description: Create a custom resource definition. Attempt to read, update and patch its status sub-resource; 144 all mutating sub-resource operations MUST be visible to subsequent reads. 145 */ 146 framework.ConformanceIt("getting/updating/patching custom resource definition status sub-resource works", func(ctx context.Context) { 147 config, err := framework.LoadConfig() 148 framework.ExpectNoError(err, "loading config") 149 apiExtensionClient, err := clientset.NewForConfig(config) 150 framework.ExpectNoError(err, "initializing apiExtensionClient") 151 dynamicClient, err := dynamic.NewForConfig(config) 152 framework.ExpectNoError(err, "initializing dynamic client") 153 gvr := v1.SchemeGroupVersion.WithResource("customresourcedefinitions") 154 resourceClient := dynamicClient.Resource(gvr) 155 156 // Create CRD and waits for the resource to be recognized and available. 157 crd := fixtures.NewRandomNameV1CustomResourceDefinition(v1.ClusterScoped) 158 crd, err = fixtures.CreateNewV1CustomResourceDefinitionWatchUnsafe(crd, apiExtensionClient) 159 framework.ExpectNoError(err, "creating CustomResourceDefinition") 160 defer func() { 161 err = fixtures.DeleteV1CustomResourceDefinition(crd, apiExtensionClient) 162 framework.ExpectNoError(err, "deleting CustomResourceDefinition") 163 }() 164 165 var updated *v1.CustomResourceDefinition 166 updateCondition := v1.CustomResourceDefinitionCondition{Message: "updated"} 167 err = retry.RetryOnConflict(retry.DefaultRetry, func() error { 168 // Use dynamic client to read the status sub-resource since typed client does not expose it. 169 u, err := resourceClient.Get(ctx, crd.GetName(), metav1.GetOptions{}, "status") 170 framework.ExpectNoError(err, "getting CustomResourceDefinition status") 171 status := unstructuredToCRD(u) 172 if !equality.Semantic.DeepEqual(status.Spec, crd.Spec) { 173 framework.Failf("Expected CustomResourceDefinition Spec to match status sub-resource Spec, but got:\n%s", cmp.Diff(status.Spec, crd.Spec)) 174 } 175 status.Status.Conditions = append(status.Status.Conditions, updateCondition) 176 updated, err = apiExtensionClient.ApiextensionsV1().CustomResourceDefinitions().UpdateStatus(ctx, status, metav1.UpdateOptions{}) 177 return err 178 }) 179 framework.ExpectNoError(err, "updating CustomResourceDefinition status") 180 expectCondition(updated.Status.Conditions, updateCondition) 181 182 patchCondition := v1.CustomResourceDefinitionCondition{Message: "patched"} 183 patched, err := apiExtensionClient.ApiextensionsV1().CustomResourceDefinitions().Patch(ctx, crd.GetName(), 184 types.JSONPatchType, 185 []byte(`[{"op": "add", "path": "/status/conditions", "value": [{"message": "patched"}]}]`), metav1.PatchOptions{}, 186 "status") 187 framework.ExpectNoError(err, "patching CustomResourceDefinition status") 188 expectCondition(updated.Status.Conditions, updateCondition) 189 expectCondition(patched.Status.Conditions, patchCondition) 190 }) 191 }) 192 193 /* 194 Release: v1.16 195 Testname: Custom Resource Definition, discovery 196 Description: Fetch /apis, /apis/apiextensions.k8s.io, and /apis/apiextensions.k8s.io/v1 discovery documents, 197 and ensure they indicate CustomResourceDefinition apiextensions.k8s.io/v1 resources are available. 198 */ 199 framework.ConformanceIt("should include custom resource definition resources in discovery documents", func(ctx context.Context) { 200 { 201 ginkgo.By("fetching the /apis discovery document") 202 apiGroupList := &metav1.APIGroupList{} 203 err := f.ClientSet.Discovery().RESTClient().Get().AbsPath("/apis").Do(ctx).Into(apiGroupList) 204 framework.ExpectNoError(err, "fetching /apis") 205 206 ginkgo.By("finding the apiextensions.k8s.io API group in the /apis discovery document") 207 var group *metav1.APIGroup 208 for _, g := range apiGroupList.Groups { 209 if g.Name == v1.GroupName { 210 group = &g 211 break 212 } 213 } 214 framework.ExpectNotEqual(group, nil, "apiextensions.k8s.io API group not found in /apis discovery document") 215 216 ginkgo.By("finding the apiextensions.k8s.io/v1 API group/version in the /apis discovery document") 217 var version *metav1.GroupVersionForDiscovery 218 for _, v := range group.Versions { 219 if v.Version == v1.SchemeGroupVersion.Version { 220 version = &v 221 break 222 } 223 } 224 framework.ExpectNotEqual(version, nil, "apiextensions.k8s.io/v1 API group version not found in /apis discovery document") 225 } 226 227 { 228 ginkgo.By("fetching the /apis/apiextensions.k8s.io discovery document") 229 group := &metav1.APIGroup{} 230 err := f.ClientSet.Discovery().RESTClient().Get().AbsPath("/apis/apiextensions.k8s.io").Do(ctx).Into(group) 231 framework.ExpectNoError(err, "fetching /apis/apiextensions.k8s.io") 232 gomega.Expect(group.Name).To(gomega.Equal(v1.GroupName), "verifying API group name in /apis/apiextensions.k8s.io discovery document") 233 234 ginkgo.By("finding the apiextensions.k8s.io/v1 API group/version in the /apis/apiextensions.k8s.io discovery document") 235 var version *metav1.GroupVersionForDiscovery 236 for _, v := range group.Versions { 237 if v.Version == v1.SchemeGroupVersion.Version { 238 version = &v 239 break 240 } 241 } 242 framework.ExpectNotEqual(version, nil, "apiextensions.k8s.io/v1 API group version not found in /apis/apiextensions.k8s.io discovery document") 243 } 244 245 { 246 ginkgo.By("fetching the /apis/apiextensions.k8s.io/v1 discovery document") 247 apiResourceList := &metav1.APIResourceList{} 248 err := f.ClientSet.Discovery().RESTClient().Get().AbsPath("/apis/apiextensions.k8s.io/v1").Do(ctx).Into(apiResourceList) 249 framework.ExpectNoError(err, "fetching /apis/apiextensions.k8s.io/v1") 250 gomega.Expect(apiResourceList.GroupVersion).To(gomega.Equal(v1.SchemeGroupVersion.String()), "verifying API group/version in /apis/apiextensions.k8s.io/v1 discovery document") 251 252 ginkgo.By("finding customresourcedefinitions resources in the /apis/apiextensions.k8s.io/v1 discovery document") 253 var crdResource *metav1.APIResource 254 for i := range apiResourceList.APIResources { 255 if apiResourceList.APIResources[i].Name == "customresourcedefinitions" { 256 crdResource = &apiResourceList.APIResources[i] 257 } 258 } 259 framework.ExpectNotEqual(crdResource, nil, "customresourcedefinitions resource not found in /apis/apiextensions.k8s.io/v1 discovery document") 260 } 261 }) 262 263 /* 264 Release: v1.17 265 Testname: Custom Resource Definition, defaulting 266 Description: Create a custom resource definition without default. Create CR. Add default and read CR until 267 the default is applied. Create another CR. Remove default, add default for another field and read CR until 268 new field is defaulted, but old default stays. 269 */ 270 framework.ConformanceIt("custom resource defaulting for requests and from storage works", func(ctx context.Context) { 271 config, err := framework.LoadConfig() 272 framework.ExpectNoError(err, "loading config") 273 apiExtensionClient, err := clientset.NewForConfig(config) 274 framework.ExpectNoError(err, "initializing apiExtensionClient") 275 dynamicClient, err := dynamic.NewForConfig(config) 276 framework.ExpectNoError(err, "initializing dynamic client") 277 278 // Create CRD without default and waits for the resource to be recognized and available. 279 crd := fixtures.NewRandomNameV1CustomResourceDefinition(v1.ClusterScoped) 280 if crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties == nil { 281 crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties = map[string]v1.JSONSchemaProps{} 282 } 283 crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["a"] = v1.JSONSchemaProps{Type: "string"} 284 crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["b"] = v1.JSONSchemaProps{Type: "string"} 285 crd, err = fixtures.CreateNewV1CustomResourceDefinitionWatchUnsafe(crd, apiExtensionClient) 286 framework.ExpectNoError(err, "creating CustomResourceDefinition") 287 defer func() { 288 err = fixtures.DeleteV1CustomResourceDefinition(crd, apiExtensionClient) 289 framework.ExpectNoError(err, "deleting CustomResourceDefinition") 290 }() 291 292 // create CR without default in storage 293 name1 := names.SimpleNameGenerator.GenerateName("cr-1") 294 gvr := schema.GroupVersionResource{ 295 Group: crd.Spec.Group, 296 Version: crd.Spec.Versions[0].Name, 297 Resource: crd.Spec.Names.Plural, 298 } 299 crClient := dynamicClient.Resource(gvr) 300 _, err = crClient.Create(ctx, &unstructured.Unstructured{Object: map[string]interface{}{ 301 "apiVersion": gvr.Group + "/" + gvr.Version, 302 "kind": crd.Spec.Names.Kind, 303 "metadata": map[string]interface{}{ 304 "name": name1, 305 }, 306 }}, metav1.CreateOptions{}) 307 framework.ExpectNoError(err, "creating CR") 308 309 // Setting default for a to "A" and waiting for the CR to get defaulted on read 310 crd, err = apiExtensionClient.ApiextensionsV1().CustomResourceDefinitions().Patch(ctx, crd.Name, types.JSONPatchType, []byte(`[ 311 {"op":"add","path":"/spec/versions/0/schema/openAPIV3Schema/properties/a/default", "value": "A"} 312 ]`), metav1.PatchOptions{}) 313 framework.ExpectNoError(err, "setting default for a to \"A\" in schema") 314 315 err = wait.PollImmediate(time.Millisecond*100, wait.ForeverTestTimeout, func() (bool, error) { 316 u1, err := crClient.Get(ctx, name1, metav1.GetOptions{}) 317 if err != nil { 318 return false, err 319 } 320 a, found, err := unstructured.NestedFieldNoCopy(u1.Object, "a") 321 if err != nil { 322 return false, err 323 } 324 if !found { 325 return false, nil 326 } 327 if a != "A" { 328 return false, fmt.Errorf("expected a:\"A\", but got a:%q", a) 329 } 330 return true, nil 331 }) 332 framework.ExpectNoError(err, "waiting for CR to be defaulted on read") 333 334 // create CR with default in storage 335 name2 := names.SimpleNameGenerator.GenerateName("cr-2") 336 u2, err := crClient.Create(ctx, &unstructured.Unstructured{Object: map[string]interface{}{ 337 "apiVersion": gvr.Group + "/" + gvr.Version, 338 "kind": crd.Spec.Names.Kind, 339 "metadata": map[string]interface{}{ 340 "name": name2, 341 }, 342 }}, metav1.CreateOptions{}) 343 framework.ExpectNoError(err, "creating CR") 344 v, found, err := unstructured.NestedFieldNoCopy(u2.Object, "a") 345 if !found { 346 framework.Failf("field `a` should have been defaulted in %+v", u2.Object) 347 } 348 gomega.Expect(v).To(gomega.Equal("A"), "\"a\" is defaulted to \"A\"") 349 350 // Deleting default for a, adding default "B" for b and waiting for the CR to get defaulted on read for b 351 crd, err = apiExtensionClient.ApiextensionsV1().CustomResourceDefinitions().Patch(ctx, crd.Name, types.JSONPatchType, []byte(`[ 352 {"op":"remove","path":"/spec/versions/0/schema/openAPIV3Schema/properties/a/default"}, 353 {"op":"add","path":"/spec/versions/0/schema/openAPIV3Schema/properties/b/default", "value": "B"} 354 ]`), metav1.PatchOptions{}) 355 framework.ExpectNoError(err, "setting default for b to \"B\" and remove default for a") 356 357 err = wait.PollImmediate(time.Millisecond*100, wait.ForeverTestTimeout, func() (bool, error) { 358 u2, err := crClient.Get(ctx, name2, metav1.GetOptions{}) 359 if err != nil { 360 return false, err 361 } 362 b, found, err := unstructured.NestedFieldNoCopy(u2.Object, "b") 363 if err != nil { 364 return false, err 365 } 366 if !found { 367 return false, nil 368 } 369 if b != "B" { 370 return false, fmt.Errorf("expected b:\"B\", but got b:%q", b) 371 } 372 a, found, err := unstructured.NestedFieldNoCopy(u2.Object, "a") 373 if err != nil { 374 return false, err 375 } 376 if !found { 377 return false, fmt.Errorf("expected a:\"A\" to be unchanged, but it was removed") 378 } 379 if a != "A" { 380 return false, fmt.Errorf("expected a:\"A\" to be unchanged, but it changed to %q", a) 381 } 382 return true, nil 383 }) 384 framework.ExpectNoError(err, "waiting for CR to be defaulted on read for b and a staying the same") 385 }) 386 387 }) 388 389 func unstructuredToCRD(obj *unstructured.Unstructured) *v1.CustomResourceDefinition { 390 crd := new(v1.CustomResourceDefinition) 391 err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, crd) 392 framework.ExpectNoError(err, "converting unstructured to CustomResourceDefinition") 393 return crd 394 } 395 396 func expectCondition(conditions []v1.CustomResourceDefinitionCondition, expected v1.CustomResourceDefinitionCondition) { 397 for _, c := range conditions { 398 if equality.Semantic.DeepEqual(c, expected) { 399 return 400 } 401 } 402 framework.Failf("Condition %#v not found in conditions %#v", expected, conditions) 403 }