k8s.io/kubernetes@v1.29.3/test/integration/controlplane/crd_test.go (about) 1 /* 2 Copyright 2017 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package controlplane 18 19 import ( 20 "context" 21 "encoding/json" 22 "fmt" 23 "strings" 24 "testing" 25 "time" 26 27 "k8s.io/apiextensions-apiserver/test/integration/fixtures" 28 29 apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" 30 31 v1 "k8s.io/api/core/v1" 32 networkingv1 "k8s.io/api/networking/v1" 33 apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" 34 apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" 35 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 36 "k8s.io/apimachinery/pkg/runtime/schema" 37 "k8s.io/apimachinery/pkg/util/wait" 38 "k8s.io/client-go/dynamic" 39 "k8s.io/client-go/kubernetes" 40 "k8s.io/kube-openapi/pkg/validation/spec" 41 kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" 42 "k8s.io/kubernetes/test/integration/etcd" 43 "k8s.io/kubernetes/test/integration/framework" 44 ) 45 46 func TestCRDShadowGroup(t *testing.T) { 47 result := kubeapiservertesting.StartTestServerOrDie(t, nil, nil, framework.SharedEtcd()) 48 defer result.TearDownFn() 49 50 testNamespace := "test-crd-shadow-group" 51 kubeclient, err := kubernetes.NewForConfig(result.ClientConfig) 52 if err != nil { 53 t.Fatalf("Unexpected error: %v", err) 54 } 55 if _, err := kubeclient.CoreV1().Namespaces().Create(context.TODO(), (&v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: testNamespace}}), metav1.CreateOptions{}); err != nil { 56 t.Fatal(err) 57 } 58 59 apiextensionsclient, err := apiextensionsclientset.NewForConfig(result.ClientConfig) 60 if err != nil { 61 t.Fatalf("Unexpected error: %v", err) 62 } 63 64 t.Logf("Creating a NetworkPolicy") 65 nwPolicy, err := kubeclient.NetworkingV1().NetworkPolicies(testNamespace).Create(context.TODO(), &networkingv1.NetworkPolicy{ 66 ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: testNamespace}, 67 Spec: networkingv1.NetworkPolicySpec{ 68 PodSelector: metav1.LabelSelector{MatchLabels: map[string]string{"foo": "bar"}}, 69 Ingress: []networkingv1.NetworkPolicyIngressRule{}, 70 }, 71 }, metav1.CreateOptions{}) 72 if err != nil { 73 t.Fatalf("Failed to create NetworkPolicy: %v", err) 74 } 75 76 t.Logf("Trying to shadow networking group") 77 crd := &apiextensionsv1.CustomResourceDefinition{ 78 ObjectMeta: metav1.ObjectMeta{ 79 Name: "foos." + networkingv1.GroupName, 80 Annotations: map[string]string{"api-approved.kubernetes.io": "unapproved, test-only"}, 81 }, 82 Spec: apiextensionsv1.CustomResourceDefinitionSpec{ 83 Group: networkingv1.GroupName, 84 Scope: apiextensionsv1.ClusterScoped, 85 Names: apiextensionsv1.CustomResourceDefinitionNames{ 86 Plural: "foos", 87 Kind: "Foo", 88 }, 89 Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ 90 { 91 Name: networkingv1.SchemeGroupVersion.Version, 92 Served: true, 93 Storage: true, 94 Schema: fixtures.AllowAllSchema(), 95 }, 96 }, 97 }, 98 } 99 etcd.CreateTestCRDs(t, apiextensionsclient, true, crd) 100 101 // wait to give aggregator time to update 102 time.Sleep(2 * time.Second) 103 104 t.Logf("Checking that we still see the NetworkPolicy") 105 _, err = kubeclient.NetworkingV1().NetworkPolicies(nwPolicy.Namespace).Get(context.TODO(), nwPolicy.Name, metav1.GetOptions{}) 106 if err != nil { 107 t.Errorf("Failed to get NetworkPolocy: %v", err) 108 } 109 110 t.Logf("Checking that crd resource does not show up in networking group") 111 if etcd.CrdExistsInDiscovery(apiextensionsclient, crd) { 112 t.Errorf("CRD resource shows up in discovery, but shouldn't.") 113 } 114 } 115 116 func TestCRD(t *testing.T) { 117 result := kubeapiservertesting.StartTestServerOrDie(t, nil, nil, framework.SharedEtcd()) 118 defer result.TearDownFn() 119 120 testNamespace := "test-crd" 121 kubeclient, err := kubernetes.NewForConfig(result.ClientConfig) 122 if err != nil { 123 t.Fatalf("Unexpected error: %v", err) 124 } 125 if _, err := kubeclient.CoreV1().Namespaces().Create(context.TODO(), (&v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: testNamespace}}), metav1.CreateOptions{}); err != nil { 126 t.Fatal(err) 127 } 128 129 apiextensionsclient, err := apiextensionsclientset.NewForConfig(result.ClientConfig) 130 if err != nil { 131 t.Fatalf("Unexpected error: %v", err) 132 } 133 134 t.Logf("Trying to create a custom resource without conflict") 135 crd := &apiextensionsv1.CustomResourceDefinition{ 136 ObjectMeta: metav1.ObjectMeta{ 137 Name: "foos.cr.bar.com", 138 }, 139 Spec: apiextensionsv1.CustomResourceDefinitionSpec{ 140 Group: "cr.bar.com", 141 Scope: apiextensionsv1.NamespaceScoped, 142 Names: apiextensionsv1.CustomResourceDefinitionNames{ 143 Plural: "foos", 144 Kind: "Foo", 145 }, 146 Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ 147 { 148 Name: networkingv1.SchemeGroupVersion.Version, 149 Served: true, 150 Storage: true, 151 Schema: fixtures.AllowAllSchema(), 152 }, 153 }, 154 }, 155 } 156 etcd.CreateTestCRDs(t, apiextensionsclient, false, crd) 157 158 t.Logf("Trying to access foos.cr.bar.com with dynamic client") 159 dynamicClient, err := dynamic.NewForConfig(result.ClientConfig) 160 if err != nil { 161 t.Fatalf("Unexpected error: %v", err) 162 } 163 fooResource := schema.GroupVersionResource{Group: "cr.bar.com", Version: "v1", Resource: "foos"} 164 _, err = dynamicClient.Resource(fooResource).Namespace(testNamespace).List(context.TODO(), metav1.ListOptions{}) 165 if err != nil { 166 t.Errorf("Failed to list foos.cr.bar.com instances: %v", err) 167 } 168 } 169 170 func TestCRDOpenAPI(t *testing.T) { 171 result := kubeapiservertesting.StartTestServerOrDie(t, nil, nil, framework.SharedEtcd()) 172 defer result.TearDownFn() 173 kubeclient, err := kubernetes.NewForConfig(result.ClientConfig) 174 if err != nil { 175 t.Fatalf("Unexpected error: %v", err) 176 } 177 apiextensionsclient, err := apiextensionsclientset.NewForConfig(result.ClientConfig) 178 if err != nil { 179 t.Fatalf("Unexpected error: %v", err) 180 } 181 dynamicClient, err := dynamic.NewForConfig(result.ClientConfig) 182 if err != nil { 183 t.Fatalf("Unexpected error: %v", err) 184 } 185 186 t.Logf("Trying to create a CustomResourceDefinitions") 187 nonStructuralBetaCRD := &apiextensionsv1beta1.CustomResourceDefinition{ 188 ObjectMeta: metav1.ObjectMeta{ 189 Name: "foos.nonstructural.cr.bar.com", 190 }, 191 Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ 192 Group: "nonstructural.cr.bar.com", 193 Version: "v1", 194 Scope: apiextensionsv1beta1.NamespaceScoped, 195 Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ 196 Plural: "foos", 197 Kind: "Foo", 198 }, 199 Validation: &apiextensionsv1beta1.CustomResourceValidation{ 200 OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{ 201 Type: "object", 202 Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ 203 "foo": {}, 204 }, 205 }, 206 }, 207 }, 208 } 209 structuralCRD := &apiextensionsv1.CustomResourceDefinition{ 210 ObjectMeta: metav1.ObjectMeta{ 211 Name: "foos.structural.cr.bar.com", 212 }, 213 Spec: apiextensionsv1.CustomResourceDefinitionSpec{ 214 Group: "structural.cr.bar.com", 215 Scope: apiextensionsv1.NamespaceScoped, 216 Names: apiextensionsv1.CustomResourceDefinitionNames{ 217 Plural: "foos", 218 Kind: "Foo", 219 }, 220 Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ 221 { 222 Name: "v1", 223 Served: true, 224 Storage: true, 225 Schema: &apiextensionsv1.CustomResourceValidation{ 226 OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ 227 Type: "object", 228 Properties: map[string]apiextensionsv1.JSONSchemaProps{ 229 "foo": {Type: "string"}, 230 }, 231 }, 232 }, 233 }, 234 }, 235 }, 236 } 237 nonStructuralCRD, err := fixtures.CreateCRDUsingRemovedAPI(result.EtcdClient, result.EtcdStoragePrefix, nonStructuralBetaCRD, apiextensionsclient, dynamicClient) 238 if err != nil { 239 t.Fatal(err) 240 } 241 etcd.CreateTestCRDs(t, apiextensionsclient, false, structuralCRD) 242 243 getPublishedSchema := func(defName string) (*spec.Schema, error) { 244 bs, err := kubeclient.RESTClient().Get().AbsPath("openapi", "v2").DoRaw(context.TODO()) 245 if err != nil { 246 return nil, err 247 } 248 spec := spec.Swagger{} 249 if err := json.Unmarshal(bs, &spec); err != nil { 250 return nil, err 251 } 252 if spec.SwaggerProps.Paths == nil { 253 return nil, nil 254 } 255 d, ok := spec.SwaggerProps.Definitions[defName] 256 if !ok { 257 return nil, nil 258 } 259 return &d, nil 260 } 261 262 waitForSpec := func(crd *apiextensionsv1.CustomResourceDefinition, expectedType string) { 263 t.Logf(`Waiting for {properties: {"foo": {"type":"%s"}}} to show up in schema`, expectedType) 264 lastMsg := "" 265 if err := wait.PollImmediate(500*time.Millisecond, 10*time.Second, func() (bool, error) { 266 lastMsg = "" 267 defName := crdDefinitionName(crd) 268 schema, err := getPublishedSchema(defName) 269 if err != nil { 270 lastMsg = err.Error() 271 return false, nil 272 } 273 if schema == nil { 274 lastMsg = fmt.Sprintf("spec.SwaggerProps.Definitions[%q] not found", defName) 275 return false, nil 276 } 277 p, ok := schema.Properties["foo"] 278 if !ok { 279 lastMsg = fmt.Sprintf(`spec.SwaggerProps.Definitions[%q].Properties["foo"] not found`, defName) 280 return false, nil 281 } 282 if !p.Type.Contains(expectedType) { 283 lastMsg = fmt.Sprintf(`spec.SwaggerProps.Definitions[%q].Properties["foo"].Type should be %q, but got: %q`, defName, expectedType, p.Type) 284 return false, nil 285 } 286 return true, nil 287 }); err != nil { 288 t.Fatalf("Failed to see %s OpenAPI spec in discovery: %v, last message: %s", structuralCRD.Name, err, lastMsg) 289 } 290 } 291 292 t.Logf("Check that structural schema is published") 293 waitForSpec(structuralCRD, "string") 294 structuralCRD, err = apiextensionsclient.ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), structuralCRD.Name, metav1.GetOptions{}) 295 if err != nil { 296 t.Fatal(err) 297 } 298 prop := structuralCRD.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["foo"] 299 prop.Type = "boolean" 300 structuralCRD.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["foo"] = prop 301 if _, err = apiextensionsclient.ApiextensionsV1().CustomResourceDefinitions().Update(context.TODO(), structuralCRD, metav1.UpdateOptions{}); err != nil { 302 t.Fatal(err) 303 } 304 waitForSpec(structuralCRD, "boolean") 305 306 t.Logf("Check that non-structural schema is not published") 307 schema, err := getPublishedSchema(crdDefinitionName(nonStructuralCRD)) 308 if err != nil { 309 t.Fatal(err) 310 } 311 if schema == nil { 312 t.Fatal("expected a non-nil schema") 313 } 314 if foo, ok := schema.Properties["foo"]; ok { 315 t.Fatalf("unexpected published 'foo' property: %#v", foo) 316 } 317 } 318 319 func crdDefinitionName(crd *apiextensionsv1.CustomResourceDefinition) string { 320 sgmts := strings.Split(crd.Spec.Group, ".") 321 reverse(sgmts) 322 return strings.Join(append(sgmts, crd.Spec.Versions[0].Name, crd.Spec.Names.Kind), ".") 323 } 324 325 func reverse(s []string) { 326 for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 { 327 s[i], s[j] = s[j], s[i] 328 } 329 }