k8s.io/kubernetes@v1.31.0-alpha.0.0.20240520171757-56147500dadc/test/integration/apiserver/apply/reset_fields_test.go (about) 1 /* 2 Copyright 2020 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package apiserver 18 19 import ( 20 "context" 21 "encoding/json" 22 "fmt" 23 "reflect" 24 "strings" 25 "testing" 26 27 v1 "k8s.io/api/core/v1" 28 apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" 29 "k8s.io/apimachinery/pkg/api/meta" 30 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 31 "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 32 "k8s.io/apimachinery/pkg/runtime/schema" 33 "k8s.io/client-go/dynamic" 34 "k8s.io/client-go/kubernetes" 35 apiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" 36 37 "k8s.io/kubernetes/test/integration/etcd" 38 "k8s.io/kubernetes/test/integration/framework" 39 "k8s.io/kubernetes/test/utils/image" 40 ) 41 42 // namespace used for all tests, do not change this 43 const resetFieldsNamespace = "reset-fields-namespace" 44 45 // resetFieldsStatusData contains statuses for all the resources in the 46 // statusData list with slightly different data to create a field manager 47 // conflict. 48 var resetFieldsStatusData = map[schema.GroupVersionResource]string{ 49 gvr("", "v1", "persistentvolumes"): `{"status": {"message": "hello2"}}`, 50 gvr("", "v1", "resourcequotas"): `{"status": {"used": {"cpu": "25M"}}}`, 51 gvr("", "v1", "services"): `{"status": {"loadBalancer": {"ingress": [{"ip": "127.0.0.2", "ipMode": "VIP"}]}}}`, 52 gvr("extensions", "v1beta1", "ingresses"): `{"status": {"loadBalancer": {"ingress": [{"ip": "127.0.0.2"}]}}}`, 53 gvr("networking.k8s.io", "v1beta1", "ingresses"): `{"status": {"loadBalancer": {"ingress": [{"ip": "127.0.0.2"}]}}}`, 54 gvr("networking.k8s.io", "v1", "ingresses"): `{"status": {"loadBalancer": {"ingress": [{"ip": "127.0.0.2"}]}}}`, 55 gvr("autoscaling", "v1", "horizontalpodautoscalers"): `{"status": {"currentReplicas": 25}}`, 56 gvr("autoscaling", "v2", "horizontalpodautoscalers"): `{"status": {"currentReplicas": 25}}`, 57 gvr("batch", "v1", "cronjobs"): `{"status": {"lastScheduleTime": "2020-01-01T00:00:00Z"}}`, 58 gvr("batch", "v1beta1", "cronjobs"): `{"status": {"lastScheduleTime": "2020-01-01T00:00:00Z"}}`, 59 gvr("storage.k8s.io", "v1", "volumeattachments"): `{"status": {"attached": false}}`, 60 gvr("policy", "v1", "poddisruptionbudgets"): `{"status": {"currentHealthy": 25}}`, 61 gvr("policy", "v1beta1", "poddisruptionbudgets"): `{"status": {"currentHealthy": 25}}`, 62 gvr("resource.k8s.io", "v1alpha2", "podschedulingcontexts"): `{"status": {"resourceClaims": [{"name": "my-claim", "unsuitableNodes": ["node2"]}]}}`, // Not really a conflict with status_test.go: Apply just stores both nodes. Conflict testing therefore gets disabled for podschedulingcontexts. 63 gvr("resource.k8s.io", "v1alpha2", "resourceclaims"): `{"status": {"driverName": "other.example.com"}}`, 64 gvr("internal.apiserver.k8s.io", "v1alpha1", "storageversions"): `{"status": {"commonEncodingVersion":"v1","storageVersions":[{"apiServerID":"1","decodableVersions":["v1","v2"],"encodingVersion":"v1"}],"conditions":[{"type":"AllEncodingVersionsEqual","status":"False","lastTransitionTime":"2020-01-01T00:00:00Z","reason":"allEncodingVersionsEqual","message":"all encoding versions are set to v1"}]}}`, 65 // standard for []metav1.Condition 66 gvr("admissionregistration.k8s.io", "v1alpha1", "validatingadmissionpolicies"): `{"status": {"conditions":[{"type":"Accepted","status":"True","lastTransitionTime":"2020-01-01T00:00:00Z","reason":"RuleApplied","message":"Rule was applied"}]}}`, 67 gvr("admissionregistration.k8s.io", "v1beta1", "validatingadmissionpolicies"): `{"status": {"conditions":[{"type":"Accepted","status":"True","lastTransitionTime":"2020-01-01T00:00:00Z","reason":"RuleApplied","message":"Rule was applied"}]}}`, 68 gvr("admissionregistration.k8s.io", "v1", "validatingadmissionpolicies"): `{"status": {"conditions":[{"type":"Accepted","status":"True","lastTransitionTime":"2020-01-01T00:00:00Z","reason":"RuleApplied","message":"Rule was applied"}]}}`, 69 gvr("networking.k8s.io", "v1alpha1", "servicecidrs"): `{"status": {"conditions":[{"type":"Accepted","status":"True","lastTransitionTime":"2020-01-01T00:00:00Z","reason":"RuleApplied","message":"Rule was applied"}]}}`, 70 } 71 72 // resetFieldsStatusDefault conflicts with statusDefault 73 const resetFieldsStatusDefault = `{"status": {"conditions": [{"type": "MyStatus", "status":"False"}]}}` 74 75 var resetFieldsSkippedResources = map[string]struct{}{} 76 77 // noConflicts is the set of reources for which 78 // a conflict cannot occur. 79 var noConflicts = map[string]struct{}{ 80 // both spec and status get wiped for CSRs, 81 // nothing is expected to be managed for it, skip it 82 "certificatesigningrequests": {}, 83 // storageVersions are skipped because their spec is empty 84 // and thus they can never have a conflict. 85 "storageversions": {}, 86 // servicecidrs are skipped because their spec is inmutable 87 // and thus they can never have a conflict. 88 "servicecidrs": {}, 89 // namespaces only have a spec.finalizers field which is also skipped, 90 // thus it will never have a conflict. 91 "namespaces": {}, 92 // podschedulingcontexts.status only has a list which contains items with a list, 93 // therefore apply works because it simply merges either the outer or 94 // the inner list. 95 "podschedulingcontexts": {}, 96 } 97 98 var image2 = image.GetE2EImage(image.Etcd) 99 100 // resetFieldsSpecData contains conflicting data with the objects in 101 // etcd.GetEtcdStorageDataForNamespace() 102 // It contains the minimal changes needed to conflict with all the fields 103 // added to resetFields by the strategy of each resource. 104 // In most cases, just one field on the spec is changed, but 105 // some also wipe metadata or other fields. 106 var resetFieldsSpecData = map[schema.GroupVersionResource]string{ 107 gvr("", "v1", "resourcequotas"): `{"spec": {"hard": {"cpu": "25M"}}}`, 108 gvr("", "v1", "namespaces"): `{"spec": {"finalizers": ["kubernetes2"]}}`, 109 gvr("", "v1", "nodes"): `{"spec": {"unschedulable": false}}`, 110 gvr("", "v1", "persistentvolumes"): `{"spec": {"capacity": {"storage": "23M"}}}`, 111 gvr("", "v1", "persistentvolumeclaims"): `{"spec": {"resources": {"limits": {"storage": "21M"}}}}`, 112 gvr("", "v1", "pods"): `{"metadata": {"deletionTimestamp": "2020-01-01T00:00:00Z", "ownerReferences":[]}, "spec": {"containers": [{"image": "` + image2 + `", "name": "container7"}]}}`, 113 gvr("", "v1", "replicationcontrollers"): `{"spec": {"selector": {"new": "stuff2"}}}`, 114 gvr("", "v1", "resourcequotas"): `{"spec": {"hard": {"cpu": "25M"}}}`, 115 gvr("", "v1", "services"): `{"spec": {"type": "ClusterIP"}}`, 116 gvr("apps", "v1", "daemonsets"): `{"spec": {"template": {"spec": {"containers": [{"image": "` + image2 + `", "name": "container6"}]}}}}`, 117 gvr("apps", "v1", "deployments"): `{"metadata": {"labels": {"a":"c"}}, "spec": {"template": {"spec": {"containers": [{"image": "` + image2 + `", "name": "container6"}]}}}}`, 118 gvr("apps", "v1", "replicasets"): `{"spec": {"template": {"spec": {"containers": [{"image": "` + image2 + `", "name": "container4"}]}}}}`, 119 gvr("apps", "v1", "statefulsets"): `{"spec": {"selector": {"matchLabels": {"a2": "b2"}}}}`, 120 gvr("autoscaling", "v1", "horizontalpodautoscalers"): `{"spec": {"maxReplicas": 23}}`, 121 gvr("autoscaling", "v2", "horizontalpodautoscalers"): `{"spec": {"maxReplicas": 23}}`, 122 gvr("autoscaling", "v2beta1", "horizontalpodautoscalers"): `{"spec": {"maxReplicas": 23}}`, 123 gvr("autoscaling", "v2beta2", "horizontalpodautoscalers"): `{"spec": {"maxReplicas": 23}}`, 124 gvr("batch", "v1", "jobs"): `{"spec": {"template": {"spec": {"containers": [{"image": "` + image2 + `", "name": "container1"}]}}}}`, 125 gvr("batch", "v1", "cronjobs"): `{"spec": {"jobTemplate": {"spec": {"template": {"spec": {"containers": [{"image": "` + image2 + `", "name": "container0"}]}}}}}}`, 126 gvr("batch", "v1beta1", "cronjobs"): `{"spec": {"jobTemplate": {"spec": {"template": {"spec": {"containers": [{"image": "` + image2 + `", "name": "container0"}]}}}}}}`, 127 gvr("certificates.k8s.io", "v1", "certificatesigningrequests"): `{}`, 128 gvr("certificates.k8s.io", "v1beta1", "certificatesigningrequests"): `{}`, 129 gvr("flowcontrol.apiserver.k8s.io", "v1alpha1", "flowschemas"): `{"metadata": {"labels":{"a":"c"}}, "spec": {"priorityLevelConfiguration": {"name": "name2"}}}`, 130 gvr("flowcontrol.apiserver.k8s.io", "v1beta1", "flowschemas"): `{"metadata": {"labels":{"a":"c"}}, "spec": {"priorityLevelConfiguration": {"name": "name2"}}}`, 131 gvr("flowcontrol.apiserver.k8s.io", "v1beta2", "flowschemas"): `{"metadata": {"labels":{"a":"c"}}, "spec": {"priorityLevelConfiguration": {"name": "name2"}}}`, 132 gvr("flowcontrol.apiserver.k8s.io", "v1beta3", "flowschemas"): `{"metadata": {"labels":{"a":"c"}}, "spec": {"priorityLevelConfiguration": {"name": "name2"}}}`, 133 gvr("flowcontrol.apiserver.k8s.io", "v1", "flowschemas"): `{"metadata": {"labels":{"a":"c"}}, "spec": {"priorityLevelConfiguration": {"name": "name2"}}}`, 134 gvr("flowcontrol.apiserver.k8s.io", "v1alpha1", "prioritylevelconfigurations"): `{"metadata": {"labels":{"a":"c"}}, "spec": {"limited": {"assuredConcurrencyShares": 23}}}`, 135 gvr("flowcontrol.apiserver.k8s.io", "v1beta1", "prioritylevelconfigurations"): `{"metadata": {"labels":{"a":"c"}}, "spec": {"limited": {"assuredConcurrencyShares": 23}}}`, 136 gvr("flowcontrol.apiserver.k8s.io", "v1beta2", "prioritylevelconfigurations"): `{"metadata": {"labels":{"a":"c"}}, "spec": {"limited": {"assuredConcurrencyShares": 23}}}`, 137 gvr("flowcontrol.apiserver.k8s.io", "v1beta3", "prioritylevelconfigurations"): `{"metadata": {"labels":{"a":"c"}}, "spec": {"limited": {"nominalConcurrencyShares": 23}}}`, 138 gvr("flowcontrol.apiserver.k8s.io", "v1", "prioritylevelconfigurations"): `{"metadata": {"labels":{"a":"c"}}, "spec": {"limited": {"nominalConcurrencyShares": 23}}}`, 139 gvr("extensions", "v1beta1", "ingresses"): `{"spec": {"backend": {"serviceName": "service2"}}}`, 140 gvr("networking.k8s.io", "v1beta1", "ingresses"): `{"spec": {"backend": {"serviceName": "service2"}}}`, 141 gvr("networking.k8s.io", "v1", "ingresses"): `{"spec": {"defaultBackend": {"service": {"name": "service2"}}}}`, 142 gvr("networking.k8s.io", "v1alpha1", "servicecidrs"): `{}`, 143 gvr("policy", "v1", "poddisruptionbudgets"): `{"spec": {"selector": {"matchLabels": {"anokkey2": "anokvalue"}}}}`, 144 gvr("policy", "v1beta1", "poddisruptionbudgets"): `{"spec": {"selector": {"matchLabels": {"anokkey2": "anokvalue"}}}}`, 145 gvr("storage.k8s.io", "v1alpha1", "volumeattachments"): `{"metadata": {"name": "va3"}, "spec": {"nodeName": "localhost2"}}`, 146 gvr("storage.k8s.io", "v1", "volumeattachments"): `{"metadata": {"name": "va3"}, "spec": {"nodeName": "localhost2"}}`, 147 gvr("apiextensions.k8s.io", "v1", "customresourcedefinitions"): `{"metadata": {"labels":{"a":"c"}}, "spec": {"group": "webconsole22.operator.openshift.io"}}`, 148 gvr("apiextensions.k8s.io", "v1beta1", "customresourcedefinitions"): `{"metadata": {"labels":{"a":"c"}}, "spec": {"group": "webconsole22.operator.openshift.io"}}`, 149 gvr("awesome.bears.com", "v1", "pandas"): `{"spec": {"replicas": 102}}`, 150 gvr("awesome.bears.com", "v3", "pandas"): `{"spec": {"replicas": 302}}`, 151 gvr("apiregistration.k8s.io", "v1beta1", "apiservices"): `{"metadata": {"labels": {"a":"c"}}, "spec": {"group": "foo2.com"}}`, 152 gvr("apiregistration.k8s.io", "v1", "apiservices"): `{"metadata": {"labels": {"a":"c"}}, "spec": {"group": "foo2.com"}}`, 153 gvr("resource.k8s.io", "v1alpha2", "podschedulingcontexts"): `{"spec": {"selectedNode": "node2name"}}`, 154 gvr("resource.k8s.io", "v1alpha2", "resourceclasses"): `{"driverName": "other.example.com"}`, 155 gvr("resource.k8s.io", "v1alpha2", "resourceclaims"): `{"spec": {"resourceClassName": "class2name"}}`, // ResourceClassName is immutable, but that doesn't matter for the test. 156 gvr("resource.k8s.io", "v1alpha2", "resourceclaimtemplates"): `{"spec": {"spec": {"resourceClassName": "class2name"}}}`, 157 gvr("internal.apiserver.k8s.io", "v1alpha1", "storageversions"): `{}`, 158 gvr("admissionregistration.k8s.io", "v1alpha1", "validatingadmissionpolicies"): `{"metadata": {"labels": {"a":"c"}}, "spec": {"paramKind": {"apiVersion": "apps/v1", "kind": "Deployment"}}}`, 159 gvr("admissionregistration.k8s.io", "v1beta1", "validatingadmissionpolicies"): `{"metadata": {"labels": {"a":"c"}}, "spec": {"paramKind": {"apiVersion": "apps/v1", "kind": "Deployment"}}}`, 160 gvr("admissionregistration.k8s.io", "v1", "validatingadmissionpolicies"): `{"metadata": {"labels": {"a":"c"}}, "spec": {"paramKind": {"apiVersion": "apps/v1", "kind": "Deployment"}}}`, 161 } 162 163 // TestResetFields makes sure that fieldManager does not own fields reset by the storage strategy. 164 // It takes 2 objects obj1 and obj2 that differ by one field in the spec and one field in the status. 165 // It applies obj1 to the spec endpoint and obj2 to the status endpoint, the lack of conflicts 166 // confirms that the fieldmanager1 is wiped of the status and fieldmanager2 is wiped of the spec. 167 // We then attempt to apply obj2 to the spec endpoint which fails with an expected conflict. 168 func TestApplyResetFields(t *testing.T) { 169 server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), []string{"--disable-admission-plugins", "ServiceAccount,TaintNodesByCondition"}, framework.SharedEtcd()) 170 if err != nil { 171 t.Fatal(err) 172 } 173 defer server.TearDownFn() 174 175 client, err := kubernetes.NewForConfig(server.ClientConfig) 176 if err != nil { 177 t.Fatal(err) 178 } 179 dynamicClient, err := dynamic.NewForConfig(server.ClientConfig) 180 if err != nil { 181 t.Fatal(err) 182 } 183 184 // create CRDs so we can make sure that custom resources do not get lost 185 etcd.CreateTestCRDs(t, apiextensionsclientset.NewForConfigOrDie(server.ClientConfig), false, etcd.GetCustomResourceDefinitionData()...) 186 187 if _, err := client.CoreV1().Namespaces().Create(context.TODO(), &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: resetFieldsNamespace}}, metav1.CreateOptions{}); err != nil { 188 t.Fatal(err) 189 } 190 191 createData := etcd.GetEtcdStorageDataForNamespace(resetFieldsNamespace) 192 // gather resources to test 193 _, resourceLists, err := client.Discovery().ServerGroupsAndResources() 194 if err != nil { 195 t.Fatalf("Failed to get ServerGroupsAndResources with error: %+v", err) 196 } 197 198 for _, resourceList := range resourceLists { 199 for _, resource := range resourceList.APIResources { 200 if !strings.HasSuffix(resource.Name, "/status") { 201 continue 202 } 203 mapping, err := createMapping(resourceList.GroupVersion, resource) 204 if err != nil { 205 t.Fatal(err) 206 } 207 t.Run(mapping.Resource.String(), func(t *testing.T) { 208 if _, ok := resetFieldsSkippedResources[mapping.Resource.Resource]; ok { 209 t.Skip() 210 } 211 212 namespace := resetFieldsNamespace 213 if mapping.Scope == meta.RESTScopeRoot { 214 namespace = "" 215 } 216 217 // assemble first object 218 status, ok := statusData[mapping.Resource] 219 if !ok { 220 status = statusDefault 221 } 222 223 resource, ok := createData[mapping.Resource] 224 if !ok { 225 t.Fatalf("no test data for %s. Please add a test for your new type to etcd.GetEtcdStorageData() or getResetFieldsEtcdStorageData()", mapping.Resource) 226 } 227 228 obj1 := unstructured.Unstructured{} 229 if err := json.Unmarshal([]byte(resource.Stub), &obj1.Object); err != nil { 230 t.Fatal(err) 231 } 232 if err := json.Unmarshal([]byte(status), &obj1.Object); err != nil { 233 t.Fatal(err) 234 } 235 236 name := obj1.GetName() 237 obj1.SetAPIVersion(mapping.GroupVersionKind.GroupVersion().String()) 238 obj1.SetKind(mapping.GroupVersionKind.Kind) 239 obj1.SetName(name) 240 241 // apply the spec of the first object 242 _, err = dynamicClient. 243 Resource(mapping.Resource). 244 Namespace(namespace). 245 Apply(context.TODO(), name, &obj1, metav1.ApplyOptions{FieldManager: "fieldmanager1"}) 246 if err != nil { 247 t.Fatalf("Failed to apply obj1: %v", err) 248 } 249 250 // create second object 251 obj2 := &unstructured.Unstructured{} 252 obj1.DeepCopyInto(obj2) 253 if err := json.Unmarshal([]byte(resetFieldsSpecData[mapping.Resource]), &obj2.Object); err != nil { 254 t.Fatal(err) 255 } 256 status2, ok := resetFieldsStatusData[mapping.Resource] 257 if !ok { 258 status2 = resetFieldsStatusDefault 259 } 260 if err := json.Unmarshal([]byte(status2), &obj2.Object); err != nil { 261 t.Fatal(err) 262 } 263 264 if reflect.DeepEqual(obj1, obj2) { 265 t.Fatalf("obj1 and obj2 should not be equal %v", obj2) 266 } 267 268 // apply the status of the second object 269 // this won't conflict if resetfields are set correctly 270 // and will conflict if they are not 271 _, err = dynamicClient. 272 Resource(mapping.Resource). 273 Namespace(namespace). 274 ApplyStatus(context.TODO(), name, obj2, metav1.ApplyOptions{FieldManager: "fieldmanager2"}) 275 if err != nil { 276 t.Fatalf("Failed to apply obj2: %v", err) 277 } 278 279 // skip checking for conflicts on resources 280 // that will never have conflicts 281 if _, ok = noConflicts[mapping.Resource.Resource]; !ok { 282 var objRet *unstructured.Unstructured 283 284 // reapply second object to the spec endpoint 285 // that should fail with a conflict 286 objRet, err = dynamicClient. 287 Resource(mapping.Resource). 288 Namespace(namespace). 289 Apply(context.TODO(), name, obj2, metav1.ApplyOptions{FieldManager: "fieldmanager2"}) 290 err = expectConflict(objRet, err, dynamicClient, mapping.Resource, namespace, name) 291 if err != nil { 292 t.Fatalf("Did not get expected conflict in spec of %s %s/%s: %v", mapping.Resource, namespace, name, err) 293 } 294 295 // reapply first object to the status endpoint 296 // that should fail with a conflict 297 objRet, err = dynamicClient. 298 Resource(mapping.Resource). 299 Namespace(namespace). 300 ApplyStatus(context.TODO(), name, &obj1, metav1.ApplyOptions{FieldManager: "fieldmanager1"}) 301 err = expectConflict(objRet, err, dynamicClient, mapping.Resource, namespace, name) 302 if err != nil { 303 t.Fatalf("Did not get expected conflict in status of %s %s/%s: %v", mapping.Resource, namespace, name, err) 304 } 305 } 306 307 // cleanup 308 rsc := dynamicClient.Resource(mapping.Resource).Namespace(namespace) 309 if err := rsc.Delete(context.TODO(), name, *metav1.NewDeleteOptions(0)); err != nil { 310 t.Fatalf("deleting final object failed: %v", err) 311 } 312 }) 313 } 314 } 315 } 316 317 func expectConflict(objRet *unstructured.Unstructured, err error, dynamicClient dynamic.Interface, resource schema.GroupVersionResource, namespace, name string) error { 318 if err != nil && strings.Contains(err.Error(), "conflict") { 319 return nil 320 } 321 which := "returned" 322 // something unexpected is going on here, let's not assume that objRet==nil if any only if err!=nil 323 if objRet == nil { 324 which = "subsequently fetched" 325 var err2 error 326 objRet, err2 = dynamicClient. 327 Resource(resource). 328 Namespace(namespace). 329 Get(context.TODO(), name, metav1.GetOptions{}) 330 if err2 != nil { 331 return fmt.Errorf("instead got error %w, and failed to Get object: %v", err, err2) 332 } 333 } 334 marshBytes, marshErr := json.Marshal(objRet) 335 var gotten string 336 if marshErr == nil { 337 gotten = string(marshBytes) 338 } else { 339 gotten = fmt.Sprintf("<failed to json.Marshall(%#+v): %v>", objRet, marshErr) 340 } 341 return fmt.Errorf("instead got error %w; %s object is %s", err, which, gotten) 342 }