k8s.io/kubernetes@v1.29.3/test/integration/auth/podsecurity_test.go (about) 1 /* 2 Copyright 2021 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package auth 18 19 import ( 20 "context" 21 "crypto/tls" 22 "fmt" 23 "io" 24 "net" 25 "net/http" 26 "net/url" 27 "testing" 28 "time" 29 30 admissionregistrationv1 "k8s.io/api/admissionregistration/v1" 31 corev1 "k8s.io/api/core/v1" 32 apierrors "k8s.io/apimachinery/pkg/api/errors" 33 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 34 "k8s.io/apimachinery/pkg/util/wait" 35 apiserver "k8s.io/apiserver/pkg/server" 36 "k8s.io/apiserver/pkg/server/dynamiccertificates" 37 utilfeature "k8s.io/apiserver/pkg/util/feature" 38 "k8s.io/client-go/kubernetes" 39 "k8s.io/client-go/rest" 40 "k8s.io/component-base/featuregate" 41 featuregatetesting "k8s.io/component-base/featuregate/testing" 42 "k8s.io/component-base/metrics/testutil" 43 kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" 44 "k8s.io/kubernetes/pkg/capabilities" 45 "k8s.io/kubernetes/pkg/features" 46 "k8s.io/kubernetes/test/integration/framework" 47 utiltest "k8s.io/kubernetes/test/utils" 48 podsecurityconfigloader "k8s.io/pod-security-admission/admission/api/load" 49 podsecurityserver "k8s.io/pod-security-admission/cmd/webhook/server" 50 podsecuritytest "k8s.io/pod-security-admission/test" 51 ) 52 53 func TestPodSecurity(t *testing.T) { 54 // Enable all feature gates needed to allow all fields to be exercised 55 defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ProcMountType, true)() 56 defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AppArmor, true)() 57 // Start server 58 server := startPodSecurityServer(t) 59 opts := podsecuritytest.Options{ 60 ClientConfig: server.ClientConfig, 61 62 // Don't pass in feature-gate info, so all testcases run 63 64 // TODO 65 ExemptClient: nil, 66 ExemptNamespaces: []string{}, 67 ExemptRuntimeClasses: []string{}, 68 } 69 podsecuritytest.Run(t, opts) 70 71 ValidatePluginMetrics(t, opts.ClientConfig) 72 } 73 74 // TestPodSecurityGAOnly ensures policies pass with only GA features enabled 75 func TestPodSecurityGAOnly(t *testing.T) { 76 // Disable all alpha and beta features 77 for k, v := range utilfeature.DefaultFeatureGate.DeepCopy().GetAll() { 78 if k == "AllAlpha" || k == "AllBeta" { 79 // Skip special features. When processed first, special features may 80 // erroneously disable other features. 81 continue 82 } else if v.PreRelease == featuregate.Alpha || v.PreRelease == featuregate.Beta { 83 defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, k, false)() 84 } 85 } 86 // Start server 87 server := startPodSecurityServer(t) 88 89 opts := podsecuritytest.Options{ 90 ClientConfig: server.ClientConfig, 91 // Pass in feature gate info so negative test cases depending on alpha or beta features can be skipped 92 Features: utilfeature.DefaultFeatureGate, 93 } 94 podsecuritytest.Run(t, opts) 95 96 ValidatePluginMetrics(t, opts.ClientConfig) 97 } 98 99 func TestPodSecurityWebhook(t *testing.T) { 100 // Enable all feature gates needed to allow all fields to be exercised 101 defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ProcMountType, true)() 102 defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AppArmor, true)() 103 104 // Start test API server. 105 capabilities.SetForTests(capabilities.Capabilities{AllowPrivileged: true}) 106 testServer := kubeapiservertesting.StartTestServerOrDie(t, kubeapiservertesting.NewDefaultTestServerOptions(), []string{ 107 "--anonymous-auth=false", 108 "--allow-privileged=true", 109 // The webhook should pass tests even when PodSecurity is disabled. 110 "--disable-admission-plugins=PodSecurity", 111 }, framework.SharedEtcd()) 112 t.Cleanup(testServer.TearDownFn) 113 114 webhookAddr, err := startPodSecurityWebhook(t, testServer) 115 if err != nil { 116 t.Fatalf("Failed to start webhook server: %v", err) 117 } 118 if err := installWebhook(t, testServer.ClientConfig, webhookAddr); err != nil { 119 t.Fatalf("Failed to install webhook configuration: %v", err) 120 } 121 122 opts := podsecuritytest.Options{ 123 ClientConfig: testServer.ClientConfig, 124 125 // Don't pass in feature-gate info, so all testcases run 126 127 // TODO 128 ExemptClient: nil, 129 ExemptNamespaces: []string{}, 130 ExemptRuntimeClasses: []string{}, 131 } 132 podsecuritytest.Run(t, opts) 133 134 ValidateWebhookMetrics(t, webhookAddr) 135 } 136 137 func startPodSecurityServer(t *testing.T) *kubeapiservertesting.TestServer { 138 // ensure the global is set to allow privileged containers 139 capabilities.SetForTests(capabilities.Capabilities{AllowPrivileged: true}) 140 141 server := kubeapiservertesting.StartTestServerOrDie(t, kubeapiservertesting.NewDefaultTestServerOptions(), []string{ 142 "--anonymous-auth=false", 143 "--enable-admission-plugins=PodSecurity", 144 "--allow-privileged=true", 145 // TODO: "--admission-control-config-file=" + admissionConfigFile.Name(), 146 }, framework.SharedEtcd()) 147 t.Cleanup(server.TearDownFn) 148 return server 149 } 150 151 func startPodSecurityWebhook(t *testing.T, testServer *kubeapiservertesting.TestServer) (addr string, err error) { 152 // listener, port, err := apiserver.CreateListener("tcp", "127.0.0.1:", net.ListenConfig{}) 153 secureListener, err := net.Listen("tcp", "127.0.0.1:") 154 if err != nil { 155 return "", err 156 } 157 insecureListener, err := net.Listen("tcp", "127.0.0.1:") 158 if err != nil { 159 return "", err 160 } 161 cert, err := dynamiccertificates.NewStaticCertKeyContent("localhost", utiltest.LocalhostCert, utiltest.LocalhostKey) 162 if err != nil { 163 return "", err 164 } 165 defaultConfig, err := podsecurityconfigloader.LoadFromData(nil) // load the default 166 if err != nil { 167 return "", err 168 } 169 170 c := podsecurityserver.Config{ 171 SecureServing: &apiserver.SecureServingInfo{ 172 Listener: secureListener, 173 Cert: cert, 174 }, 175 InsecureServing: &apiserver.DeprecatedInsecureServingInfo{ 176 Listener: insecureListener, 177 }, 178 KubeConfig: testServer.ClientConfig, 179 PodSecurityConfig: defaultConfig, 180 } 181 182 t.Logf("Starting webhook server...") 183 webhookServer, err := podsecurityserver.Setup(&c) 184 if err != nil { 185 return "", err 186 } 187 188 ctx, cancel := context.WithCancel(context.Background()) 189 go webhookServer.Start(ctx) 190 t.Cleanup(cancel) 191 192 // Wait for server to be ready 193 t.Logf("Waiting for webhook server /readyz to be ok...") 194 readyz := (&url.URL{ 195 Scheme: "http", 196 Host: c.InsecureServing.Listener.Addr().String(), 197 Path: "/readyz", 198 }).String() 199 if err := wait.PollImmediate(100*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { 200 resp, err := http.Get(readyz) 201 if err != nil { 202 return false, err 203 } 204 defer resp.Body.Close() 205 return resp.StatusCode == 200, nil 206 }); err != nil { 207 return "", err 208 } 209 210 return c.SecureServing.Listener.Addr().String(), nil 211 } 212 213 func installWebhook(t *testing.T, clientConfig *rest.Config, addr string) error { 214 client, err := kubernetes.NewForConfig(clientConfig) 215 if err != nil { 216 return fmt.Errorf("error creating client: %w", err) 217 } 218 219 fail := admissionregistrationv1.Fail 220 equivalent := admissionregistrationv1.Equivalent 221 none := admissionregistrationv1.SideEffectClassNone 222 endpoint := (&url.URL{ 223 Scheme: "https", 224 Host: addr, 225 }).String() 226 227 // Installing Admission webhook to API server 228 _, err = client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Create(context.TODO(), &admissionregistrationv1.ValidatingWebhookConfiguration{ 229 ObjectMeta: metav1.ObjectMeta{Name: "podsecurity-webhook.integration.test"}, 230 Webhooks: []admissionregistrationv1.ValidatingWebhook{ 231 { 232 Name: "podsecurity-webhook.integration.test", 233 ClientConfig: admissionregistrationv1.WebhookClientConfig{ 234 URL: &endpoint, 235 CABundle: utiltest.LocalhostCert, 236 }, 237 Rules: []admissionregistrationv1.RuleWithOperations{ 238 { 239 Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create, admissionregistrationv1.Update}, 240 Rule: admissionregistrationv1.Rule{ 241 APIGroups: []string{""}, 242 APIVersions: []string{"v1"}, 243 Resources: []string{"namespaces", "pods", "pods/ephemeralcontainers", "replicationcontrollers", "podtemplates"}, 244 }, 245 }, 246 { 247 Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create, admissionregistrationv1.Update}, 248 Rule: admissionregistrationv1.Rule{ 249 APIGroups: []string{"apps"}, 250 APIVersions: []string{"v1"}, 251 Resources: []string{"replicasets", "deployments", "statefulsets", "daemonsets"}, 252 }, 253 }, 254 { 255 Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create, admissionregistrationv1.Update}, 256 Rule: admissionregistrationv1.Rule{ 257 APIGroups: []string{"batch"}, 258 APIVersions: []string{"v1"}, 259 Resources: []string{"cronjobs", "jobs"}, 260 }, 261 }, 262 }, 263 FailurePolicy: &fail, 264 MatchPolicy: &equivalent, 265 AdmissionReviewVersions: []string{"v1"}, 266 SideEffects: &none, 267 }, 268 }, 269 }, metav1.CreateOptions{}) 270 if err != nil { 271 return err 272 } 273 274 t.Logf("Waiting for webhook to be established...") 275 invalidNamespace := &corev1.Namespace{ 276 ObjectMeta: metav1.ObjectMeta{ 277 Name: "validation-fail", 278 Labels: map[string]string{ 279 "pod-security.kubernetes.io/enforce": "invalid", 280 }, 281 }, 282 } 283 // Wait for the invalid namespace to be rejected. 284 if err := wait.PollImmediate(100*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { 285 _, err := client.CoreV1().Namespaces().Create(context.TODO(), invalidNamespace, metav1.CreateOptions{DryRun: []string{metav1.DryRunAll}}) 286 if err != nil && apierrors.IsInvalid(err) { 287 return true, nil // An Invalid error indicates the webhook rejected the invalid level. 288 } 289 return false, nil 290 }); err != nil { 291 return err 292 } 293 294 return nil 295 } 296 297 func ValidatePluginMetrics(t *testing.T, clientConfig *rest.Config) { 298 client, err := kubernetes.NewForConfig(clientConfig) 299 if err != nil { 300 t.Fatalf("Error creating client: %v", err) 301 } 302 ctx := context.Background() 303 data, err := client.CoreV1().RESTClient().Get().AbsPath("metrics").DoRaw(ctx) 304 if err != nil { 305 t.Fatalf("Failed to read metrics: %v", err) 306 } 307 validateMetrics(t, data) 308 } 309 310 func ValidateWebhookMetrics(t *testing.T, webhookAddr string) { 311 endpoint := &url.URL{ 312 Scheme: "https", 313 Host: webhookAddr, 314 Path: "/metrics", 315 } 316 client := &http.Client{Transport: &http.Transport{ 317 TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, 318 }} 319 resp, err := client.Get(endpoint.String()) 320 if err != nil { 321 t.Fatalf("Failed to fetch metrics from %s: %v", endpoint.String(), err) 322 } 323 defer resp.Body.Close() 324 if resp.StatusCode != http.StatusOK { 325 t.Fatalf("Non-200 response trying to scrape metrics from %s: %v", endpoint.String(), resp) 326 } 327 data, err := io.ReadAll(resp.Body) 328 if err != nil { 329 t.Fatalf("Unable to read metrics response: %v", err) 330 } 331 validateMetrics(t, data) 332 } 333 334 func validateMetrics(t *testing.T, rawMetrics []byte) { 335 metrics := testutil.NewMetrics() 336 if err := testutil.ParseMetrics(string(rawMetrics), &metrics); err != nil { 337 t.Fatalf("Failed to parse metrics: %v", err) 338 } 339 340 if err := testutil.ValidateMetrics(metrics, "pod_security_evaluations_total", 341 "decision", "policy_level", "policy_version", "mode", "request_operation", "resource", "subresource"); err != nil { 342 t.Errorf("Metric validation failed: %v", err) 343 } 344 if err := testutil.ValidateMetrics(metrics, "pod_security_exemptions_total", 345 "request_operation", "resource", "subresource"); err != nil { 346 t.Errorf("Metric validation failed: %v", err) 347 } 348 }