k8s.io/kubernetes@v1.29.3/test/integration/apiserver/podlogs/podlogs_test.go (about) 1 /* 2 Copyright 2018 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package podlogs 18 19 import ( 20 "context" 21 "crypto/ecdsa" 22 "crypto/elliptic" 23 "crypto/rand" 24 "crypto/tls" 25 "crypto/x509" 26 "crypto/x509/pkix" 27 "encoding/pem" 28 "fmt" 29 "math" 30 "math/big" 31 "net" 32 "net/http" 33 "net/http/httptest" 34 "net/url" 35 "os" 36 "strconv" 37 "strings" 38 "testing" 39 "time" 40 41 corev1 "k8s.io/api/core/v1" 42 "k8s.io/apimachinery/pkg/api/errors" 43 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 44 "k8s.io/apimachinery/pkg/util/wait" 45 "k8s.io/apiserver/pkg/authentication/authenticatorfactory" 46 "k8s.io/apiserver/pkg/endpoints/filters" 47 "k8s.io/apiserver/pkg/server/dynamiccertificates" 48 "k8s.io/client-go/kubernetes" 49 "k8s.io/client-go/transport" 50 certutil "k8s.io/client-go/util/cert" 51 "k8s.io/client-go/util/keyutil" 52 "k8s.io/kubernetes/cmd/kube-apiserver/app/options" 53 "k8s.io/kubernetes/test/integration/framework" 54 "k8s.io/kubernetes/test/utils/ktesting" 55 ) 56 57 func TestInsecurePodLogs(t *testing.T) { 58 badCA := writeDataToTempFile(t, []byte(` 59 -----BEGIN CERTIFICATE----- 60 MIIDMDCCAhigAwIBAgIIHNPD7sig7YIwDQYJKoZIhvcNAQELBQAwNjESMBAGA1UE 61 CxMJb3BlbnNoaWZ0MSAwHgYDVQQDExdhZG1pbi1rdWJlY29uZmlnLXNpZ25lcjAe 62 Fw0xOTA1MzAxNTA3MzlaFw0yOTA1MjcxNTA3MzlaMDYxEjAQBgNVBAsTCW9wZW5z 63 aGlmdDEgMB4GA1UEAxMXYWRtaW4ta3ViZWNvbmZpZy1zaWduZXIwggEiMA0GCSqG 64 SIb3DQEBAQUAA4IBDwAwggEKAoIBAQD0dHk23lHRcuq06FzYDOl9J9+s8pnGxqA3 65 IPcARI6ag/98aYe3ENwAB5e1i7AU2F2WiDZgj444w374XLdVgIK8zgQEm9yoqrlc 66 +/ayO7ceKklrKHOMwh63LvGLEOqzhol2nFmBhXAZt+HyIoZHXN0IqlA92196+Dml 67 0WOn1F4ce6JbAtEceFHPgLeI7KFmVaPz2796pBXh23ii6r7WvV1Rn9MKlMSBJQR4 68 0LZzu9/j+GdnFXewdLAAMfgPzwEqv6h3PzvtUCjgdraHEm8Rs7s15S3PUmLK4RQS 69 PsThx5BhJEGd/W6EzQ3BKoQfochhu3mnAQtW1J07CullySQ5Gg9fAgMBAAGjQjBA 70 MA4GA1UdDwEB/wQEAwICpDAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQkTaaw 71 YJSZ5k2Wd+OsM4GFMTGdqzANBgkqhkiG9w0BAQsFAAOCAQEAHK7+zBZPLqK+f9DT 72 UEnpwRmZ0aeGS4YgbGIkqpjxJymVOwkRd5A1wslvVfGZ6yOQthF6KlCmqnPyJJMR 73 I7FHw8j0h2ci90fEQ6IS90Y/ZJXkcgiK9Ncwa35GFGs8QrBxN4leGhtm84BnnBHN 74 cTWpa4zcBwru0CRG7iHc66VX16X8jHB1iFeZ5W/FgY4MsE+G1Vze4mCXSPVI4BZ2 75 /qlAgogjBivvSwQ9SFuCszg7IPjvT2ksm+Cf+8eT4YBqW41F85vBGR+FYK14yIla 76 Bgqc+dJN9xS9Ah5gLiGQJ6C4niUA11piCpvMsy+j/LQ1Erx47KMar5fuMXYk7iPq 77 1vqIwg== 78 -----END CERTIFICATE----- 79 `)) 80 81 _, ctx := ktesting.NewTestContext(t) 82 ctx, cancel := context.WithCancel(ctx) 83 defer cancel() 84 85 clientSet, _, tearDownFn := framework.StartTestServer(ctx, t, framework.TestServerSetup{ 86 ModifyServerRunOptions: func(opts *options.ServerRunOptions) { 87 opts.GenericServerRunOptions.MaxRequestBodyBytes = 1024 * 1024 88 // I have no idea what this cert is, but it doesn't matter, we just want something that always fails validation 89 opts.KubeletConfig.TLSClientConfig.CAFile = badCA 90 }, 91 }) 92 defer tearDownFn() 93 94 fakeKubeletServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 95 w.Write([]byte("fake-log")) 96 w.WriteHeader(http.StatusOK) 97 })) 98 defer fakeKubeletServer.Close() 99 100 pod := prepareFakeNodeAndPod(ctx, t, clientSet, fakeKubeletServer) 101 102 insecureResult := clientSet.CoreV1().Pods("ns").GetLogs(pod.Name, &corev1.PodLogOptions{InsecureSkipTLSVerifyBackend: true}).Do(context.TODO()) 103 if err := insecureResult.Error(); err != nil { 104 t.Fatal(err) 105 } 106 insecureStatusCode := 0 107 insecureResult.StatusCode(&insecureStatusCode) 108 if insecureStatusCode != http.StatusOK { 109 t.Fatal(insecureStatusCode) 110 } 111 112 secureResult := clientSet.CoreV1().Pods("ns").GetLogs(pod.Name, &corev1.PodLogOptions{}).Do(ctx) 113 if err := secureResult.Error(); err == nil || !strings.Contains(err.Error(), "x509: certificate signed by unknown authority") { 114 t.Fatal(err) 115 } 116 secureStatusCode := 0 117 secureResult.StatusCode(&secureStatusCode) 118 if secureStatusCode == http.StatusOK { 119 raw, rawErr := secureResult.Raw() 120 if rawErr != nil { 121 t.Log(rawErr) 122 } 123 t.Log(string(raw)) 124 t.Fatal(secureStatusCode) 125 } 126 } 127 128 func prepareFakeNodeAndPod(ctx context.Context, t *testing.T, clientSet kubernetes.Interface, fakeKubeletServer *httptest.Server) *corev1.Pod { 129 t.Helper() 130 131 fakeKubeletURL, err := url.Parse(fakeKubeletServer.URL) 132 if err != nil { 133 t.Fatal(err) 134 } 135 fakeKubeletHost, fakeKubeletPortStr, err := net.SplitHostPort(fakeKubeletURL.Host) 136 if err != nil { 137 t.Fatal(err) 138 } 139 fakeKubeletPort, err := strconv.ParseUint(fakeKubeletPortStr, 10, 32) 140 if err != nil { 141 t.Fatal(err) 142 } 143 144 node, err := clientSet.CoreV1().Nodes().Create(ctx, &corev1.Node{ 145 ObjectMeta: metav1.ObjectMeta{Name: "fake"}, 146 }, metav1.CreateOptions{}) 147 if err != nil { 148 t.Fatal(err) 149 } 150 node.Status = corev1.NodeStatus{ 151 Addresses: []corev1.NodeAddress{ 152 { 153 Type: corev1.NodeExternalIP, 154 Address: fakeKubeletHost, 155 }, 156 }, 157 DaemonEndpoints: corev1.NodeDaemonEndpoints{ 158 KubeletEndpoint: corev1.DaemonEndpoint{ 159 Port: int32(fakeKubeletPort), 160 }, 161 }, 162 } 163 node, err = clientSet.CoreV1().Nodes().UpdateStatus(ctx, node, metav1.UpdateOptions{}) 164 if err != nil { 165 t.Fatal(err) 166 } 167 168 _, err = clientSet.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ 169 ObjectMeta: metav1.ObjectMeta{Name: "ns"}, 170 }, metav1.CreateOptions{}) 171 if err != nil { 172 t.Fatal(err) 173 } 174 175 _, err = clientSet.CoreV1().ServiceAccounts("ns").Create(ctx, &corev1.ServiceAccount{ 176 ObjectMeta: metav1.ObjectMeta{Name: "default", Namespace: "ns"}, 177 }, metav1.CreateOptions{}) 178 if err != nil { 179 t.Fatal(err) 180 } 181 182 falseRef := false 183 pod, err := clientSet.CoreV1().Pods("ns").Create(ctx, &corev1.Pod{ 184 ObjectMeta: metav1.ObjectMeta{Name: "test-pod", Namespace: "ns"}, 185 Spec: corev1.PodSpec{ 186 Containers: []corev1.Container{ 187 { 188 Name: "foo", 189 Image: "some/image:latest", 190 }, 191 }, 192 NodeName: node.Name, 193 AutomountServiceAccountToken: &falseRef, 194 }, 195 }, metav1.CreateOptions{}) 196 if err != nil { 197 t.Fatal(err) 198 } 199 200 return pod 201 } 202 203 func TestPodLogsKubeletClientCertReload(t *testing.T) { 204 ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) 205 t.Cleanup(cancel) 206 207 origCertCallbackRefreshDuration := transport.CertCallbackRefreshDuration 208 origDialerStopCh := transport.DialerStopCh 209 transport.CertCallbackRefreshDuration = time.Second // make client cert reloading fast 210 transport.DialerStopCh = ctx.Done() 211 t.Cleanup(func() { 212 transport.CertCallbackRefreshDuration = origCertCallbackRefreshDuration 213 transport.DialerStopCh = origDialerStopCh 214 }) 215 216 // create a CA to sign the API server's kubelet client cert 217 startingCerts := generateClientCert(t) 218 219 dynamicCAContentFromFile, err := dynamiccertificates.NewDynamicCAContentFromFile("client-ca-bundle", startingCerts.caFile) 220 if err != nil { 221 t.Fatal(err) 222 } 223 if err := dynamicCAContentFromFile.RunOnce(ctx); err != nil { 224 t.Fatal(err) 225 } 226 go dynamicCAContentFromFile.Run(ctx, 1) 227 authenticatorConfig := authenticatorfactory.DelegatingAuthenticatorConfig{ 228 ClientCertificateCAContentProvider: dynamicCAContentFromFile, 229 } 230 authenticator, _, err := authenticatorConfig.New() 231 if err != nil { 232 t.Fatal(err) 233 } 234 235 // this fake kubelet will perform per request authentication using the configured CA (which is dynamically reloaded) 236 fakeKubeletServer := httptest.NewUnstartedServer( 237 filters.WithAuthentication( 238 http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { 239 _, _ = w.Write([]byte("pod-logs-here")) 240 }), 241 authenticator, 242 http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { 243 w.WriteHeader(http.StatusUnauthorized) 244 }), 245 nil, 246 nil, 247 ), 248 ) 249 fakeKubeletServer.TLS = &tls.Config{ClientAuth: tls.RequestClientCert} 250 fakeKubeletServer.StartTLS() 251 t.Cleanup(fakeKubeletServer.Close) 252 253 kubeletCA := writeDataToTempFile(t, pem.EncodeToMemory(&pem.Block{ 254 Type: "CERTIFICATE", 255 Bytes: fakeKubeletServer.Certificate().Raw, 256 })) 257 258 clientSet, _, tearDownFn := framework.StartTestServer(ctx, t, framework.TestServerSetup{ 259 ModifyServerRunOptions: func(opts *options.ServerRunOptions) { 260 opts.GenericServerRunOptions.MaxRequestBodyBytes = 1024 * 1024 261 opts.KubeletConfig.TLSClientConfig.CAFile = kubeletCA 262 opts.KubeletConfig.TLSClientConfig.CertFile = startingCerts.clientCertFile 263 opts.KubeletConfig.TLSClientConfig.KeyFile = startingCerts.clientCertKeyFile 264 }, 265 }) 266 t.Cleanup(tearDownFn) 267 268 pod := prepareFakeNodeAndPod(ctx, t, clientSet, fakeKubeletServer) 269 270 // verify that the starting state works as expected 271 podLogs, err := clientSet.CoreV1().Pods("ns").GetLogs(pod.Name, &corev1.PodLogOptions{}).DoRaw(ctx) 272 if err != nil { 273 t.Fatal(err) 274 } 275 if l := string(podLogs); l != "pod-logs-here" { 276 t.Fatalf("unexpected pod logs: %s", l) 277 } 278 279 // generate a new CA and overwrite the existing CA that the kubelet is using for request authentication 280 newCerts := generateClientCert(t) 281 if err := os.Rename(newCerts.caFile, startingCerts.caFile); err != nil { 282 t.Fatal(err) 283 } 284 285 // wait until the kubelet observes the new CA 286 if err := wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (bool, error) { 287 _, errLog := clientSet.CoreV1().Pods("ns").GetLogs(pod.Name, &corev1.PodLogOptions{}).DoRaw(ctx) 288 if errors.IsUnauthorized(errLog) { 289 return true, nil 290 } 291 return false, errLog 292 }); err != nil { 293 t.Fatal(err) 294 } 295 296 // now update the API server's kubelet client cert to use the new cert 297 if err := os.Rename(newCerts.clientCertFile, startingCerts.clientCertFile); err != nil { 298 t.Fatal(err) 299 } 300 if err := os.Rename(newCerts.clientCertKeyFile, startingCerts.clientCertKeyFile); err != nil { 301 t.Fatal(err) 302 } 303 304 // confirm that the API server observes the new client cert and closes existing connections to use it 305 if err := wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (bool, error) { 306 fixedPodLogs, errLog := clientSet.CoreV1().Pods("ns").GetLogs(pod.Name, &corev1.PodLogOptions{}).DoRaw(ctx) 307 if errors.IsUnauthorized(errLog) { 308 t.Log("api server has not observed new client cert") 309 return false, nil 310 } 311 if errLog != nil { 312 return false, errLog 313 } 314 if l := string(fixedPodLogs); l != "pod-logs-here" { 315 return false, fmt.Errorf("unexpected pod logs: %s", l) 316 } 317 return true, nil 318 }); err != nil { 319 t.Fatal(err) 320 } 321 } 322 323 type testCerts struct { 324 caFile, clientCertFile, clientCertKeyFile string 325 } 326 327 func generateClientCert(t *testing.T) testCerts { 328 t.Helper() 329 330 caPrivateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) 331 if err != nil { 332 t.Fatal(err) 333 } 334 caCert, err := certutil.NewSelfSignedCACert(certutil.Config{CommonName: "test-ca"}, caPrivateKey) 335 if err != nil { 336 t.Fatal(err) 337 } 338 339 clientCertKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) 340 if err != nil { 341 t.Fatal(err) 342 } 343 344 clientCertKeyBytes, err := keyutil.MarshalPrivateKeyToPEM(clientCertKey) 345 if err != nil { 346 t.Fatal(err) 347 } 348 349 // returns a uniform random value in [0, max-1), then add 1 to serial to make it a uniform random value in [1, max). 350 serial, err := rand.Int(rand.Reader, new(big.Int).SetInt64(math.MaxInt64-1)) 351 if err != nil { 352 t.Fatal(err) 353 } 354 serial = new(big.Int).Add(serial, big.NewInt(1)) 355 certTmpl := x509.Certificate{ 356 Subject: pkix.Name{ 357 CommonName: "the-api-server-user", 358 }, 359 NotBefore: caCert.NotBefore, 360 SerialNumber: serial, 361 NotAfter: time.Now().Add(time.Hour).UTC(), 362 KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, 363 ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, 364 } 365 clientCertDERBytes, err := x509.CreateCertificate(rand.Reader, &certTmpl, caCert, clientCertKey.Public(), caPrivateKey) 366 if err != nil { 367 t.Fatal(err) 368 } 369 370 clientCert, err := x509.ParseCertificate(clientCertDERBytes) 371 if err != nil { 372 t.Fatal(err) 373 } 374 375 return testCerts{ 376 caFile: writeDataToTempFile(t, pem.EncodeToMemory(&pem.Block{ 377 Type: "CERTIFICATE", 378 Bytes: caCert.Raw, 379 })), 380 clientCertFile: writeDataToTempFile(t, pem.EncodeToMemory(&pem.Block{ 381 Type: "CERTIFICATE", 382 Bytes: clientCert.Raw, 383 })), 384 clientCertKeyFile: writeDataToTempFile(t, clientCertKeyBytes), 385 } 386 } 387 388 func writeDataToTempFile(t *testing.T, data []byte) string { 389 t.Helper() 390 391 file, err := os.CreateTemp("", "pod-logs-test-") 392 if err != nil { 393 t.Fatal(err) 394 } 395 if _, err := file.Write(data); err != nil { 396 t.Fatal(err) 397 } 398 t.Cleanup(func() { 399 _ = os.Remove(file.Name()) 400 }) 401 return file.Name() 402 }