k8s.io/kubernetes@v1.31.0-alpha.0.0.20240520171757-56147500dadc/test/integration/apiserver/oidc/oidc_test.go (about) 1 /* 2 Copyright 2023 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package oidc 18 19 import ( 20 "context" 21 "crypto/ecdsa" 22 "crypto/elliptic" 23 "crypto/rand" 24 "crypto/rsa" 25 "crypto/tls" 26 "crypto/x509" 27 "encoding/json" 28 "fmt" 29 "net" 30 "net/http" 31 "net/url" 32 "os" 33 "path/filepath" 34 "regexp" 35 "strings" 36 "testing" 37 "time" 38 39 "github.com/google/go-cmp/cmp" 40 "github.com/stretchr/testify/assert" 41 "github.com/stretchr/testify/require" 42 "gopkg.in/square/go-jose.v2" 43 44 authenticationv1 "k8s.io/api/authentication/v1" 45 rbacv1 "k8s.io/api/rbac/v1" 46 apierrors "k8s.io/apimachinery/pkg/api/errors" 47 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 48 utilrand "k8s.io/apimachinery/pkg/util/rand" 49 "k8s.io/apimachinery/pkg/util/wait" 50 "k8s.io/apiserver/pkg/features" 51 genericapiserver "k8s.io/apiserver/pkg/server" 52 authenticationconfigmetrics "k8s.io/apiserver/pkg/server/options/authenticationconfig/metrics" 53 utilfeature "k8s.io/apiserver/pkg/util/feature" 54 "k8s.io/client-go/kubernetes" 55 _ "k8s.io/client-go/plugin/pkg/client/auth/oidc" 56 "k8s.io/client-go/rest" 57 "k8s.io/client-go/tools/clientcmd/api" 58 certutil "k8s.io/client-go/util/cert" 59 featuregatetesting "k8s.io/component-base/featuregate/testing" 60 kubeapiserverapptesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" 61 "k8s.io/kubernetes/pkg/apis/rbac" 62 "k8s.io/kubernetes/pkg/kubeapiserver/options" 63 "k8s.io/kubernetes/test/integration/framework" 64 utilsoidc "k8s.io/kubernetes/test/utils/oidc" 65 utilsnet "k8s.io/utils/net" 66 ) 67 68 const ( 69 defaultNamespace = "default" 70 defaultOIDCClientID = "f403b682-603f-4ec9-b3e4-cf111ef36f7c" 71 defaultOIDCClaimedUsername = "john_doe" 72 defaultOIDCUsernamePrefix = "k8s-" 73 defaultRBACRoleName = "developer-role" 74 defaultRBACRoleBindingName = "developer-role-binding" 75 76 defaultStubRefreshToken = "_fake_refresh_token_" 77 defaultStubAccessToken = "_fake_access_token_" 78 79 rsaKeyBitSize = 2048 80 ) 81 82 var ( 83 defaultRole = &rbacv1.Role{ 84 TypeMeta: metav1.TypeMeta{APIVersion: "rbac.authorization.k8s.io/v1", Kind: "Role"}, 85 ObjectMeta: metav1.ObjectMeta{Name: defaultRBACRoleName}, 86 Rules: []rbacv1.PolicyRule{ 87 { 88 Verbs: []string{"list"}, 89 Resources: []string{"pods"}, 90 APIGroups: []string{""}, 91 ResourceNames: []string{}, 92 }, 93 }, 94 } 95 defaultRoleBinding = &rbacv1.RoleBinding{ 96 TypeMeta: metav1.TypeMeta{APIVersion: "rbac.authorization.k8s.io/v1", Kind: "RoleBinding"}, 97 ObjectMeta: metav1.ObjectMeta{Name: defaultRBACRoleBindingName}, 98 Subjects: []rbacv1.Subject{ 99 { 100 APIGroup: rbac.GroupName, 101 Kind: rbacv1.UserKind, 102 Name: defaultOIDCUsernamePrefix + defaultOIDCClaimedUsername, 103 }, 104 }, 105 RoleRef: rbacv1.RoleRef{ 106 APIGroup: rbac.GroupName, 107 Kind: "Role", 108 Name: defaultRBACRoleName, 109 }, 110 } 111 ) 112 113 // authenticationConfigFunc is a function that returns a string representation of an authentication config. 114 type authenticationConfigFunc func(t *testing.T, issuerURL, caCert string) string 115 116 type apiServerOIDCConfig struct { 117 oidcURL string 118 oidcClientID string 119 oidcCAFilePath string 120 oidcUsernamePrefix string 121 oidcUsernameClaim string 122 authenticationConfigYAML string 123 } 124 125 func TestOIDC(t *testing.T) { 126 t.Log("Testing OIDC authenticator with --oidc-* flags") 127 runTests(t, false) 128 } 129 130 func TestStructuredAuthenticationConfig(t *testing.T) { 131 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StructuredAuthenticationConfiguration, true) 132 133 t.Log("Testing OIDC authenticator with authentication config") 134 runTests(t, true) 135 } 136 137 func runTests(t *testing.T, useAuthenticationConfig bool) { 138 var tests = []singleTest[*rsa.PrivateKey, *rsa.PublicKey]{ 139 { 140 name: "ID token is ok", 141 configureInfrastructure: func(t *testing.T, fn authenticationConfigFunc, keyFunc func(t *testing.T) (*rsa.PrivateKey, *rsa.PublicKey)) ( 142 oidcServer *utilsoidc.TestServer, 143 apiServer *kubeapiserverapptesting.TestServer, 144 signingPrivateKey *rsa.PrivateKey, 145 caCertContent []byte, 146 caFilePath string, 147 ) { 148 caCertContent, _, caFilePath, caKeyFilePath := generateCert(t) 149 signingPrivateKey, publicKey := keyFunc(t) 150 oidcServer = utilsoidc.BuildAndRunTestServer(t, caFilePath, caKeyFilePath, "") 151 152 if useAuthenticationConfig { 153 authenticationConfig := fmt.Sprintf(` 154 apiVersion: apiserver.config.k8s.io/v1beta1 155 kind: AuthenticationConfiguration 156 jwt: 157 - issuer: 158 url: %s 159 audiences: 160 - %s 161 certificateAuthority: | 162 %s 163 claimMappings: 164 username: 165 claim: user 166 prefix: %s 167 `, oidcServer.URL(), defaultOIDCClientID, indentCertificateAuthority(string(caCertContent)), defaultOIDCUsernamePrefix) 168 apiServer = startTestAPIServerForOIDC(t, apiServerOIDCConfig{authenticationConfigYAML: authenticationConfig}, &signingPrivateKey.PublicKey) 169 } else { 170 apiServer = startTestAPIServerForOIDC(t, apiServerOIDCConfig{oidcURL: oidcServer.URL(), oidcClientID: defaultOIDCClientID, 171 oidcCAFilePath: caFilePath, oidcUsernamePrefix: defaultOIDCUsernamePrefix, oidcUsernameClaim: "user"}, &signingPrivateKey.PublicKey) 172 } 173 oidcServer.JwksHandler().EXPECT().KeySet().AnyTimes().DoAndReturn(utilsoidc.DefaultJwksHandlerBehavior(t, publicKey)) 174 175 adminClient := kubernetes.NewForConfigOrDie(apiServer.ClientConfig) 176 configureRBAC(t, adminClient, defaultRole, defaultRoleBinding) 177 178 return oidcServer, apiServer, signingPrivateKey, caCertContent, caFilePath 179 }, configureOIDCServerBehaviour: func(t *testing.T, oidcServer *utilsoidc.TestServer, signingPrivateKey *rsa.PrivateKey) { 180 idTokenLifetime := time.Second * 1200 181 oidcServer.TokenHandler().EXPECT().Token().Times(1).DoAndReturn(utilsoidc.TokenHandlerBehaviorReturningPredefinedJWT( 182 t, 183 signingPrivateKey, 184 // This asserts the minimum valid claims for an ID token required by the authenticator. 185 // "iss", "aud", "exp" and a claim for the username. 186 map[string]interface{}{ 187 "iss": oidcServer.URL(), 188 "user": defaultOIDCClaimedUsername, 189 "aud": defaultOIDCClientID, 190 "exp": time.Now().Add(idTokenLifetime).Unix(), 191 }, 192 defaultStubAccessToken, 193 defaultStubRefreshToken, 194 )) 195 }, 196 configureClient: configureClientFetchingOIDCCredentials, 197 assertErrFn: func(t *testing.T, errorToCheck error) { 198 assert.NoError(t, errorToCheck) 199 }, 200 }, 201 { 202 name: "ID token is expired", 203 configureInfrastructure: configureTestInfrastructure[*rsa.PrivateKey, *rsa.PublicKey], 204 configureOIDCServerBehaviour: func(t *testing.T, oidcServer *utilsoidc.TestServer, signingPrivateKey *rsa.PrivateKey) { 205 configureOIDCServerToReturnExpiredIDToken(t, 2, oidcServer, signingPrivateKey) 206 }, 207 configureClient: configureClientFetchingOIDCCredentials, 208 assertErrFn: func(t *testing.T, errorToCheck error) { 209 assert.True(t, apierrors.IsUnauthorized(errorToCheck), errorToCheck) 210 }, 211 }, 212 { 213 name: "wrong client ID", 214 configureInfrastructure: configureTestInfrastructure[*rsa.PrivateKey, *rsa.PublicKey], 215 configureOIDCServerBehaviour: func(t *testing.T, oidcServer *utilsoidc.TestServer, _ *rsa.PrivateKey) { 216 oidcServer.TokenHandler().EXPECT().Token().Times(2).Return(utilsoidc.Token{}, utilsoidc.ErrBadClientID) 217 }, 218 configureClient: configureClientWithEmptyIDToken, 219 assertErrFn: func(t *testing.T, errorToCheck error) { 220 urlError, ok := errorToCheck.(*url.Error) 221 require.True(t, ok) 222 assert.Equal( 223 t, 224 "failed to refresh token: oauth2: cannot fetch token: 400 Bad Request\nResponse: client ID is bad\n", 225 urlError.Err.Error(), 226 ) 227 }, 228 }, 229 { 230 name: "client has wrong CA", 231 configureInfrastructure: configureTestInfrastructure[*rsa.PrivateKey, *rsa.PublicKey], 232 configureOIDCServerBehaviour: func(t *testing.T, oidcServer *utilsoidc.TestServer, _ *rsa.PrivateKey) {}, 233 configureClient: func(t *testing.T, restCfg *rest.Config, caCert []byte, _, oidcServerURL, oidcServerTokenURL string) kubernetes.Interface { 234 tempDir := t.TempDir() 235 certFilePath := filepath.Join(tempDir, "localhost_127.0.0.1_.crt") 236 237 _, _, wantErr := certutil.GenerateSelfSignedCertKeyWithFixtures("localhost", []net.IP{utilsnet.ParseIPSloppy("127.0.0.1")}, nil, tempDir) 238 require.NoError(t, wantErr) 239 240 return configureClientWithEmptyIDToken(t, restCfg, caCert, certFilePath, oidcServerURL, oidcServerTokenURL) 241 }, 242 assertErrFn: func(t *testing.T, errorToCheck error) { 243 expectedErr := new(x509.UnknownAuthorityError) 244 assert.ErrorAs(t, errorToCheck, expectedErr) 245 }, 246 }, 247 { 248 name: "refresh flow does not return ID Token", 249 configureInfrastructure: configureTestInfrastructure[*rsa.PrivateKey, *rsa.PublicKey], 250 configureOIDCServerBehaviour: func(t *testing.T, oidcServer *utilsoidc.TestServer, signingPrivateKey *rsa.PrivateKey) { 251 configureOIDCServerToReturnExpiredIDToken(t, 1, oidcServer, signingPrivateKey) 252 oidcServer.TokenHandler().EXPECT().Token().Times(1).Return(utilsoidc.Token{ 253 IDToken: "", 254 AccessToken: defaultStubAccessToken, 255 RefreshToken: defaultStubRefreshToken, 256 ExpiresIn: time.Now().Add(time.Second * 1200).Unix(), 257 }, nil) 258 }, 259 configureClient: configureClientFetchingOIDCCredentials, 260 assertErrFn: func(t *testing.T, errorToCheck error) { 261 expectedError := new(apierrors.StatusError) 262 assert.ErrorAs(t, errorToCheck, &expectedError) 263 assert.Equal( 264 t, 265 `pods is forbidden: User "system:anonymous" cannot list resource "pods" in API group "" in the namespace "default"`, 266 errorToCheck.Error(), 267 ) 268 }, 269 }, 270 { 271 name: "ID token signature can not be verified due to wrong JWKs", 272 configureInfrastructure: func(t *testing.T, fn authenticationConfigFunc, keyFunc func(t *testing.T) (*rsa.PrivateKey, *rsa.PublicKey)) ( 273 oidcServer *utilsoidc.TestServer, 274 apiServer *kubeapiserverapptesting.TestServer, 275 signingPrivateKey *rsa.PrivateKey, 276 caCertContent []byte, 277 caFilePath string, 278 ) { 279 caCertContent, _, caFilePath, caKeyFilePath := generateCert(t) 280 281 signingPrivateKey, _ = keyFunc(t) 282 283 oidcServer = utilsoidc.BuildAndRunTestServer(t, caFilePath, caKeyFilePath, "") 284 285 if useAuthenticationConfig { 286 authenticationConfig := fmt.Sprintf(` 287 apiVersion: apiserver.config.k8s.io/v1alpha1 288 kind: AuthenticationConfiguration 289 jwt: 290 - issuer: 291 url: %s 292 audiences: 293 - %s 294 certificateAuthority: | 295 %s 296 claimMappings: 297 username: 298 claim: sub 299 prefix: %s 300 `, oidcServer.URL(), defaultOIDCClientID, indentCertificateAuthority(string(caCertContent)), defaultOIDCUsernamePrefix) 301 apiServer = startTestAPIServerForOIDC(t, apiServerOIDCConfig{authenticationConfigYAML: authenticationConfig}, &signingPrivateKey.PublicKey) 302 } else { 303 apiServer = startTestAPIServerForOIDC(t, apiServerOIDCConfig{oidcURL: oidcServer.URL(), oidcClientID: defaultOIDCClientID, oidcCAFilePath: caFilePath, oidcUsernamePrefix: defaultOIDCUsernamePrefix}, &signingPrivateKey.PublicKey) 304 } 305 306 adminClient := kubernetes.NewForConfigOrDie(apiServer.ClientConfig) 307 configureRBAC(t, adminClient, defaultRole, defaultRoleBinding) 308 309 anotherSigningPrivateKey, _ := keyFunc(t) 310 311 oidcServer.JwksHandler().EXPECT().KeySet().AnyTimes().DoAndReturn(utilsoidc.DefaultJwksHandlerBehavior(t, &anotherSigningPrivateKey.PublicKey)) 312 313 return oidcServer, apiServer, signingPrivateKey, caCertContent, caFilePath 314 }, 315 configureOIDCServerBehaviour: func(t *testing.T, oidcServer *utilsoidc.TestServer, signingPrivateKey *rsa.PrivateKey) { 316 oidcServer.TokenHandler().EXPECT().Token().Times(1).DoAndReturn(utilsoidc.TokenHandlerBehaviorReturningPredefinedJWT( 317 t, 318 signingPrivateKey, 319 map[string]interface{}{ 320 "iss": oidcServer.URL(), 321 "sub": defaultOIDCClaimedUsername, 322 "aud": defaultOIDCClientID, 323 "exp": time.Now().Add(time.Second * 1200).Unix(), 324 }, 325 defaultStubAccessToken, 326 defaultStubRefreshToken, 327 )) 328 }, 329 configureClient: configureClientFetchingOIDCCredentials, 330 assertErrFn: func(t *testing.T, errorToCheck error) { 331 assert.True(t, apierrors.IsUnauthorized(errorToCheck), errorToCheck) 332 }, 333 }, 334 { 335 name: "ID token is okay but username is empty", 336 configureInfrastructure: func(t *testing.T, fn authenticationConfigFunc, keyFunc func(t *testing.T) (*rsa.PrivateKey, *rsa.PublicKey)) ( 337 oidcServer *utilsoidc.TestServer, 338 apiServer *kubeapiserverapptesting.TestServer, 339 signingPrivateKey *rsa.PrivateKey, 340 caCertContent []byte, 341 caFilePath string, 342 ) { 343 caCertContent, _, caFilePath, caKeyFilePath := generateCert(t) 344 345 signingPrivateKey, _ = keyFunc(t) 346 347 oidcServer = utilsoidc.BuildAndRunTestServer(t, caFilePath, caKeyFilePath, "") 348 349 if useAuthenticationConfig { 350 authenticationConfig := fmt.Sprintf(` 351 apiVersion: apiserver.config.k8s.io/v1alpha1 352 kind: AuthenticationConfiguration 353 jwt: 354 - issuer: 355 url: %s 356 audiences: 357 - %s 358 certificateAuthority: | 359 %s 360 claimMappings: 361 username: 362 expression: claims.sub 363 `, oidcServer.URL(), defaultOIDCClientID, indentCertificateAuthority(string(caCertContent))) 364 apiServer = startTestAPIServerForOIDC(t, apiServerOIDCConfig{authenticationConfigYAML: authenticationConfig}, &signingPrivateKey.PublicKey) 365 } else { 366 apiServer = startTestAPIServerForOIDC(t, apiServerOIDCConfig{ 367 oidcURL: oidcServer.URL(), oidcClientID: defaultOIDCClientID, oidcCAFilePath: caFilePath, oidcUsernamePrefix: "-", 368 }, 369 &signingPrivateKey.PublicKey) 370 } 371 372 oidcServer.JwksHandler().EXPECT().KeySet().AnyTimes().DoAndReturn(utilsoidc.DefaultJwksHandlerBehavior(t, &signingPrivateKey.PublicKey)) 373 374 return oidcServer, apiServer, signingPrivateKey, caCertContent, caFilePath 375 }, 376 configureOIDCServerBehaviour: func(t *testing.T, oidcServer *utilsoidc.TestServer, signingPrivateKey *rsa.PrivateKey) { 377 oidcServer.TokenHandler().EXPECT().Token().Times(1).DoAndReturn(utilsoidc.TokenHandlerBehaviorReturningPredefinedJWT( 378 t, 379 signingPrivateKey, 380 map[string]interface{}{ 381 "iss": oidcServer.URL(), 382 "sub": "", 383 "aud": defaultOIDCClientID, 384 "exp": time.Now().Add(time.Second * 1200).Unix(), 385 }, 386 defaultStubAccessToken, 387 defaultStubRefreshToken, 388 )) 389 }, 390 configureClient: configureClientFetchingOIDCCredentials, 391 assertErrFn: func(t *testing.T, errorToCheck error) { 392 if useAuthenticationConfig { // since the config uses a CEL expression 393 assert.True(t, apierrors.IsUnauthorized(errorToCheck), errorToCheck) 394 } else { 395 // the claim based approach is still allowed to use empty usernames 396 _ = assert.True(t, apierrors.IsForbidden(errorToCheck), errorToCheck) && 397 assert.Equal( 398 t, 399 `pods is forbidden: User "" cannot list resource "pods" in API group "" in the namespace "default"`, 400 errorToCheck.Error(), 401 ) 402 } 403 }, 404 }, 405 } 406 407 for _, tt := range tests { 408 t.Run(tt.name, singleTestRunner(useAuthenticationConfig, rsaGenerateKey, tt)) 409 } 410 411 for _, tt := range []singleTest[*ecdsa.PrivateKey, *ecdsa.PublicKey]{ 412 { 413 name: "ID token is ok", 414 configureInfrastructure: configureTestInfrastructure[*ecdsa.PrivateKey, *ecdsa.PublicKey], 415 configureOIDCServerBehaviour: func(t *testing.T, oidcServer *utilsoidc.TestServer, signingPrivateKey *ecdsa.PrivateKey) { 416 idTokenLifetime := time.Second * 1200 417 oidcServer.TokenHandler().EXPECT().Token().Times(1).DoAndReturn(utilsoidc.TokenHandlerBehaviorReturningPredefinedJWT( 418 t, 419 signingPrivateKey, 420 map[string]interface{}{ 421 "iss": oidcServer.URL(), 422 "sub": defaultOIDCClaimedUsername, 423 "aud": defaultOIDCClientID, 424 "exp": time.Now().Add(idTokenLifetime).Unix(), 425 }, 426 defaultStubAccessToken, 427 defaultStubRefreshToken, 428 )) 429 }, 430 configureClient: configureClientFetchingOIDCCredentials, 431 assertErrFn: func(t *testing.T, errorToCheck error) { 432 assert.NoError(t, errorToCheck) 433 }, 434 }, 435 } { 436 t.Run(tt.name, singleTestRunner(useAuthenticationConfig, ecdsaGenerateKey, tt)) 437 } 438 } 439 440 type singleTest[K utilsoidc.JosePrivateKey, L utilsoidc.JosePublicKey] struct { 441 name string 442 configureInfrastructure func(t *testing.T, fn authenticationConfigFunc, keyFunc func(t *testing.T) (K, L)) ( 443 oidcServer *utilsoidc.TestServer, 444 apiServer *kubeapiserverapptesting.TestServer, 445 signingPrivateKey K, 446 caCertContent []byte, 447 caFilePath string, 448 ) 449 configureOIDCServerBehaviour func(t *testing.T, oidcServer *utilsoidc.TestServer, signingPrivateKey K) 450 configureClient func( 451 t *testing.T, 452 restCfg *rest.Config, 453 caCert []byte, 454 certPath, 455 oidcServerURL, 456 oidcServerTokenURL string, 457 ) kubernetes.Interface 458 assertErrFn func(t *testing.T, errorToCheck error) 459 } 460 461 func singleTestRunner[K utilsoidc.JosePrivateKey, L utilsoidc.JosePublicKey]( 462 useAuthenticationConfig bool, 463 keyFunc func(t *testing.T) (K, L), 464 tt singleTest[K, L], 465 ) func(t *testing.T) { 466 return func(t *testing.T) { 467 fn := func(t *testing.T, issuerURL, caCert string) string { return "" } 468 if useAuthenticationConfig { 469 fn = func(t *testing.T, issuerURL, caCert string) string { 470 return fmt.Sprintf(` 471 apiVersion: apiserver.config.k8s.io/v1alpha1 472 kind: AuthenticationConfiguration 473 jwt: 474 - issuer: 475 url: %s 476 audiences: 477 - %s 478 certificateAuthority: | 479 %s 480 claimMappings: 481 username: 482 claim: sub 483 prefix: %s 484 `, issuerURL, defaultOIDCClientID, indentCertificateAuthority(caCert), defaultOIDCUsernamePrefix) 485 } 486 } 487 oidcServer, apiServer, signingPrivateKey, caCert, certPath := tt.configureInfrastructure(t, fn, keyFunc) 488 489 tt.configureOIDCServerBehaviour(t, oidcServer, signingPrivateKey) 490 491 tokenURL, err := oidcServer.TokenURL() 492 require.NoError(t, err) 493 494 client := tt.configureClient(t, apiServer.ClientConfig, caCert, certPath, oidcServer.URL(), tokenURL) 495 496 ctx := testContext(t) 497 _, err = client.CoreV1().Pods(defaultNamespace).List(ctx, metav1.ListOptions{}) 498 499 tt.assertErrFn(t, err) 500 } 501 } 502 503 func TestUpdatingRefreshTokenInCaseOfExpiredIDToken(t *testing.T) { 504 type testRun[K utilsoidc.JosePrivateKey] struct { 505 name string 506 configureUpdatingTokenBehaviour func(t *testing.T, oidcServer *utilsoidc.TestServer, signingPrivateKey K) 507 assertErrFn func(t *testing.T, errorToCheck error) 508 } 509 510 var tests = []testRun[*rsa.PrivateKey]{ 511 { 512 name: "cache returns stale client if refresh token is not updated in config", 513 configureUpdatingTokenBehaviour: func(t *testing.T, oidcServer *utilsoidc.TestServer, signingPrivateKey *rsa.PrivateKey) { 514 oidcServer.TokenHandler().EXPECT().Token().Times(1).DoAndReturn(utilsoidc.TokenHandlerBehaviorReturningPredefinedJWT( 515 t, 516 signingPrivateKey, 517 map[string]interface{}{ 518 "iss": oidcServer.URL(), 519 "sub": defaultOIDCClaimedUsername, 520 "aud": defaultOIDCClientID, 521 "exp": time.Now().Add(time.Second * 1200).Unix(), 522 }, 523 defaultStubAccessToken, 524 defaultStubRefreshToken, 525 )) 526 configureOIDCServerToReturnExpiredRefreshTokenErrorOnTryingToUpdateIDToken(oidcServer) 527 }, 528 assertErrFn: func(t *testing.T, errorToCheck error) { 529 urlError, ok := errorToCheck.(*url.Error) 530 require.True(t, ok) 531 assert.Equal( 532 t, 533 "failed to refresh token: oauth2: cannot fetch token: 400 Bad Request\nResponse: refresh token is expired\n", 534 urlError.Err.Error(), 535 ) 536 }, 537 }, 538 } 539 540 oidcServer, apiServer, signingPrivateKey, caCert, certPath := configureTestInfrastructure(t, func(t *testing.T, _, _ string) string { return "" }, rsaGenerateKey) 541 542 tokenURL, err := oidcServer.TokenURL() 543 require.NoError(t, err) 544 545 for _, tt := range tests { 546 t.Run(tt.name, func(t *testing.T) { 547 expiredIDToken, stubRefreshToken := fetchExpiredToken(t, oidcServer, caCert, signingPrivateKey) 548 clientConfig := configureClientConfigForOIDC(t, apiServer.ClientConfig, defaultOIDCClientID, certPath, expiredIDToken, stubRefreshToken, oidcServer.URL()) 549 expiredClient := kubernetes.NewForConfigOrDie(clientConfig) 550 configureOIDCServerToReturnExpiredRefreshTokenErrorOnTryingToUpdateIDToken(oidcServer) 551 552 ctx := testContext(t) 553 _, err = expiredClient.CoreV1().Pods(defaultNamespace).List(ctx, metav1.ListOptions{}) 554 assert.Error(t, err) 555 556 tt.configureUpdatingTokenBehaviour(t, oidcServer, signingPrivateKey) 557 idToken, stubRefreshToken := fetchOIDCCredentials(t, tokenURL, caCert) 558 clientConfig = configureClientConfigForOIDC(t, apiServer.ClientConfig, defaultOIDCClientID, certPath, idToken, stubRefreshToken, oidcServer.URL()) 559 expectedOkClient := kubernetes.NewForConfigOrDie(clientConfig) 560 _, err = expectedOkClient.CoreV1().Pods(defaultNamespace).List(ctx, metav1.ListOptions{}) 561 562 tt.assertErrFn(t, err) 563 }) 564 } 565 } 566 567 func TestStructuredAuthenticationConfigCEL(t *testing.T) { 568 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StructuredAuthenticationConfiguration, true) 569 570 type testRun[K utilsoidc.JosePrivateKey, L utilsoidc.JosePublicKey] struct { 571 name string 572 authConfigFn authenticationConfigFunc 573 configureInfrastructure func(t *testing.T, fn authenticationConfigFunc, keyFunc func(t *testing.T) (K, L)) ( 574 oidcServer *utilsoidc.TestServer, 575 apiServer *kubeapiserverapptesting.TestServer, 576 signingPrivateKey *rsa.PrivateKey, 577 caCertContent []byte, 578 caFilePath string, 579 ) 580 configureOIDCServerBehaviour func(t *testing.T, oidcServer *utilsoidc.TestServer, signingPrivateKey K) 581 configureClient func( 582 t *testing.T, 583 restCfg *rest.Config, 584 caCert []byte, 585 certPath, 586 oidcServerURL, 587 oidcServerTokenURL string, 588 ) kubernetes.Interface 589 assertErrFn func(t *testing.T, errorToCheck error) 590 wantUser *authenticationv1.UserInfo 591 } 592 593 tests := []testRun[*rsa.PrivateKey, *rsa.PublicKey]{ 594 { 595 name: "username CEL expression is ok", 596 authConfigFn: func(t *testing.T, issuerURL, caCert string) string { 597 return fmt.Sprintf(` 598 apiVersion: apiserver.config.k8s.io/v1alpha1 599 kind: AuthenticationConfiguration 600 jwt: 601 - issuer: 602 url: %s 603 audiences: 604 - %s 605 - another-audience 606 audienceMatchPolicy: MatchAny 607 certificateAuthority: | 608 %s 609 claimMappings: 610 username: 611 expression: "'k8s-' + claims.sub" 612 `, issuerURL, defaultOIDCClientID, indentCertificateAuthority(caCert)) 613 }, 614 configureInfrastructure: configureTestInfrastructure[*rsa.PrivateKey, *rsa.PublicKey], 615 configureOIDCServerBehaviour: func(t *testing.T, oidcServer *utilsoidc.TestServer, signingPrivateKey *rsa.PrivateKey) { 616 idTokenLifetime := time.Second * 1200 617 oidcServer.TokenHandler().EXPECT().Token().Times(1).DoAndReturn(utilsoidc.TokenHandlerBehaviorReturningPredefinedJWT( 618 t, 619 signingPrivateKey, 620 map[string]interface{}{ 621 "iss": oidcServer.URL(), 622 "sub": defaultOIDCClaimedUsername, 623 "aud": defaultOIDCClientID, 624 "exp": time.Now().Add(idTokenLifetime).Unix(), 625 }, 626 defaultStubAccessToken, 627 defaultStubRefreshToken, 628 )) 629 }, 630 configureClient: configureClientFetchingOIDCCredentials, 631 assertErrFn: func(t *testing.T, errorToCheck error) { 632 assert.NoError(t, errorToCheck) 633 }, 634 wantUser: &authenticationv1.UserInfo{ 635 Username: "k8s-john_doe", 636 Groups: []string{"system:authenticated"}, 637 }, 638 }, 639 { 640 name: "groups CEL expression is ok", 641 authConfigFn: func(t *testing.T, issuerURL, caCert string) string { 642 return fmt.Sprintf(` 643 apiVersion: apiserver.config.k8s.io/v1alpha1 644 kind: AuthenticationConfiguration 645 jwt: 646 - issuer: 647 url: %s 648 audiences: 649 - %s 650 - another-audience 651 audienceMatchPolicy: MatchAny 652 certificateAuthority: | 653 %s 654 claimMappings: 655 username: 656 expression: "'k8s-' + claims.sub" 657 groups: 658 expression: '(claims.roles.split(",") + claims.other_roles.split(",")).map(role, "prefix:" + role)' 659 `, issuerURL, defaultOIDCClientID, indentCertificateAuthority(caCert)) 660 }, 661 configureInfrastructure: configureTestInfrastructure[*rsa.PrivateKey, *rsa.PublicKey], 662 configureOIDCServerBehaviour: func(t *testing.T, oidcServer *utilsoidc.TestServer, signingPrivateKey *rsa.PrivateKey) { 663 idTokenLifetime := time.Second * 1200 664 oidcServer.TokenHandler().EXPECT().Token().Times(1).DoAndReturn(utilsoidc.TokenHandlerBehaviorReturningPredefinedJWT( 665 t, 666 signingPrivateKey, 667 map[string]interface{}{ 668 "iss": oidcServer.URL(), 669 "sub": defaultOIDCClaimedUsername, 670 "aud": defaultOIDCClientID, 671 "exp": time.Now().Add(idTokenLifetime).Unix(), 672 "roles": "foo,bar", 673 "other_roles": "baz,qux", 674 }, 675 defaultStubAccessToken, 676 defaultStubRefreshToken, 677 )) 678 }, 679 configureClient: configureClientFetchingOIDCCredentials, 680 assertErrFn: func(t *testing.T, errorToCheck error) { 681 assert.NoError(t, errorToCheck) 682 }, 683 wantUser: &authenticationv1.UserInfo{ 684 Username: "k8s-john_doe", 685 Groups: []string{"prefix:foo", "prefix:bar", "prefix:baz", "prefix:qux", "system:authenticated"}, 686 }, 687 }, 688 { 689 name: "claim validation rule fails", 690 authConfigFn: func(t *testing.T, issuerURL, caCert string) string { 691 return fmt.Sprintf(` 692 apiVersion: apiserver.config.k8s.io/v1alpha1 693 kind: AuthenticationConfiguration 694 jwt: 695 - issuer: 696 url: %s 697 audiences: 698 - %s 699 - another-audience 700 audienceMatchPolicy: MatchAny 701 certificateAuthority: | 702 %s 703 claimMappings: 704 username: 705 expression: "'k8s-' + claims.sub" 706 claimValidationRules: 707 - expression: 'claims.hd == "example.com"' 708 message: "the hd claim must be set to example.com" 709 `, issuerURL, defaultOIDCClientID, indentCertificateAuthority(caCert)) 710 }, 711 configureInfrastructure: configureTestInfrastructure[*rsa.PrivateKey, *rsa.PublicKey], 712 configureOIDCServerBehaviour: func(t *testing.T, oidcServer *utilsoidc.TestServer, signingPrivateKey *rsa.PrivateKey) { 713 idTokenLifetime := time.Second * 1200 714 oidcServer.TokenHandler().EXPECT().Token().Times(1).DoAndReturn(utilsoidc.TokenHandlerBehaviorReturningPredefinedJWT( 715 t, 716 signingPrivateKey, 717 map[string]interface{}{ 718 "iss": oidcServer.URL(), 719 "sub": defaultOIDCClaimedUsername, 720 "aud": defaultOIDCClientID, 721 "exp": time.Now().Add(idTokenLifetime).Unix(), 722 "hd": "notexample.com", 723 }, 724 defaultStubAccessToken, 725 defaultStubRefreshToken, 726 )) 727 }, 728 configureClient: configureClientFetchingOIDCCredentials, 729 assertErrFn: func(t *testing.T, errorToCheck error) { 730 assert.True(t, apierrors.IsUnauthorized(errorToCheck), errorToCheck) 731 }, 732 }, 733 { 734 name: "extra mapping CEL expressions are ok", 735 authConfigFn: func(t *testing.T, issuerURL, caCert string) string { 736 return fmt.Sprintf(` 737 apiVersion: apiserver.config.k8s.io/v1alpha1 738 kind: AuthenticationConfiguration 739 jwt: 740 - issuer: 741 url: %s 742 audiences: 743 - %s 744 - another-audience 745 audienceMatchPolicy: MatchAny 746 certificateAuthority: | 747 %s 748 claimMappings: 749 username: 750 expression: "'k8s-' + claims.sub" 751 extra: 752 - key: "example.org/foo" 753 valueExpression: "'bar'" 754 - key: "example.org/baz" 755 valueExpression: "claims.baz" 756 userValidationRules: 757 - expression: "'bar' in user.extra['example.org/foo'] && 'qux' in user.extra['example.org/baz']" 758 message: "example.org/foo must be bar and example.org/baz must be qux" 759 `, issuerURL, defaultOIDCClientID, indentCertificateAuthority(caCert)) 760 }, 761 configureInfrastructure: configureTestInfrastructure[*rsa.PrivateKey, *rsa.PublicKey], 762 configureOIDCServerBehaviour: func(t *testing.T, oidcServer *utilsoidc.TestServer, signingPrivateKey *rsa.PrivateKey) { 763 idTokenLifetime := time.Second * 1200 764 oidcServer.TokenHandler().EXPECT().Token().Times(1).DoAndReturn(utilsoidc.TokenHandlerBehaviorReturningPredefinedJWT( 765 t, 766 signingPrivateKey, 767 map[string]interface{}{ 768 "iss": oidcServer.URL(), 769 "sub": defaultOIDCClaimedUsername, 770 "aud": defaultOIDCClientID, 771 "exp": time.Now().Add(idTokenLifetime).Unix(), 772 "baz": "qux", 773 }, 774 defaultStubAccessToken, 775 defaultStubRefreshToken, 776 )) 777 }, 778 configureClient: configureClientFetchingOIDCCredentials, 779 assertErrFn: func(t *testing.T, errorToCheck error) { 780 assert.NoError(t, errorToCheck) 781 }, 782 wantUser: &authenticationv1.UserInfo{ 783 Username: "k8s-john_doe", 784 Groups: []string{"system:authenticated"}, 785 Extra: map[string]authenticationv1.ExtraValue{ 786 "example.org/foo": {"bar"}, 787 "example.org/baz": {"qux"}, 788 }, 789 }, 790 }, 791 { 792 name: "uid CEL expression is ok", 793 authConfigFn: func(t *testing.T, issuerURL, caCert string) string { 794 return fmt.Sprintf(` 795 apiVersion: apiserver.config.k8s.io/v1alpha1 796 kind: AuthenticationConfiguration 797 jwt: 798 - issuer: 799 url: %s 800 audiences: 801 - %s 802 - another-audience 803 audienceMatchPolicy: MatchAny 804 certificateAuthority: | 805 %s 806 claimMappings: 807 username: 808 expression: "'k8s-' + claims.sub" 809 uid: 810 expression: "claims.uid" 811 `, issuerURL, defaultOIDCClientID, indentCertificateAuthority(caCert)) 812 }, 813 configureInfrastructure: configureTestInfrastructure[*rsa.PrivateKey, *rsa.PublicKey], 814 configureOIDCServerBehaviour: func(t *testing.T, oidcServer *utilsoidc.TestServer, signingPrivateKey *rsa.PrivateKey) { 815 idTokenLifetime := time.Second * 1200 816 oidcServer.TokenHandler().EXPECT().Token().Times(1).DoAndReturn(utilsoidc.TokenHandlerBehaviorReturningPredefinedJWT( 817 t, 818 signingPrivateKey, 819 map[string]interface{}{ 820 "iss": oidcServer.URL(), 821 "sub": defaultOIDCClaimedUsername, 822 "aud": defaultOIDCClientID, 823 "exp": time.Now().Add(idTokenLifetime).Unix(), 824 "uid": "1234", 825 }, 826 defaultStubAccessToken, 827 defaultStubRefreshToken, 828 )) 829 }, 830 configureClient: configureClientFetchingOIDCCredentials, 831 assertErrFn: func(t *testing.T, errorToCheck error) { 832 assert.NoError(t, errorToCheck) 833 }, 834 wantUser: &authenticationv1.UserInfo{ 835 Username: "k8s-john_doe", 836 Groups: []string{"system:authenticated"}, 837 UID: "1234", 838 }, 839 }, 840 { 841 name: "user validation rule fails", 842 authConfigFn: func(t *testing.T, issuerURL, caCert string) string { 843 return fmt.Sprintf(` 844 apiVersion: apiserver.config.k8s.io/v1alpha1 845 kind: AuthenticationConfiguration 846 jwt: 847 - issuer: 848 url: %s 849 audiences: 850 - %s 851 - another-audience 852 audienceMatchPolicy: MatchAny 853 certificateAuthority: | 854 %s 855 claimMappings: 856 username: 857 expression: "'k8s-' + claims.sub" 858 groups: 859 expression: '(claims.roles.split(",") + claims.other_roles.split(",")).map(role, "system:" + role)' 860 userValidationRules: 861 - expression: "user.groups.all(group, !group.startsWith('system:'))" 862 message: "groups cannot used reserved system: prefix" 863 `, issuerURL, defaultOIDCClientID, indentCertificateAuthority(caCert)) 864 }, 865 configureInfrastructure: configureTestInfrastructure[*rsa.PrivateKey, *rsa.PublicKey], 866 configureOIDCServerBehaviour: func(t *testing.T, oidcServer *utilsoidc.TestServer, signingPrivateKey *rsa.PrivateKey) { 867 idTokenLifetime := time.Second * 1200 868 oidcServer.TokenHandler().EXPECT().Token().Times(1).DoAndReturn(utilsoidc.TokenHandlerBehaviorReturningPredefinedJWT( 869 t, 870 signingPrivateKey, 871 map[string]interface{}{ 872 "iss": oidcServer.URL(), 873 "sub": defaultOIDCClaimedUsername, 874 "aud": defaultOIDCClientID, 875 "exp": time.Now().Add(idTokenLifetime).Unix(), 876 "roles": "foo,bar", 877 "other_roles": "baz,qux", 878 }, 879 defaultStubAccessToken, 880 defaultStubRefreshToken, 881 )) 882 }, 883 configureClient: configureClientFetchingOIDCCredentials, 884 assertErrFn: func(t *testing.T, errorToCheck error) { 885 assert.True(t, apierrors.IsUnauthorized(errorToCheck), errorToCheck) 886 }, 887 wantUser: nil, 888 }, 889 { 890 name: "multiple audiences check with claim validation rule is ok", 891 authConfigFn: func(t *testing.T, issuerURL, caCert string) string { 892 return fmt.Sprintf(` 893 apiVersion: apiserver.config.k8s.io/v1alpha1 894 kind: AuthenticationConfiguration 895 jwt: 896 - issuer: 897 url: %s 898 audiences: 899 - baz 900 - foo 901 audienceMatchPolicy: MatchAny 902 certificateAuthority: | 903 %s 904 claimMappings: 905 username: 906 expression: "'k8s-' + claims.sub" 907 uid: 908 expression: "claims.uid" 909 claimValidationRules: 910 - expression: 'sets.equivalent(claims.aud, ["bar", "foo", "baz"])' 911 message: 'aud claim must be exactly match list ["bar", "foo", "baz"]' 912 `, issuerURL, indentCertificateAuthority(caCert)) 913 }, 914 configureInfrastructure: configureTestInfrastructure[*rsa.PrivateKey, *rsa.PublicKey], 915 configureOIDCServerBehaviour: func(t *testing.T, oidcServer *utilsoidc.TestServer, signingPrivateKey *rsa.PrivateKey) { 916 idTokenLifetime := time.Second * 1200 917 oidcServer.TokenHandler().EXPECT().Token().Times(1).DoAndReturn(utilsoidc.TokenHandlerBehaviorReturningPredefinedJWT( 918 t, 919 signingPrivateKey, 920 map[string]interface{}{ 921 "iss": oidcServer.URL(), 922 "sub": defaultOIDCClaimedUsername, 923 "aud": []string{"foo", "bar", "baz"}, 924 "exp": time.Now().Add(idTokenLifetime).Unix(), 925 "uid": "1234", 926 }, 927 defaultStubAccessToken, 928 defaultStubRefreshToken, 929 )) 930 }, 931 configureClient: configureClientFetchingOIDCCredentials, 932 assertErrFn: func(t *testing.T, errorToCheck error) { 933 assert.NoError(t, errorToCheck) 934 }, 935 wantUser: &authenticationv1.UserInfo{ 936 Username: "k8s-john_doe", 937 Groups: []string{"system:authenticated"}, 938 UID: "1234", 939 }, 940 }, 941 } 942 943 for _, tt := range tests { 944 t.Run(tt.name, func(t *testing.T) { 945 oidcServer, apiServer, signingPrivateKey, caCert, certPath := tt.configureInfrastructure(t, tt.authConfigFn, rsaGenerateKey) 946 947 tt.configureOIDCServerBehaviour(t, oidcServer, signingPrivateKey) 948 949 tokenURL, err := oidcServer.TokenURL() 950 require.NoError(t, err) 951 952 client := tt.configureClient(t, apiServer.ClientConfig, caCert, certPath, oidcServer.URL(), tokenURL) 953 954 ctx := testContext(t) 955 956 if tt.wantUser != nil { 957 res, err := client.AuthenticationV1().SelfSubjectReviews().Create(ctx, &authenticationv1.SelfSubjectReview{}, metav1.CreateOptions{}) 958 require.NoError(t, err) 959 assert.Equal(t, *tt.wantUser, res.Status.UserInfo) 960 } 961 962 _, err = client.CoreV1().Pods(defaultNamespace).List(ctx, metav1.ListOptions{}) 963 tt.assertErrFn(t, err) 964 }) 965 } 966 } 967 968 func TestStructuredAuthenticationConfigReload(t *testing.T) { 969 genericapiserver.SetHostnameFuncForTests("testAPIServerID") 970 const hardCodedTokenCacheTTLAndPollInterval = 10 * time.Second 971 972 origUpdateAuthenticationConfigTimeout := options.UpdateAuthenticationConfigTimeout 973 t.Cleanup(func() { options.UpdateAuthenticationConfigTimeout = origUpdateAuthenticationConfigTimeout }) 974 options.UpdateAuthenticationConfigTimeout = 2 * hardCodedTokenCacheTTLAndPollInterval // needs to be large enough for polling to run multiple times 975 976 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StructuredAuthenticationConfiguration, true) 977 978 tests := []struct { 979 name string 980 authConfigFn, newAuthConfigFn authenticationConfigFunc 981 assertErrFn, newAssertErrFn func(t *testing.T, errorToCheck error) 982 wantUser, newWantUser *authenticationv1.UserInfo 983 ignoreTransitionErrFn func(error) bool 984 waitAfterConfigSwap bool 985 wantMetricStrings []string 986 }{ 987 { 988 name: "old valid config to new valid config", 989 authConfigFn: func(t *testing.T, issuerURL, caCert string) string { 990 return fmt.Sprintf(` 991 apiVersion: apiserver.config.k8s.io/v1alpha1 992 kind: AuthenticationConfiguration 993 jwt: 994 - issuer: 995 url: %s 996 audiences: 997 - %s 998 - another-audience 999 audienceMatchPolicy: MatchAny 1000 certificateAuthority: | 1001 %s 1002 claimMappings: 1003 username: 1004 expression: "'k8s-' + claims.sub" 1005 `, issuerURL, defaultOIDCClientID, indentCertificateAuthority(caCert)) 1006 }, 1007 newAuthConfigFn: func(t *testing.T, issuerURL, caCert string) string { 1008 return fmt.Sprintf(` 1009 apiVersion: apiserver.config.k8s.io/v1alpha1 1010 kind: AuthenticationConfiguration 1011 jwt: 1012 - issuer: 1013 url: %s 1014 audiences: 1015 - %s 1016 - another-audience 1017 audienceMatchPolicy: MatchAny 1018 certificateAuthority: | 1019 %s 1020 claimMappings: 1021 username: 1022 expression: "'panda-' + claims.sub" # this is the only new part of the config 1023 `, issuerURL, defaultOIDCClientID, indentCertificateAuthority(caCert)) 1024 }, 1025 assertErrFn: func(t *testing.T, errorToCheck error) { 1026 assert.NoError(t, errorToCheck) 1027 }, 1028 wantUser: &authenticationv1.UserInfo{ 1029 Username: "k8s-john_doe", 1030 Groups: []string{"system:authenticated"}, 1031 }, 1032 newAssertErrFn: func(t *testing.T, errorToCheck error) { 1033 _ = assert.True(t, apierrors.IsForbidden(errorToCheck)) && 1034 assert.Equal( 1035 t, 1036 `pods is forbidden: User "panda-john_doe" cannot list resource "pods" in API group "" in the namespace "default"`, 1037 errorToCheck.Error(), 1038 ) 1039 }, 1040 newWantUser: &authenticationv1.UserInfo{ 1041 Username: "panda-john_doe", 1042 Groups: []string{"system:authenticated"}, 1043 }, 1044 wantMetricStrings: []string{ 1045 `apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} FP`, 1046 `apiserver_authentication_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} 1`, 1047 }, 1048 }, 1049 { 1050 name: "old empty config to new valid config", 1051 authConfigFn: func(t *testing.T, _, _ string) string { 1052 return ` 1053 apiVersion: apiserver.config.k8s.io/v1alpha1 1054 kind: AuthenticationConfiguration 1055 ` 1056 }, 1057 newAuthConfigFn: func(t *testing.T, issuerURL, caCert string) string { 1058 return fmt.Sprintf(` 1059 apiVersion: apiserver.config.k8s.io/v1alpha1 1060 kind: AuthenticationConfiguration 1061 jwt: 1062 - issuer: 1063 url: %s 1064 audiences: 1065 - %s 1066 - another-audience 1067 audienceMatchPolicy: MatchAny 1068 certificateAuthority: | 1069 %s 1070 claimMappings: 1071 username: 1072 expression: "'snorlax-' + claims.sub" 1073 `, issuerURL, defaultOIDCClientID, indentCertificateAuthority(caCert)) 1074 }, 1075 assertErrFn: func(t *testing.T, errorToCheck error) { 1076 assert.True(t, apierrors.IsUnauthorized(errorToCheck)) 1077 }, 1078 wantUser: nil, 1079 ignoreTransitionErrFn: apierrors.IsUnauthorized, 1080 newAssertErrFn: func(t *testing.T, errorToCheck error) { 1081 _ = assert.True(t, apierrors.IsForbidden(errorToCheck)) && 1082 assert.Equal( 1083 t, 1084 `pods is forbidden: User "snorlax-john_doe" cannot list resource "pods" in API group "" in the namespace "default"`, 1085 errorToCheck.Error(), 1086 ) 1087 }, 1088 newWantUser: &authenticationv1.UserInfo{ 1089 Username: "snorlax-john_doe", 1090 Groups: []string{"system:authenticated"}, 1091 }, 1092 wantMetricStrings: []string{ 1093 `apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} FP`, 1094 `apiserver_authentication_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} 1`, 1095 }, 1096 }, 1097 { 1098 name: "old invalid config to new valid config", 1099 authConfigFn: func(t *testing.T, issuerURL, _ string) string { 1100 return fmt.Sprintf(` 1101 apiVersion: apiserver.config.k8s.io/v1alpha1 1102 kind: AuthenticationConfiguration 1103 jwt: 1104 - issuer: 1105 url: %s 1106 audiences: 1107 - %s 1108 - another-audience 1109 audienceMatchPolicy: MatchAny 1110 certificateAuthority: "" # missing CA 1111 claimMappings: 1112 username: 1113 expression: "'k8s-' + claims.sub" 1114 `, issuerURL, defaultOIDCClientID) 1115 }, 1116 newAuthConfigFn: func(t *testing.T, issuerURL, caCert string) string { 1117 return fmt.Sprintf(` 1118 apiVersion: apiserver.config.k8s.io/v1alpha1 1119 kind: AuthenticationConfiguration 1120 jwt: 1121 - issuer: 1122 url: %s 1123 audiences: 1124 - %s 1125 - another-audience 1126 audienceMatchPolicy: MatchAny 1127 # this is the only new part of the config 1128 certificateAuthority: | 1129 %s 1130 claimMappings: 1131 username: 1132 expression: "'k8s-' + claims.sub" 1133 `, issuerURL, defaultOIDCClientID, indentCertificateAuthority(caCert)) 1134 }, 1135 assertErrFn: func(t *testing.T, errorToCheck error) { 1136 assert.True(t, apierrors.IsUnauthorized(errorToCheck)) 1137 }, 1138 wantUser: nil, 1139 ignoreTransitionErrFn: apierrors.IsUnauthorized, 1140 newAssertErrFn: func(t *testing.T, errorToCheck error) { 1141 assert.NoError(t, errorToCheck) 1142 }, 1143 newWantUser: &authenticationv1.UserInfo{ 1144 Username: "k8s-john_doe", 1145 Groups: []string{"system:authenticated"}, 1146 }, 1147 wantMetricStrings: []string{ 1148 `apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} FP`, 1149 `apiserver_authentication_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} 1`, 1150 }, 1151 }, 1152 { 1153 name: "old valid config to new structurally invalid config (should be ignored)", 1154 authConfigFn: func(t *testing.T, issuerURL, caCert string) string { 1155 return fmt.Sprintf(` 1156 apiVersion: apiserver.config.k8s.io/v1alpha1 1157 kind: AuthenticationConfiguration 1158 jwt: 1159 - issuer: 1160 url: %s 1161 audiences: 1162 - %s 1163 - another-audience 1164 audienceMatchPolicy: MatchAny 1165 certificateAuthority: | 1166 %s 1167 claimMappings: 1168 username: 1169 expression: "'k8s-' + claims.sub" 1170 `, issuerURL, defaultOIDCClientID, indentCertificateAuthority(caCert)) 1171 }, 1172 newAuthConfigFn: func(t *testing.T, issuerURL, caCert string) string { 1173 return fmt.Sprintf(` 1174 apiVersion: apiserver.config.k8s.io/v1alpha1 1175 kind: AuthenticationConfiguration 1176 jwt: 1177 - issuer: 1178 url: %s 1179 audiences: 1180 - %s 1181 - another-audience 1182 audienceMatchPolicy: MatchAny 1183 certificateAuthority: | 1184 %s 1185 claimMappings: 1186 username: 1187 expression: "'k8s-' + claimss.sub" # has typo 1188 `, issuerURL, defaultOIDCClientID, indentCertificateAuthority(caCert)) 1189 }, 1190 assertErrFn: func(t *testing.T, errorToCheck error) { 1191 assert.NoError(t, errorToCheck) 1192 }, 1193 wantUser: &authenticationv1.UserInfo{ 1194 Username: "k8s-john_doe", 1195 Groups: []string{"system:authenticated"}, 1196 }, 1197 newAssertErrFn: func(t *testing.T, errorToCheck error) { 1198 assert.NoError(t, errorToCheck) 1199 }, 1200 newWantUser: &authenticationv1.UserInfo{ 1201 Username: "k8s-john_doe", 1202 Groups: []string{"system:authenticated"}, 1203 }, 1204 waitAfterConfigSwap: true, 1205 wantMetricStrings: []string{ 1206 `apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="failure"} FP`, 1207 `apiserver_authentication_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="failure"} 1`, 1208 }, 1209 }, 1210 { 1211 name: "old valid config to new valid empty config (should cause tokens to stop working)", 1212 authConfigFn: func(t *testing.T, issuerURL, caCert string) string { 1213 return fmt.Sprintf(` 1214 apiVersion: apiserver.config.k8s.io/v1alpha1 1215 kind: AuthenticationConfiguration 1216 jwt: 1217 - issuer: 1218 url: %s 1219 audiences: 1220 - %s 1221 - another-audience 1222 audienceMatchPolicy: MatchAny 1223 certificateAuthority: | 1224 %s 1225 claimMappings: 1226 username: 1227 expression: "'k8s-' + claims.sub" 1228 `, issuerURL, defaultOIDCClientID, indentCertificateAuthority(caCert)) 1229 }, 1230 newAuthConfigFn: func(t *testing.T, _, _ string) string { 1231 return ` 1232 apiVersion: apiserver.config.k8s.io/v1alpha1 1233 kind: AuthenticationConfiguration 1234 ` 1235 }, 1236 assertErrFn: func(t *testing.T, errorToCheck error) { 1237 assert.NoError(t, errorToCheck) 1238 }, 1239 wantUser: &authenticationv1.UserInfo{ 1240 Username: "k8s-john_doe", 1241 Groups: []string{"system:authenticated"}, 1242 }, 1243 newAssertErrFn: func(t *testing.T, errorToCheck error) { 1244 assert.True(t, apierrors.IsUnauthorized(errorToCheck)) 1245 }, 1246 newWantUser: nil, 1247 waitAfterConfigSwap: true, 1248 wantMetricStrings: []string{ 1249 `apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} FP`, 1250 `apiserver_authentication_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} 1`, 1251 }, 1252 }, 1253 { 1254 name: "old valid config to new valid config with typo (should be ignored)", 1255 authConfigFn: func(t *testing.T, issuerURL, caCert string) string { 1256 return fmt.Sprintf(` 1257 apiVersion: apiserver.config.k8s.io/v1alpha1 1258 kind: AuthenticationConfiguration 1259 jwt: 1260 - issuer: 1261 url: %s 1262 audiences: 1263 - %s 1264 - another-audience 1265 audienceMatchPolicy: MatchAny 1266 certificateAuthority: | 1267 %s 1268 claimMappings: 1269 username: 1270 expression: "'k8s-' + claims.sub" 1271 `, issuerURL, defaultOIDCClientID, indentCertificateAuthority(caCert)) 1272 }, 1273 newAuthConfigFn: func(t *testing.T, issuerURL, _ string) string { 1274 return fmt.Sprintf(` 1275 apiVersion: apiserver.config.k8s.io/v1alpha1 1276 kind: AuthenticationConfiguration 1277 jwt: 1278 - issuer: 1279 url: %s 1280 audiences: 1281 - %s 1282 - another-audience 1283 audienceMatchPolicy: MatchAny 1284 certificateAuthority: "" # missing CA 1285 claimMappings: 1286 username: 1287 expression: "'k8s-' + claims.sub" 1288 `, issuerURL, defaultOIDCClientID) 1289 }, 1290 assertErrFn: func(t *testing.T, errorToCheck error) { 1291 assert.NoError(t, errorToCheck) 1292 }, 1293 wantUser: &authenticationv1.UserInfo{ 1294 Username: "k8s-john_doe", 1295 Groups: []string{"system:authenticated"}, 1296 }, 1297 newAssertErrFn: func(t *testing.T, errorToCheck error) { 1298 assert.NoError(t, errorToCheck) 1299 }, 1300 newWantUser: &authenticationv1.UserInfo{ 1301 Username: "k8s-john_doe", 1302 Groups: []string{"system:authenticated"}, 1303 }, 1304 waitAfterConfigSwap: true, 1305 wantMetricStrings: []string{ 1306 `apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="failure"} FP`, 1307 `apiserver_authentication_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="failure"} 1`, 1308 }, 1309 }, 1310 } 1311 1312 for _, tt := range tests { 1313 t.Run(tt.name, func(t *testing.T) { 1314 authenticationconfigmetrics.ResetMetricsForTest() 1315 defer authenticationconfigmetrics.ResetMetricsForTest() 1316 1317 ctx := testContext(t) 1318 1319 oidcServer, apiServer, caCert, certPath := configureBasicTestInfrastructureWithRandomKeyType(t, tt.authConfigFn) 1320 1321 tokenURL, err := oidcServer.TokenURL() 1322 require.NoError(t, err) 1323 1324 client := configureClientFetchingOIDCCredentials(t, apiServer.ClientConfig, caCert, certPath, oidcServer.URL(), tokenURL) 1325 1326 if tt.wantUser != nil { 1327 res, err := client.AuthenticationV1().SelfSubjectReviews().Create(ctx, &authenticationv1.SelfSubjectReview{}, metav1.CreateOptions{}) 1328 require.NoError(t, err) 1329 assert.Equal(t, *tt.wantUser, res.Status.UserInfo) 1330 } 1331 1332 _, err = client.CoreV1().Pods(defaultNamespace).List(ctx, metav1.ListOptions{}) 1333 tt.assertErrFn(t, err) 1334 1335 // Create a temporary file 1336 tempFile, err := os.CreateTemp("", "tempfile") 1337 require.NoError(t, err) 1338 defer func() { 1339 _ = tempFile.Close() 1340 }() 1341 1342 // Write the new content to the temporary file 1343 _, err = tempFile.Write([]byte(tt.newAuthConfigFn(t, oidcServer.URL(), string(caCert)))) 1344 require.NoError(t, err) 1345 1346 // Atomically replace the original file with the temporary file 1347 err = os.Rename(tempFile.Name(), apiServer.ServerOpts.Authentication.AuthenticationConfigFile) 1348 require.NoError(t, err) 1349 1350 if tt.waitAfterConfigSwap { 1351 time.Sleep(options.UpdateAuthenticationConfigTimeout + hardCodedTokenCacheTTLAndPollInterval) // has to be longer than UpdateAuthenticationConfigTimeout 1352 } 1353 1354 if tt.newWantUser != nil { 1355 start := time.Now() 1356 err = wait.PollUntilContextTimeout(ctx, time.Second, 3*hardCodedTokenCacheTTLAndPollInterval, true, func(ctx context.Context) (done bool, err error) { 1357 res, err := client.AuthenticationV1().SelfSubjectReviews().Create(ctx, &authenticationv1.SelfSubjectReview{}, metav1.CreateOptions{}) 1358 if err != nil { 1359 if tt.ignoreTransitionErrFn != nil && tt.ignoreTransitionErrFn(err) { 1360 return false, nil 1361 } 1362 return false, err 1363 } 1364 1365 diff := cmp.Diff(*tt.newWantUser, res.Status.UserInfo) 1366 if len(diff) > 0 && time.Since(start) > 2*hardCodedTokenCacheTTLAndPollInterval { 1367 t.Logf("%s saw new user diff:\n%s", t.Name(), diff) 1368 } 1369 1370 return len(diff) == 0, nil 1371 }) 1372 require.NoError(t, err, "new authentication config not loaded") 1373 } 1374 1375 _, err = client.CoreV1().Pods(defaultNamespace).List(ctx, metav1.ListOptions{}) 1376 tt.newAssertErrFn(t, err) 1377 1378 adminClient := kubernetes.NewForConfigOrDie(apiServer.ClientConfig) 1379 body, err := adminClient.RESTClient().Get().AbsPath("/metrics").DoRaw(ctx) 1380 require.NoError(t, err) 1381 var gotMetricStrings []string 1382 trimFP := regexp.MustCompile(`(.*)(} \d+\.\d+.*)`) 1383 for _, line := range strings.Split(string(body), "\n") { 1384 if strings.HasPrefix(line, "apiserver_authentication_config_controller_") { 1385 if strings.Contains(line, "_seconds") { 1386 line = trimFP.ReplaceAllString(line, `$1`) + "} FP" // ignore floating point metric values 1387 } 1388 gotMetricStrings = append(gotMetricStrings, line) 1389 } 1390 } 1391 if diff := cmp.Diff(tt.wantMetricStrings, gotMetricStrings); diff != "" { 1392 t.Errorf("unexpected metrics diff (-want +got): %s", diff) 1393 } 1394 }) 1395 } 1396 } 1397 1398 func configureBasicTestInfrastructureWithRandomKeyType(t *testing.T, fn authenticationConfigFunc) ( 1399 oidcServer *utilsoidc.TestServer, 1400 apiServer *kubeapiserverapptesting.TestServer, 1401 caCertContent []byte, 1402 caFilePath string, 1403 ) { 1404 t.Helper() 1405 1406 if randomBool() { 1407 return configureBasicTestInfrastructure(t, fn, rsaGenerateKey) 1408 } 1409 1410 return configureBasicTestInfrastructure(t, fn, ecdsaGenerateKey) 1411 } 1412 1413 func configureBasicTestInfrastructure[K utilsoidc.JosePrivateKey, L utilsoidc.JosePublicKey](t *testing.T, fn authenticationConfigFunc, keyFunc func(t *testing.T) (K, L)) ( 1414 oidcServer *utilsoidc.TestServer, 1415 apiServer *kubeapiserverapptesting.TestServer, 1416 caCertContent []byte, 1417 caFilePath string, 1418 ) { 1419 t.Helper() 1420 1421 oidcServer, apiServer, signingPrivateKey, caCertContent, caFilePath := configureTestInfrastructure(t, fn, keyFunc) 1422 1423 oidcServer.TokenHandler().EXPECT().Token().Times(1).DoAndReturn(utilsoidc.TokenHandlerBehaviorReturningPredefinedJWT( 1424 t, 1425 signingPrivateKey, 1426 map[string]interface{}{ 1427 "iss": oidcServer.URL(), 1428 "sub": defaultOIDCClaimedUsername, 1429 "aud": defaultOIDCClientID, 1430 "exp": time.Now().Add(10 * time.Minute).Unix(), 1431 }, 1432 defaultStubAccessToken, 1433 defaultStubRefreshToken, 1434 )) 1435 1436 return oidcServer, apiServer, caCertContent, caFilePath 1437 } 1438 1439 // TestStructuredAuthenticationDiscoveryURL tests that the discovery URL configured in jwt.issuer.discoveryURL is used to 1440 // fetch the discovery document and the issuer in jwt.issuer.url is used to validate the ID token. 1441 func TestStructuredAuthenticationDiscoveryURL(t *testing.T) { 1442 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StructuredAuthenticationConfiguration, true) 1443 1444 tests := []struct { 1445 name string 1446 issuerURL string 1447 discoveryURL func(baseURL string) string 1448 }{ 1449 { 1450 name: "discovery url and issuer url with no path", 1451 issuerURL: "https://example.com", 1452 discoveryURL: func(baseURL string) string { return baseURL }, 1453 }, 1454 { 1455 name: "discovery url has path, issuer url has no path", 1456 issuerURL: "https://example.com", 1457 discoveryURL: func(baseURL string) string { return fmt.Sprintf("%s/c/d/bar", baseURL) }, 1458 }, 1459 { 1460 name: "discovery url has no path, issuer url has path", 1461 issuerURL: "https://example.com/a/b/foo", 1462 discoveryURL: func(baseURL string) string { return baseURL }, 1463 }, 1464 { 1465 name: "discovery url and issuer url have paths", 1466 issuerURL: "https://example.com/a/b/foo", 1467 discoveryURL: func(baseURL string) string { 1468 return fmt.Sprintf("%s/c/d/bar", baseURL) 1469 }, 1470 }, 1471 } 1472 1473 for _, tt := range tests { 1474 t.Run(tt.name, func(t *testing.T) { 1475 caCertContent, _, caFilePath, caKeyFilePath := generateCert(t) 1476 signingPrivateKey, publicKey := rsaGenerateKey(t) 1477 // set the issuer in the discovery document to issuer url (different from the discovery URL) to assert 1478 // 1. discovery URL is used to fetch the discovery document and 1479 // 2. issuer in the discovery document is used to validate the ID token 1480 oidcServer := utilsoidc.BuildAndRunTestServer(t, caFilePath, caKeyFilePath, tt.issuerURL) 1481 discoveryURL := strings.TrimSuffix(tt.discoveryURL(oidcServer.URL()), "/") + "/.well-known/openid-configuration" 1482 1483 authenticationConfig := fmt.Sprintf(` 1484 apiVersion: apiserver.config.k8s.io/v1alpha1 1485 kind: AuthenticationConfiguration 1486 jwt: 1487 - issuer: 1488 url: %s 1489 discoveryURL: %s 1490 audiences: 1491 - foo 1492 audienceMatchPolicy: MatchAny 1493 certificateAuthority: | 1494 %s 1495 claimMappings: 1496 username: 1497 expression: "'k8s-' + claims.sub" 1498 claimValidationRules: 1499 - expression: 'claims.hd == "example.com"' 1500 message: "the hd claim must be set to example.com" 1501 `, tt.issuerURL, discoveryURL, indentCertificateAuthority(string(caCertContent))) 1502 1503 oidcServer.JwksHandler().EXPECT().KeySet().AnyTimes().DoAndReturn(utilsoidc.DefaultJwksHandlerBehavior(t, publicKey)) 1504 1505 apiServer := startTestAPIServerForOIDC(t, apiServerOIDCConfig{authenticationConfigYAML: authenticationConfig}, publicKey) 1506 1507 idTokenLifetime := time.Second * 1200 1508 oidcServer.TokenHandler().EXPECT().Token().Times(1).DoAndReturn(utilsoidc.TokenHandlerBehaviorReturningPredefinedJWT( 1509 t, 1510 signingPrivateKey, 1511 map[string]interface{}{ 1512 "iss": tt.issuerURL, // issuer in the discovery document is used to validate the ID token 1513 "sub": defaultOIDCClaimedUsername, 1514 "aud": "foo", 1515 "exp": time.Now().Add(idTokenLifetime).Unix(), 1516 "hd": "example.com", 1517 }, 1518 defaultStubAccessToken, 1519 defaultStubRefreshToken, 1520 )) 1521 1522 tokenURL, err := oidcServer.TokenURL() 1523 require.NoError(t, err) 1524 1525 client := configureClientFetchingOIDCCredentials(t, apiServer.ClientConfig, caCertContent, caFilePath, oidcServer.URL(), tokenURL) 1526 ctx := testContext(t) 1527 res, err := client.AuthenticationV1().SelfSubjectReviews().Create(ctx, &authenticationv1.SelfSubjectReview{}, metav1.CreateOptions{}) 1528 require.NoError(t, err) 1529 assert.Equal(t, authenticationv1.UserInfo{ 1530 Username: "k8s-john_doe", 1531 Groups: []string{"system:authenticated"}, 1532 }, res.Status.UserInfo) 1533 }) 1534 } 1535 } 1536 1537 func TestMultipleJWTAuthenticators(t *testing.T) { 1538 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StructuredAuthenticationConfiguration, true) 1539 1540 caCertContent1, _, caFilePath1, caKeyFilePath1 := generateCert(t) 1541 signingPrivateKey1, publicKey1 := rsaGenerateKey(t) 1542 oidcServer1 := utilsoidc.BuildAndRunTestServer(t, caFilePath1, caKeyFilePath1, "") 1543 1544 caCertContent2, _, caFilePath2, caKeyFilePath2 := generateCert(t) 1545 signingPrivateKey2, publicKey2 := rsaGenerateKey(t) 1546 oidcServer2 := utilsoidc.BuildAndRunTestServer(t, caFilePath2, caKeyFilePath2, "https://example.com") 1547 1548 authenticationConfig := fmt.Sprintf(` 1549 apiVersion: apiserver.config.k8s.io/v1alpha1 1550 kind: AuthenticationConfiguration 1551 jwt: 1552 - issuer: 1553 url: %s 1554 audiences: 1555 - foo 1556 audienceMatchPolicy: MatchAny 1557 certificateAuthority: | 1558 %s 1559 claimMappings: 1560 username: 1561 expression: "'k8s-' + claims.sub" 1562 claimValidationRules: 1563 - expression: 'claims.hd == "example.com"' 1564 message: "the hd claim must be set to example.com" 1565 - issuer: 1566 url: "https://example.com" 1567 discoveryURL: %s/.well-known/openid-configuration 1568 audiences: 1569 - bar 1570 audienceMatchPolicy: MatchAny 1571 certificateAuthority: | 1572 %s 1573 claimMappings: 1574 username: 1575 expression: "'k8s-' + claims.sub" 1576 groups: 1577 expression: '(claims.roles.split(",") + claims.other_roles.split(",")).map(role, "system:" + role)' 1578 uid: 1579 expression: "claims.uid" 1580 `, oidcServer1.URL(), indentCertificateAuthority(string(caCertContent1)), oidcServer2.URL(), indentCertificateAuthority(string(caCertContent2))) 1581 1582 oidcServer1.JwksHandler().EXPECT().KeySet().AnyTimes().DoAndReturn(utilsoidc.DefaultJwksHandlerBehavior(t, publicKey1)) 1583 oidcServer2.JwksHandler().EXPECT().KeySet().AnyTimes().DoAndReturn(utilsoidc.DefaultJwksHandlerBehavior(t, publicKey2)) 1584 1585 apiServer := startTestAPIServerForOIDC(t, apiServerOIDCConfig{authenticationConfigYAML: authenticationConfig}, publicKey1) 1586 1587 idTokenLifetime := time.Second * 1200 1588 oidcServer1.TokenHandler().EXPECT().Token().Times(1).DoAndReturn(utilsoidc.TokenHandlerBehaviorReturningPredefinedJWT( 1589 t, 1590 signingPrivateKey1, 1591 map[string]interface{}{ 1592 "iss": oidcServer1.URL(), 1593 "sub": defaultOIDCClaimedUsername, 1594 "aud": "foo", 1595 "exp": time.Now().Add(idTokenLifetime).Unix(), 1596 "hd": "example.com", 1597 }, 1598 defaultStubAccessToken, 1599 defaultStubRefreshToken, 1600 )) 1601 1602 oidcServer2.TokenHandler().EXPECT().Token().Times(1).DoAndReturn(utilsoidc.TokenHandlerBehaviorReturningPredefinedJWT( 1603 t, 1604 signingPrivateKey2, 1605 map[string]interface{}{ 1606 "iss": "https://example.com", 1607 "sub": "not_john_doe", 1608 "aud": "bar", 1609 "roles": "role1,role2", 1610 "other_roles": "role3,role4", 1611 "exp": time.Now().Add(idTokenLifetime).Unix(), 1612 "uid": "1234", 1613 }, 1614 defaultStubAccessToken, 1615 defaultStubRefreshToken, 1616 )) 1617 1618 tokenURL1, err := oidcServer1.TokenURL() 1619 require.NoError(t, err) 1620 1621 tokenURL2, err := oidcServer2.TokenURL() 1622 require.NoError(t, err) 1623 1624 client1 := configureClientFetchingOIDCCredentials(t, apiServer.ClientConfig, caCertContent1, caFilePath1, oidcServer1.URL(), tokenURL1) 1625 client2 := configureClientFetchingOIDCCredentials(t, apiServer.ClientConfig, caCertContent2, caFilePath2, oidcServer2.URL(), tokenURL2) 1626 1627 ctx := testContext(t) 1628 res, err := client1.AuthenticationV1().SelfSubjectReviews().Create(ctx, &authenticationv1.SelfSubjectReview{}, metav1.CreateOptions{}) 1629 require.NoError(t, err) 1630 assert.Equal(t, authenticationv1.UserInfo{ 1631 Username: "k8s-john_doe", 1632 Groups: []string{"system:authenticated"}, 1633 }, res.Status.UserInfo) 1634 1635 res, err = client2.AuthenticationV1().SelfSubjectReviews().Create(ctx, &authenticationv1.SelfSubjectReview{}, metav1.CreateOptions{}) 1636 require.NoError(t, err) 1637 assert.Equal(t, authenticationv1.UserInfo{ 1638 Username: "k8s-not_john_doe", 1639 Groups: []string{"system:role1", "system:role2", "system:role3", "system:role4", "system:authenticated"}, 1640 UID: "1234", 1641 }, res.Status.UserInfo) 1642 } 1643 1644 func rsaGenerateKey(t *testing.T) (*rsa.PrivateKey, *rsa.PublicKey) { 1645 t.Helper() 1646 1647 privateKey, err := rsa.GenerateKey(rand.Reader, rsaKeyBitSize) 1648 require.NoError(t, err) 1649 1650 return privateKey, &privateKey.PublicKey 1651 } 1652 1653 func ecdsaGenerateKey(t *testing.T) (*ecdsa.PrivateKey, *ecdsa.PublicKey) { 1654 t.Helper() 1655 1656 privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) 1657 require.NoError(t, err) 1658 1659 return privateKey, &privateKey.PublicKey 1660 } 1661 1662 func configureTestInfrastructure[K utilsoidc.JosePrivateKey, L utilsoidc.JosePublicKey](t *testing.T, fn authenticationConfigFunc, keyFunc func(t *testing.T) (K, L)) ( 1663 oidcServer *utilsoidc.TestServer, 1664 apiServer *kubeapiserverapptesting.TestServer, 1665 signingPrivateKey K, 1666 caCertContent []byte, 1667 caFilePath string, 1668 ) { 1669 t.Helper() 1670 1671 caCertContent, _, caFilePath, caKeyFilePath := generateCert(t) 1672 1673 signingPrivateKey, publicKey := keyFunc(t) 1674 1675 oidcServer = utilsoidc.BuildAndRunTestServer(t, caFilePath, caKeyFilePath, "") 1676 1677 authenticationConfig := fn(t, oidcServer.URL(), string(caCertContent)) 1678 if len(authenticationConfig) > 0 { 1679 apiServer = startTestAPIServerForOIDC(t, apiServerOIDCConfig{authenticationConfigYAML: authenticationConfig}, publicKey) 1680 } else { 1681 apiServer = startTestAPIServerForOIDC(t, apiServerOIDCConfig{oidcURL: oidcServer.URL(), oidcClientID: defaultOIDCClientID, oidcCAFilePath: caFilePath, oidcUsernamePrefix: defaultOIDCUsernamePrefix}, publicKey) 1682 } 1683 1684 oidcServer.JwksHandler().EXPECT().KeySet().AnyTimes().DoAndReturn(utilsoidc.DefaultJwksHandlerBehavior(t, publicKey)) 1685 1686 adminClient := kubernetes.NewForConfigOrDie(apiServer.ClientConfig) 1687 configureRBAC(t, adminClient, defaultRole, defaultRoleBinding) 1688 1689 return oidcServer, apiServer, signingPrivateKey, caCertContent, caFilePath 1690 } 1691 1692 func configureClientFetchingOIDCCredentials(t *testing.T, restCfg *rest.Config, caCert []byte, certPath, oidcServerURL, oidcServerTokenURL string) kubernetes.Interface { 1693 idToken, stubRefreshToken := fetchOIDCCredentials(t, oidcServerTokenURL, caCert) 1694 clientConfig := configureClientConfigForOIDC(t, restCfg, defaultOIDCClientID, certPath, idToken, stubRefreshToken, oidcServerURL) 1695 return kubernetes.NewForConfigOrDie(clientConfig) 1696 } 1697 1698 func configureClientWithEmptyIDToken(t *testing.T, restCfg *rest.Config, _ []byte, certPath, oidcServerURL, _ string) kubernetes.Interface { 1699 emptyIDToken, stubRefreshToken := "", defaultStubRefreshToken 1700 clientConfig := configureClientConfigForOIDC(t, restCfg, defaultOIDCClientID, certPath, emptyIDToken, stubRefreshToken, oidcServerURL) 1701 return kubernetes.NewForConfigOrDie(clientConfig) 1702 } 1703 1704 func configureRBAC(t *testing.T, clientset kubernetes.Interface, role *rbacv1.Role, binding *rbacv1.RoleBinding) { 1705 t.Helper() 1706 1707 ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) 1708 defer cancel() 1709 1710 _, err := clientset.RbacV1().Roles(defaultNamespace).Create(ctx, role, metav1.CreateOptions{}) 1711 require.NoError(t, err) 1712 _, err = clientset.RbacV1().RoleBindings(defaultNamespace).Create(ctx, binding, metav1.CreateOptions{}) 1713 require.NoError(t, err) 1714 } 1715 1716 func configureClientConfigForOIDC(t *testing.T, config *rest.Config, clientID, caFilePath, idToken, refreshToken, oidcServerURL string) *rest.Config { 1717 t.Helper() 1718 cfg := rest.AnonymousClientConfig(config) 1719 cfg.AuthProvider = &api.AuthProviderConfig{ 1720 Name: "oidc", 1721 Config: map[string]string{ 1722 "client-id": clientID, 1723 "id-token": idToken, 1724 "idp-issuer-url": oidcServerURL, 1725 "idp-certificate-authority": caFilePath, 1726 "refresh-token": refreshToken, 1727 }, 1728 } 1729 1730 return cfg 1731 } 1732 1733 func startTestAPIServerForOIDC[L utilsoidc.JosePublicKey](t *testing.T, c apiServerOIDCConfig, publicKey L) *kubeapiserverapptesting.TestServer { 1734 t.Helper() 1735 1736 var customFlags []string 1737 if len(c.authenticationConfigYAML) > 0 { 1738 customFlags = []string{fmt.Sprintf("--authentication-config=%s", writeTempFile(t, c.authenticationConfigYAML))} 1739 } else { 1740 customFlags = []string{ 1741 fmt.Sprintf("--oidc-issuer-url=%s", c.oidcURL), 1742 fmt.Sprintf("--oidc-client-id=%s", c.oidcClientID), 1743 fmt.Sprintf("--oidc-ca-file=%s", c.oidcCAFilePath), 1744 fmt.Sprintf("--oidc-username-prefix=%s", c.oidcUsernamePrefix), 1745 } 1746 if len(c.oidcUsernameClaim) > 0 { 1747 customFlags = append(customFlags, fmt.Sprintf("--oidc-username-claim=%s", c.oidcUsernameClaim)) 1748 } 1749 customFlags = append(customFlags, maybeSetSigningAlgs(publicKey)...) 1750 } 1751 customFlags = append(customFlags, "--authorization-mode=RBAC") 1752 1753 server, err := kubeapiserverapptesting.StartTestServer( 1754 t, 1755 kubeapiserverapptesting.NewDefaultTestServerOptions(), 1756 customFlags, 1757 framework.SharedEtcd(), 1758 ) 1759 require.NoError(t, err) 1760 1761 t.Cleanup(server.TearDownFn) 1762 1763 return &server 1764 } 1765 1766 func maybeSetSigningAlgs[K utilsoidc.JoseKey](key K) []string { 1767 alg := utilsoidc.GetSignatureAlgorithm(key) 1768 if alg == jose.RS256 && randomBool() { 1769 return nil // check the default case of RS256 by not always setting the flag 1770 } 1771 return []string{ 1772 fmt.Sprintf("--oidc-signing-algs=%s", alg), // all other algs need to be manually set 1773 } 1774 } 1775 1776 func randomBool() bool { return utilrand.Int()%2 == 1 } 1777 1778 func fetchOIDCCredentials(t *testing.T, oidcTokenURL string, caCertContent []byte) (idToken, refreshToken string) { 1779 t.Helper() 1780 1781 req, err := http.NewRequest(http.MethodGet, oidcTokenURL, http.NoBody) 1782 require.NoError(t, err) 1783 1784 caPool := x509.NewCertPool() 1785 ok := caPool.AppendCertsFromPEM(caCertContent) 1786 require.True(t, ok) 1787 1788 client := http.Client{Transport: &http.Transport{ 1789 TLSClientConfig: &tls.Config{ 1790 RootCAs: caPool, 1791 }, 1792 }} 1793 1794 token := new(utilsoidc.Token) 1795 1796 resp, err := client.Do(req) 1797 require.NoError(t, err) 1798 1799 err = json.NewDecoder(resp.Body).Decode(token) 1800 require.NoError(t, err) 1801 1802 return token.IDToken, token.RefreshToken 1803 } 1804 1805 func fetchExpiredToken(t *testing.T, oidcServer *utilsoidc.TestServer, caCertContent []byte, signingPrivateKey *rsa.PrivateKey) (expiredToken, stubRefreshToken string) { 1806 t.Helper() 1807 1808 tokenURL, err := oidcServer.TokenURL() 1809 require.NoError(t, err) 1810 1811 configureOIDCServerToReturnExpiredIDToken(t, 1, oidcServer, signingPrivateKey) 1812 expiredToken, stubRefreshToken = fetchOIDCCredentials(t, tokenURL, caCertContent) 1813 1814 return expiredToken, stubRefreshToken 1815 } 1816 1817 func configureOIDCServerToReturnExpiredIDToken(t *testing.T, returningExpiredTokenTimes int, oidcServer *utilsoidc.TestServer, signingPrivateKey *rsa.PrivateKey) { 1818 t.Helper() 1819 1820 oidcServer.TokenHandler().EXPECT().Token().Times(returningExpiredTokenTimes).DoAndReturn(func() (utilsoidc.Token, error) { 1821 token, err := utilsoidc.TokenHandlerBehaviorReturningPredefinedJWT( 1822 t, 1823 signingPrivateKey, 1824 map[string]interface{}{ 1825 "iss": oidcServer.URL(), 1826 "sub": defaultOIDCClaimedUsername, 1827 "aud": defaultOIDCClientID, 1828 "exp": time.Now().Add(-time.Millisecond).Unix(), 1829 }, 1830 defaultStubAccessToken, 1831 defaultStubRefreshToken, 1832 )() 1833 return token, err 1834 }) 1835 } 1836 1837 func configureOIDCServerToReturnExpiredRefreshTokenErrorOnTryingToUpdateIDToken(oidcServer *utilsoidc.TestServer) { 1838 oidcServer.TokenHandler().EXPECT().Token().Times(2).Return(utilsoidc.Token{}, utilsoidc.ErrRefreshTokenExpired) 1839 } 1840 1841 func generateCert(t *testing.T) (cert, key []byte, certFilePath, keyFilePath string) { 1842 t.Helper() 1843 1844 tempDir := t.TempDir() 1845 certFilePath = filepath.Join(tempDir, "localhost_127.0.0.1_.crt") 1846 keyFilePath = filepath.Join(tempDir, "localhost_127.0.0.1_.key") 1847 1848 cert, key, err := certutil.GenerateSelfSignedCertKeyWithFixtures("localhost", []net.IP{utilsnet.ParseIPSloppy("127.0.0.1")}, nil, tempDir) 1849 require.NoError(t, err) 1850 1851 return cert, key, certFilePath, keyFilePath 1852 } 1853 1854 func writeTempFile(t *testing.T, content string) string { 1855 t.Helper() 1856 file, err := os.CreateTemp("", "oidc-test") 1857 if err != nil { 1858 t.Fatal(err) 1859 } 1860 t.Cleanup(func() { 1861 if err := os.Remove(file.Name()); err != nil { 1862 t.Fatal(err) 1863 } 1864 }) 1865 if err := os.WriteFile(file.Name(), []byte(content), 0600); err != nil { 1866 t.Fatal(err) 1867 } 1868 return file.Name() 1869 } 1870 1871 // indentCertificateAuthority indents the certificate authority to match 1872 // the format of the generated authentication config. 1873 func indentCertificateAuthority(caCert string) string { 1874 return strings.ReplaceAll(caCert, "\n", "\n ") 1875 } 1876 1877 func testContext(t *testing.T) context.Context { 1878 ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) 1879 t.Cleanup(cancel) 1880 return ctx 1881 }