k8s.io/apiserver@v0.31.1/pkg/server/options/encryptionconfig/config_test.go (about) 1 /* 2 Copyright 2017 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package encryptionconfig 18 19 import ( 20 "bytes" 21 "context" 22 "encoding/base64" 23 "errors" 24 "fmt" 25 "io" 26 "reflect" 27 "strings" 28 "sync" 29 "testing" 30 "time" 31 32 "github.com/google/go-cmp/cmp" 33 34 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 35 "k8s.io/apimachinery/pkg/runtime/schema" 36 "k8s.io/apimachinery/pkg/util/sets" 37 "k8s.io/apiserver/pkg/apis/apiserver" 38 "k8s.io/apiserver/pkg/features" 39 "k8s.io/apiserver/pkg/storage/value" 40 "k8s.io/apiserver/pkg/storage/value/encrypt/envelope" 41 envelopekmsv2 "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2" 42 kmstypes "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/v2" 43 "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics" 44 utilfeature "k8s.io/apiserver/pkg/util/feature" 45 featuregatetesting "k8s.io/component-base/featuregate/testing" 46 "k8s.io/component-base/metrics/legacyregistry" 47 "k8s.io/component-base/metrics/testutil" 48 "k8s.io/klog/v2" 49 kmsservice "k8s.io/kms/pkg/service" 50 "k8s.io/utils/pointer" 51 ) 52 53 const ( 54 sampleText = "abcdefghijklmnopqrstuvwxyz" 55 sampleContextText = "0123456789" 56 ) 57 58 var ( 59 sampleInvalidKeyID = string(make([]byte, envelopekmsv2.KeyIDMaxSize+1)) 60 ) 61 62 // testEnvelopeService is a mock envelope service which can be used to simulate remote Envelope services 63 // for testing of the envelope transformer with other transformers. 64 type testEnvelopeService struct { 65 err error 66 } 67 68 func (t *testEnvelopeService) Decrypt(data []byte) ([]byte, error) { 69 if t.err != nil { 70 return nil, t.err 71 } 72 return base64.StdEncoding.DecodeString(string(data)) 73 } 74 75 func (t *testEnvelopeService) Encrypt(data []byte) ([]byte, error) { 76 if t.err != nil { 77 return nil, t.err 78 } 79 return []byte(base64.StdEncoding.EncodeToString(data)), nil 80 } 81 82 // testKMSv2EnvelopeService is a mock kmsv2 envelope service which can be used to simulate remote Envelope v2 services 83 // for testing of the envelope transformer with other transformers. 84 type testKMSv2EnvelopeService struct { 85 err error 86 keyID string 87 encryptCalls int 88 encryptAnnotations map[string][]byte 89 } 90 91 func (t *testKMSv2EnvelopeService) Decrypt(ctx context.Context, uid string, req *kmsservice.DecryptRequest) ([]byte, error) { 92 if t.err != nil { 93 return nil, t.err 94 } 95 return base64.StdEncoding.DecodeString(string(req.Ciphertext)) 96 } 97 98 func (t *testKMSv2EnvelopeService) Encrypt(ctx context.Context, uid string, data []byte) (*kmsservice.EncryptResponse, error) { 99 t.encryptCalls++ 100 if t.err != nil { 101 return nil, t.err 102 } 103 return &kmsservice.EncryptResponse{ 104 Ciphertext: []byte(base64.StdEncoding.EncodeToString(data)), 105 KeyID: t.keyID, 106 Annotations: t.encryptAnnotations, 107 }, nil 108 } 109 110 func (t *testKMSv2EnvelopeService) Status(ctx context.Context) (*kmsservice.StatusResponse, error) { 111 if t.err != nil { 112 return nil, t.err 113 } 114 return &kmsservice.StatusResponse{Healthz: "ok", KeyID: t.keyID, Version: "v2beta1"}, nil 115 } 116 117 // The factory method to create mock envelope service. 118 func newMockEnvelopeService(ctx context.Context, endpoint string, timeout time.Duration) (envelope.Service, error) { 119 return &testEnvelopeService{nil}, nil 120 } 121 122 // The factory method to create mock envelope service which always returns error. 123 func newMockErrorEnvelopeService(endpoint string, timeout time.Duration) (envelope.Service, error) { 124 return &testEnvelopeService{errors.New("test")}, nil 125 } 126 127 // The factory method to create mock envelope kmsv2 service. 128 func newMockEnvelopeKMSv2Service(ctx context.Context, endpoint, providerName string, timeout time.Duration) (kmsservice.Service, error) { 129 return &testKMSv2EnvelopeService{nil, "1", 0, nil}, nil 130 } 131 132 // The factory method to create mock envelope kmsv2 service which always returns error. 133 func newMockErrorEnvelopeKMSv2Service(endpoint string, timeout time.Duration) (kmsservice.Service, error) { 134 return &testKMSv2EnvelopeService{errors.New("test"), "1", 0, nil}, nil 135 } 136 137 // The factory method to create mock envelope kmsv2 service that always returns invalid keyID. 138 func newMockInvalidKeyIDEnvelopeKMSv2Service(ctx context.Context, endpoint string, timeout time.Duration, keyID string) (kmsservice.Service, error) { 139 return &testKMSv2EnvelopeService{nil, keyID, 0, nil}, nil 140 } 141 142 func TestLegacyConfig(t *testing.T) { 143 legacyV1Config := "testdata/valid-configs/legacy.yaml" 144 legacyConfigObject, _, err := loadConfig(legacyV1Config, false) 145 cacheSize := int32(10) 146 if err != nil { 147 t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, legacyV1Config) 148 } 149 150 expected := &apiserver.EncryptionConfiguration{ 151 Resources: []apiserver.ResourceConfiguration{ 152 { 153 Resources: []string{"secrets", "namespaces"}, 154 Providers: []apiserver.ProviderConfiguration{ 155 {Identity: &apiserver.IdentityConfiguration{}}, 156 {AESGCM: &apiserver.AESConfiguration{ 157 Keys: []apiserver.Key{ 158 {Name: "key1", Secret: "c2VjcmV0IGlzIHNlY3VyZQ=="}, 159 {Name: "key2", Secret: "dGhpcyBpcyBwYXNzd29yZA=="}, 160 }, 161 }}, 162 {KMS: &apiserver.KMSConfiguration{ 163 APIVersion: "v1", 164 Name: "testprovider", 165 Endpoint: "unix:///tmp/testprovider.sock", 166 CacheSize: &cacheSize, 167 Timeout: &metav1.Duration{Duration: 3 * time.Second}, 168 }}, 169 {AESCBC: &apiserver.AESConfiguration{ 170 Keys: []apiserver.Key{ 171 {Name: "key1", Secret: "c2VjcmV0IGlzIHNlY3VyZQ=="}, 172 {Name: "key2", Secret: "dGhpcyBpcyBwYXNzd29yZA=="}, 173 }, 174 }}, 175 {Secretbox: &apiserver.SecretboxConfiguration{ 176 Keys: []apiserver.Key{ 177 {Name: "key1", Secret: "YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXoxMjM0NTY="}, 178 }, 179 }}, 180 }, 181 }, 182 }, 183 } 184 if d := cmp.Diff(expected, legacyConfigObject); d != "" { 185 t.Fatalf("EncryptionConfig mismatch (-want +got):\n%s", d) 186 } 187 } 188 189 func TestEncryptionProviderConfigCorrect(t *testing.T) { 190 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true) 191 192 // Set factory for mock envelope service 193 factory := envelopeServiceFactory 194 factoryKMSv2 := EnvelopeKMSv2ServiceFactory 195 envelopeServiceFactory = newMockEnvelopeService 196 EnvelopeKMSv2ServiceFactory = newMockEnvelopeKMSv2Service 197 defer func() { 198 envelopeServiceFactory = factory 199 EnvelopeKMSv2ServiceFactory = factoryKMSv2 200 }() 201 202 ctx := testContext(t) 203 204 // Creates compound/prefix transformers with different ordering of available transformers. 205 // Transforms data using one of them, and tries to untransform using the others. 206 // Repeats this for all possible combinations. 207 // Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163 208 expectedKMSCloseGracePeriod := 46 * time.Second 209 correctConfigWithIdentityFirst := "testdata/valid-configs/identity-first.yaml" 210 identityFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithIdentityFirst, false, "") 211 if err != nil { 212 t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithIdentityFirst) 213 } 214 if identityFirstEncryptionConfiguration.KMSCloseGracePeriod != expectedKMSCloseGracePeriod { 215 t.Fatalf("KMSCloseGracePeriod mismatch (-want +got):\n%s", cmp.Diff(expectedKMSCloseGracePeriod, identityFirstEncryptionConfiguration.KMSCloseGracePeriod)) 216 } 217 218 // Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163 219 expectedKMSCloseGracePeriod = 32 * time.Second 220 correctConfigWithAesGcmFirst := "testdata/valid-configs/aes-gcm-first.yaml" 221 aesGcmFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithAesGcmFirst, false, "") 222 if err != nil { 223 t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithAesGcmFirst) 224 } 225 if aesGcmFirstEncryptionConfiguration.KMSCloseGracePeriod != expectedKMSCloseGracePeriod { 226 t.Fatalf("KMSCloseGracePeriod mismatch (-want +got):\n%s", cmp.Diff(expectedKMSCloseGracePeriod, aesGcmFirstEncryptionConfiguration.KMSCloseGracePeriod)) 227 } 228 229 invalidConfigWithAesGcm := "testdata/invalid-configs/invalid-aes-gcm.yaml" 230 _, err = LoadEncryptionConfig(ctx, invalidConfigWithAesGcm, false, "") 231 if !strings.Contains(errString(err), "error while parsing file") { 232 t.Fatalf("should result in error while parsing configuration file: %s.\nThe file was:\n%s", err, invalidConfigWithAesGcm) 233 } 234 235 invalidConfigWithTypo := "testdata/invalid-configs/invalid-typo.yaml" 236 _, err = LoadEncryptionConfig(ctx, invalidConfigWithTypo, false, "") 237 if got, wantSubString := errString(err), `strict decoding error: unknown field "resources[0].providers[3].kms.pandas"`; !strings.Contains(got, wantSubString) { 238 t.Fatalf("should result in strict decode error while parsing configuration file %q:\ngot: %q\nwant substring: %q", invalidConfigWithTypo, got, wantSubString) 239 } 240 241 // Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163 242 expectedKMSCloseGracePeriod = 26 * time.Second 243 correctConfigWithAesCbcFirst := "testdata/valid-configs/aes-cbc-first.yaml" 244 aesCbcFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithAesCbcFirst, false, "") 245 if err != nil { 246 t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithAesCbcFirst) 247 } 248 if aesCbcFirstEncryptionConfiguration.KMSCloseGracePeriod != expectedKMSCloseGracePeriod { 249 t.Fatalf("KMSCloseGracePeriod mismatch (-want +got):\n%s", cmp.Diff(expectedKMSCloseGracePeriod, aesCbcFirstEncryptionConfiguration.KMSCloseGracePeriod)) 250 } 251 252 // Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163 253 expectedKMSCloseGracePeriod = 14 * time.Second 254 correctConfigWithSecretboxFirst := "testdata/valid-configs/secret-box-first.yaml" 255 secretboxFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithSecretboxFirst, false, "") 256 if err != nil { 257 t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithSecretboxFirst) 258 } 259 if secretboxFirstEncryptionConfiguration.KMSCloseGracePeriod != expectedKMSCloseGracePeriod { 260 t.Fatalf("KMSCloseGracePeriod mismatch (-want +got):\n%s", cmp.Diff(expectedKMSCloseGracePeriod, secretboxFirstEncryptionConfiguration.KMSCloseGracePeriod)) 261 } 262 263 // Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163 264 expectedKMSCloseGracePeriod = 34 * time.Second 265 correctConfigWithKMSFirst := "testdata/valid-configs/kms-first.yaml" 266 kmsFirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithKMSFirst, false, "") 267 if err != nil { 268 t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithKMSFirst) 269 } 270 if kmsFirstEncryptionConfiguration.KMSCloseGracePeriod != expectedKMSCloseGracePeriod { 271 t.Fatalf("KMSCloseGracePeriod mismatch (-want +got):\n%s", cmp.Diff(expectedKMSCloseGracePeriod, kmsFirstEncryptionConfiguration.KMSCloseGracePeriod)) 272 } 273 274 // Math for GracePeriod is explained at - https://github.com/kubernetes/kubernetes/blob/c9ed04762f94a319d7b1fb718dc345491a32bea6/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go#L159-L163 275 expectedKMSCloseGracePeriod = 42 * time.Second 276 correctConfigWithKMSv2First := "testdata/valid-configs/kmsv2-first.yaml" 277 kmsv2FirstEncryptionConfiguration, err := LoadEncryptionConfig(ctx, correctConfigWithKMSv2First, false, "") 278 if err != nil { 279 t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithKMSv2First) 280 } 281 if kmsv2FirstEncryptionConfiguration.KMSCloseGracePeriod != expectedKMSCloseGracePeriod { 282 t.Fatalf("KMSCloseGracePeriod mismatch (-want +got):\n%s", cmp.Diff(expectedKMSCloseGracePeriod, kmsv2FirstEncryptionConfiguration.KMSCloseGracePeriod)) 283 } 284 285 // Pick the transformer for any of the returned resources. 286 identityFirstTransformer := identityFirstEncryptionConfiguration.Transformers[schema.ParseGroupResource("secrets")] 287 aesGcmFirstTransformer := aesGcmFirstEncryptionConfiguration.Transformers[schema.ParseGroupResource("secrets")] 288 aesCbcFirstTransformer := aesCbcFirstEncryptionConfiguration.Transformers[schema.ParseGroupResource("secrets")] 289 secretboxFirstTransformer := secretboxFirstEncryptionConfiguration.Transformers[schema.ParseGroupResource("secrets")] 290 kmsFirstTransformer := kmsFirstEncryptionConfiguration.Transformers[schema.ParseGroupResource("secrets")] 291 kmsv2FirstTransformer := kmsv2FirstEncryptionConfiguration.Transformers[schema.ParseGroupResource("secrets")] 292 293 dataCtx := value.DefaultContext(sampleContextText) 294 originalText := []byte(sampleText) 295 296 transformers := []struct { 297 Transformer value.Transformer 298 Name string 299 }{ 300 {aesGcmFirstTransformer, "aesGcmFirst"}, 301 {aesCbcFirstTransformer, "aesCbcFirst"}, 302 {secretboxFirstTransformer, "secretboxFirst"}, 303 {identityFirstTransformer, "identityFirst"}, 304 {kmsFirstTransformer, "kmsFirst"}, 305 {kmsv2FirstTransformer, "kmvs2First"}, 306 } 307 308 for _, testCase := range transformers { 309 transformedData, err := testCase.Transformer.TransformToStorage(ctx, originalText, dataCtx) 310 if err != nil { 311 t.Fatalf("%s: error while transforming data to storage: %s", testCase.Name, err) 312 } 313 314 for _, transformer := range transformers { 315 untransformedData, stale, err := transformer.Transformer.TransformFromStorage(ctx, transformedData, dataCtx) 316 if err != nil { 317 t.Fatalf("%s: error while reading using %s transformer: %s", testCase.Name, transformer.Name, err) 318 } 319 if stale != (transformer.Name != testCase.Name) { 320 t.Fatalf("%s: wrong stale information on reading using %s transformer, should be %v", testCase.Name, transformer.Name, testCase.Name == transformer.Name) 321 } 322 if !bytes.Equal(untransformedData, originalText) { 323 t.Fatalf("%s: %s transformer transformed data incorrectly. Expected: %v, got %v", testCase.Name, transformer.Name, originalText, untransformedData) 324 } 325 } 326 } 327 } 328 329 func TestKMSv1Deprecation(t *testing.T) { 330 testCases := []struct { 331 name string 332 kmsv1Enabled bool 333 expectedErr string 334 }{ 335 { 336 name: "config with kmsv1, KMSv1=false", 337 kmsv1Enabled: false, 338 expectedErr: "KMSv1 is deprecated and will only receive security updates going forward. Use KMSv2 instead. Set --feature-gates=KMSv1=true to use the deprecated KMSv1 feature.", 339 }, 340 { 341 name: "config with kmsv1, KMSv1=true", 342 kmsv1Enabled: true, 343 expectedErr: "", 344 }, 345 } 346 347 for _, testCase := range testCases { 348 t.Run(testCase.name, func(t *testing.T) { 349 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, testCase.kmsv1Enabled) 350 351 kmsv1Config := "testdata/valid-configs/kms/multiple-providers.yaml" 352 _, err := LoadEncryptionConfig(testContext(t), kmsv1Config, false, "") 353 if !strings.Contains(errString(err), testCase.expectedErr) { 354 t.Fatalf("expected error %q, got %q", testCase.expectedErr, errString(err)) 355 } 356 }) 357 } 358 } 359 360 func TestKMSvsEnablement(t *testing.T) { 361 testCases := []struct { 362 name string 363 filePath string 364 expectedErr string 365 }{ 366 { 367 name: "config with kmsv2 and kmsv1, KMSv2=true, KMSv1=false, should fail when feature is disabled", 368 filePath: "testdata/valid-configs/kms/multiple-providers-mixed.yaml", 369 expectedErr: "KMSv1 is deprecated and will only receive security updates going forward. Use KMSv2 instead", 370 }, 371 { 372 name: "config with kmsv2, KMSv2=true, KMSv1=false", 373 filePath: "testdata/valid-configs/kms/multiple-providers-kmsv2.yaml", 374 expectedErr: "", 375 }, 376 } 377 378 for _, testCase := range testCases { 379 t.Run(testCase.name, func(t *testing.T) { 380 // only the KMSv2 feature flag is enabled 381 _, err := LoadEncryptionConfig(testContext(t), testCase.filePath, false, "") 382 383 if len(testCase.expectedErr) > 0 && !strings.Contains(errString(err), testCase.expectedErr) { 384 t.Fatalf("expected error %q, got %q", testCase.expectedErr, errString(err)) 385 } 386 if len(testCase.expectedErr) == 0 && err != nil { 387 t.Fatalf("unexpected error %q", errString(err)) 388 } 389 390 }) 391 } 392 tts := []struct { 393 name string 394 kmsv2Enabled bool 395 expectedErr string 396 expectedTimeout time.Duration 397 config apiserver.EncryptionConfiguration 398 wantV2Used bool 399 }{ 400 { 401 name: "with kmsv1 and kmsv2, KMSv2=true", 402 kmsv2Enabled: true, 403 config: apiserver.EncryptionConfiguration{ 404 Resources: []apiserver.ResourceConfiguration{ 405 { 406 Resources: []string{"secrets"}, 407 Providers: []apiserver.ProviderConfiguration{ 408 { 409 KMS: &apiserver.KMSConfiguration{ 410 Name: "kms", 411 APIVersion: "v1", 412 Timeout: &metav1.Duration{ 413 Duration: 1 * time.Second, 414 }, 415 Endpoint: "unix:///tmp/testprovider.sock", 416 CacheSize: pointer.Int32(1000), 417 }, 418 }, 419 { 420 KMS: &apiserver.KMSConfiguration{ 421 Name: "another-kms", 422 APIVersion: "v2", 423 Timeout: &metav1.Duration{ 424 Duration: 1 * time.Second, 425 }, 426 Endpoint: "unix:///tmp/anothertestprovider.sock", 427 CacheSize: pointer.Int32(1000), 428 }, 429 }, 430 }, 431 }, 432 }, 433 }, 434 expectedErr: "", 435 wantV2Used: true, 436 }, 437 } 438 439 for _, tt := range tts { 440 t.Run(tt.name, func(t *testing.T) { 441 // Just testing KMSv2 feature flag 442 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true) 443 444 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, tt.kmsv2Enabled) 445 446 ctx, cancel := context.WithCancel(context.Background()) 447 cancel() // cancel this upfront so the kms v2 checks do not block 448 449 _, _, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(ctx, &tt.config, "") 450 if err == nil { 451 if kmsUsed == nil || kmsUsed.v2Used != tt.wantV2Used { 452 t.Fatalf("unexpected kmsUsed value, expected: %v, got: %v", tt.wantV2Used, kmsUsed) 453 } 454 455 } 456 if !strings.Contains(errString(err), tt.expectedErr) { 457 t.Fatalf("expecting error calling prefixTransformersAndProbes, expected: %s, got: %s", tt.expectedErr, errString(err)) 458 } 459 }) 460 } 461 } 462 463 func TestKMSMaxTimeout(t *testing.T) { 464 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true) 465 466 testCases := []struct { 467 name string 468 expectedErr string 469 expectedTimeout time.Duration 470 config apiserver.EncryptionConfiguration 471 }{ 472 { 473 name: "config with bad provider", 474 config: apiserver.EncryptionConfiguration{ 475 Resources: []apiserver.ResourceConfiguration{ 476 { 477 Resources: []string{"secrets"}, 478 Providers: []apiserver.ProviderConfiguration{ 479 { 480 KMS: nil, 481 }, 482 }, 483 }, 484 }, 485 }, 486 expectedErr: "provider does not contain any of the expected providers: KMS, AESGCM, AESCBC, Secretbox, Identity", 487 expectedTimeout: 6 * time.Second, 488 }, 489 { 490 name: "default timeout", 491 config: apiserver.EncryptionConfiguration{ 492 Resources: []apiserver.ResourceConfiguration{ 493 { 494 Resources: []string{"secrets"}, 495 Providers: []apiserver.ProviderConfiguration{ 496 { 497 KMS: &apiserver.KMSConfiguration{ 498 Name: "kms", 499 APIVersion: "v1", 500 Timeout: &metav1.Duration{ 501 // default timeout is 3s 502 // this will be set automatically if not provided in config file 503 Duration: 3 * time.Second, 504 }, 505 Endpoint: "unix:///tmp/testprovider.sock", 506 }, 507 }, 508 }, 509 }, 510 }, 511 }, 512 expectedErr: "", 513 expectedTimeout: 6 * time.Second, 514 }, 515 { 516 name: "with v1 provider", 517 config: apiserver.EncryptionConfiguration{ 518 Resources: []apiserver.ResourceConfiguration{ 519 { 520 Resources: []string{"secrets"}, 521 Providers: []apiserver.ProviderConfiguration{ 522 { 523 KMS: &apiserver.KMSConfiguration{ 524 Name: "kms", 525 APIVersion: "v1", 526 Timeout: &metav1.Duration{ 527 // default timeout is 3s 528 // this will be set automatically if not provided in config file 529 Duration: 3 * time.Second, 530 }, 531 Endpoint: "unix:///tmp/testprovider.sock", 532 }, 533 }, 534 }, 535 }, 536 { 537 Resources: []string{"configmaps"}, 538 Providers: []apiserver.ProviderConfiguration{ 539 { 540 KMS: &apiserver.KMSConfiguration{ 541 Name: "kms", 542 APIVersion: "v1", 543 Timeout: &metav1.Duration{ 544 // default timeout is 3s 545 // this will be set automatically if not provided in config file 546 Duration: 3 * time.Second, 547 }, 548 Endpoint: "unix:///tmp/testprovider.sock", 549 }, 550 }, 551 }, 552 }, 553 }, 554 }, 555 expectedErr: "", 556 expectedTimeout: 12 * time.Second, 557 }, 558 { 559 name: "with v2 provider", 560 config: apiserver.EncryptionConfiguration{ 561 Resources: []apiserver.ResourceConfiguration{ 562 { 563 Resources: []string{"secrets"}, 564 Providers: []apiserver.ProviderConfiguration{ 565 { 566 KMS: &apiserver.KMSConfiguration{ 567 Name: "kms", 568 APIVersion: "v2", 569 Timeout: &metav1.Duration{ 570 Duration: 15 * time.Second, 571 }, 572 Endpoint: "unix:///tmp/testprovider.sock", 573 }, 574 }, 575 { 576 KMS: &apiserver.KMSConfiguration{ 577 Name: "new-kms", 578 APIVersion: "v2", 579 Timeout: &metav1.Duration{ 580 Duration: 5 * time.Second, 581 }, 582 Endpoint: "unix:///tmp/anothertestprovider.sock", 583 }, 584 }, 585 }, 586 }, 587 { 588 Resources: []string{"configmaps"}, 589 Providers: []apiserver.ProviderConfiguration{ 590 { 591 KMS: &apiserver.KMSConfiguration{ 592 Name: "another-kms", 593 APIVersion: "v2", 594 Timeout: &metav1.Duration{ 595 Duration: 10 * time.Second, 596 }, 597 Endpoint: "unix:///tmp/testprovider.sock", 598 }, 599 }, 600 { 601 KMS: &apiserver.KMSConfiguration{ 602 Name: "yet-another-kms", 603 APIVersion: "v2", 604 Timeout: &metav1.Duration{ 605 Duration: 2 * time.Second, 606 }, 607 Endpoint: "unix:///tmp/anothertestprovider.sock", 608 }, 609 }, 610 }, 611 }, 612 }, 613 }, 614 expectedErr: "", 615 expectedTimeout: 32 * time.Second, 616 }, 617 { 618 name: "with v1 and v2 provider", 619 config: apiserver.EncryptionConfiguration{ 620 Resources: []apiserver.ResourceConfiguration{ 621 { 622 Resources: []string{"secrets"}, 623 Providers: []apiserver.ProviderConfiguration{ 624 { 625 KMS: &apiserver.KMSConfiguration{ 626 Name: "kms", 627 APIVersion: "v1", 628 Timeout: &metav1.Duration{ 629 Duration: 1 * time.Second, 630 }, 631 Endpoint: "unix:///tmp/testprovider.sock", 632 }, 633 }, 634 { 635 KMS: &apiserver.KMSConfiguration{ 636 Name: "another-kms", 637 APIVersion: "v2", 638 Timeout: &metav1.Duration{ 639 Duration: 1 * time.Second, 640 }, 641 Endpoint: "unix:///tmp/anothertestprovider.sock", 642 }, 643 }, 644 }, 645 }, 646 { 647 Resources: []string{"configmaps"}, 648 Providers: []apiserver.ProviderConfiguration{ 649 { 650 KMS: &apiserver.KMSConfiguration{ 651 Name: "kms", 652 APIVersion: "v1", 653 Timeout: &metav1.Duration{ 654 Duration: 4 * time.Second, 655 }, 656 Endpoint: "unix:///tmp/testprovider.sock", 657 }, 658 }, 659 { 660 KMS: &apiserver.KMSConfiguration{ 661 Name: "yet-another-kms", 662 APIVersion: "v1", 663 Timeout: &metav1.Duration{ 664 Duration: 2 * time.Second, 665 }, 666 Endpoint: "unix:///tmp/anothertestprovider.sock", 667 }, 668 }, 669 }, 670 }, 671 }, 672 }, 673 expectedErr: "", 674 expectedTimeout: 15 * time.Second, 675 }, 676 } 677 678 for _, testCase := range testCases { 679 t.Run(testCase.name, func(t *testing.T) { 680 cacheSize := int32(1000) 681 for _, resource := range testCase.config.Resources { 682 for _, provider := range resource.Providers { 683 if provider.KMS != nil { 684 provider.KMS.CacheSize = &cacheSize 685 } 686 } 687 } 688 689 ctx, cancel := context.WithCancel(context.Background()) 690 cancel() // cancel this upfront so the kms v2 checks do not block 691 692 _, _, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(ctx, &testCase.config, "") 693 694 if !strings.Contains(errString(err), testCase.expectedErr) { 695 t.Fatalf("expecting error calling prefixTransformersAndProbes, expected: %s, got: %s", testCase.expectedErr, errString(err)) 696 } 697 if len(testCase.expectedErr) == 0 { 698 if kmsUsed == nil { 699 t.Fatal("kmsUsed should not be nil") 700 } 701 702 if kmsUsed.kmsTimeoutSum != testCase.expectedTimeout { 703 t.Fatalf("expected timeout %v, got %v", testCase.expectedTimeout, kmsUsed.kmsTimeoutSum) 704 } 705 } 706 707 }) 708 } 709 } 710 711 func TestKMSPluginHealthz(t *testing.T) { 712 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true) 713 714 kmsv2Probe := &kmsv2PluginProbe{ 715 name: "foo", 716 ttl: 3 * time.Second, 717 apiServerID: "", 718 } 719 keyID := "1" 720 kmsv2Probe.state.Store(&envelopekmsv2.State{EncryptedObject: kmstypes.EncryptedObject{KeyID: keyID}}) 721 722 testCases := []struct { 723 desc string 724 config string 725 want []healthChecker 726 wantErr string 727 kmsv2 bool 728 kmsv1 bool 729 }{ 730 { 731 desc: "Invalid config file path", 732 config: "invalid/path", 733 want: nil, 734 wantErr: `error reading encryption provider configuration file "invalid/path"`, 735 }, 736 { 737 desc: "Empty config file content", 738 config: "testdata/invalid-configs/kms/invalid-content.yaml", 739 want: nil, 740 wantErr: `encryption provider configuration file "testdata/invalid-configs/kms/invalid-content.yaml" is empty`, 741 }, 742 { 743 desc: "Unable to decode", 744 config: "testdata/invalid-configs/kms/invalid-gvk.yaml", 745 want: nil, 746 wantErr: `error decoding encryption provider configuration file`, 747 }, 748 { 749 desc: "Unexpected config type", 750 config: "testdata/invalid-configs/kms/invalid-config-type.yaml", 751 want: nil, 752 wantErr: `no kind "EncryptionConfigurations" is registered for version "apiserver.config.k8s.io/v1"`, 753 }, 754 { 755 desc: "Install Healthz", 756 config: "testdata/valid-configs/kms/default-timeout.yaml", 757 want: []healthChecker{ 758 &kmsPluginProbe{ 759 name: "foo", 760 ttl: 3 * time.Second, 761 }, 762 }, 763 kmsv1: true, 764 }, 765 { 766 desc: "Install multiple healthz", 767 config: "testdata/valid-configs/kms/multiple-providers.yaml", 768 want: []healthChecker{ 769 &kmsPluginProbe{ 770 name: "foo", 771 ttl: 3 * time.Second, 772 }, 773 &kmsPluginProbe{ 774 name: "bar", 775 ttl: 3 * time.Second, 776 }, 777 }, 778 kmsv1: true, 779 }, 780 { 781 desc: "No KMS Providers", 782 config: "testdata/valid-configs/aes/aes-gcm.yaml", 783 }, 784 { 785 desc: "Install multiple healthz with v1 and v2", 786 config: "testdata/valid-configs/kms/multiple-providers-mixed.yaml", 787 want: []healthChecker{ 788 kmsv2Probe, 789 &kmsPluginProbe{ 790 name: "bar", 791 ttl: 3 * time.Second, 792 }, 793 }, 794 kmsv2: true, 795 kmsv1: true, 796 }, 797 { 798 desc: "Invalid API version", 799 config: "testdata/invalid-configs/kms/invalid-apiversion.yaml", 800 want: nil, 801 wantErr: `resources[0].providers[0].kms.apiVersion: Invalid value: "v3": unsupported apiVersion apiVersion for KMS provider, only v1 and v2 are supported`, 802 }, 803 } 804 805 for _, tt := range testCases { 806 t.Run(tt.desc, func(t *testing.T) { 807 config, _, err := loadConfig(tt.config, false) 808 if errStr := errString(err); !strings.Contains(errStr, tt.wantErr) { 809 t.Fatalf("unexpected error state got=%s want=%s", errStr, tt.wantErr) 810 } 811 if len(tt.wantErr) > 0 { 812 return 813 } 814 815 ctx, cancel := context.WithCancel(context.Background()) 816 cancel() // cancel this upfront so the kms v2 healthz check poll does not run 817 _, got, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(ctx, config, "") 818 if err != nil { 819 t.Fatal(err) 820 } 821 822 // unset fields that are not relevant to the test 823 for i := range got { 824 checker := got[i] 825 switch p := checker.(type) { 826 case *kmsPluginProbe: 827 p.service = nil 828 p.l = nil 829 p.lastResponse = nil 830 case *kmsv2PluginProbe: 831 p.service = nil 832 p.l = nil 833 p.lastResponse = nil 834 p.state.Store(kmsv2Probe.state.Load()) 835 default: 836 t.Fatalf("unexpected probe type %T", p) 837 } 838 } 839 840 if tt.kmsv2 != kmsUsed.v2Used { 841 t.Errorf("incorrect kms v2 detection: want=%v got=%v", tt.kmsv2, kmsUsed.v2Used) 842 } 843 if tt.kmsv1 != kmsUsed.v1Used { 844 t.Errorf("incorrect kms v1 detection: want=%v got=%v", tt.kmsv1, kmsUsed.v1Used) 845 } 846 847 if d := cmp.Diff(tt.want, got, 848 cmp.Comparer(func(a, b *kmsPluginProbe) bool { 849 return *a == *b 850 }), 851 cmp.Comparer(func(a, b *kmsv2PluginProbe) bool { 852 return *a == *b 853 }), 854 ); d != "" { 855 t.Fatalf("HealthzConfig mismatch (-want +got):\n%s", d) 856 } 857 }) 858 } 859 } 860 861 // tests for masking rules 862 func TestWildcardMasking(t *testing.T) { 863 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true) 864 865 testCases := []struct { 866 desc string 867 config *apiserver.EncryptionConfiguration 868 expectedError string 869 }{ 870 { 871 desc: "resources masked by *. group", 872 config: &apiserver.EncryptionConfiguration{ 873 Resources: []apiserver.ResourceConfiguration{ 874 { 875 Resources: []string{ 876 "configmaps", 877 "*.", 878 "secrets", 879 }, 880 Providers: []apiserver.ProviderConfiguration{ 881 { 882 KMS: &apiserver.KMSConfiguration{ 883 Name: "kms", 884 APIVersion: "v1", 885 Timeout: &metav1.Duration{Duration: 3 * time.Second}, 886 Endpoint: "unix:///tmp/testprovider.sock", 887 CacheSize: pointer.Int32(10), 888 }, 889 }, 890 }, 891 }, 892 }, 893 }, 894 expectedError: "resource \"secrets\" is masked by earlier rule \"*.\"", 895 }, 896 { 897 desc: "*. masked by *. group", 898 config: &apiserver.EncryptionConfiguration{ 899 Resources: []apiserver.ResourceConfiguration{ 900 { 901 Resources: []string{ 902 "*.", 903 }, 904 Providers: []apiserver.ProviderConfiguration{ 905 { 906 KMS: &apiserver.KMSConfiguration{ 907 Name: "kms", 908 APIVersion: "v1", 909 Timeout: &metav1.Duration{Duration: 3 * time.Second}, 910 Endpoint: "unix:///tmp/testprovider.sock", 911 CacheSize: pointer.Int32(10), 912 }, 913 }, 914 }, 915 }, 916 { 917 Resources: []string{ 918 "*.", 919 }, 920 Providers: []apiserver.ProviderConfiguration{ 921 { 922 KMS: &apiserver.KMSConfiguration{ 923 Name: "kms2", 924 APIVersion: "v1", 925 Timeout: &metav1.Duration{Duration: 3 * time.Second}, 926 Endpoint: "unix:///tmp/testprovider.sock", 927 CacheSize: pointer.Int32(10), 928 }, 929 }, 930 }, 931 }, 932 }, 933 }, 934 expectedError: "resource \"*.\" is masked by earlier rule \"*.\"", 935 }, 936 { 937 desc: "*.foo masked by *.foo", 938 config: &apiserver.EncryptionConfiguration{ 939 Resources: []apiserver.ResourceConfiguration{ 940 { 941 Resources: []string{ 942 "*.foo", 943 }, 944 Providers: []apiserver.ProviderConfiguration{ 945 { 946 KMS: &apiserver.KMSConfiguration{ 947 Name: "kms", 948 APIVersion: "v1", 949 Timeout: &metav1.Duration{Duration: 3 * time.Second}, 950 Endpoint: "unix:///tmp/testprovider.sock", 951 CacheSize: pointer.Int32(10), 952 }, 953 }, 954 }, 955 }, 956 { 957 Resources: []string{ 958 "*.foo", 959 }, 960 Providers: []apiserver.ProviderConfiguration{ 961 { 962 KMS: &apiserver.KMSConfiguration{ 963 Name: "kms2", 964 APIVersion: "v1", 965 Timeout: &metav1.Duration{Duration: 3 * time.Second}, 966 Endpoint: "unix:///tmp/testprovider.sock", 967 CacheSize: pointer.Int32(10), 968 }, 969 }, 970 }, 971 }, 972 }, 973 }, 974 expectedError: "resource \"*.foo\" is masked by earlier rule \"*.foo\"", 975 }, 976 { 977 desc: "*.* masked by *.*", 978 config: &apiserver.EncryptionConfiguration{ 979 Resources: []apiserver.ResourceConfiguration{ 980 { 981 Resources: []string{ 982 "*.*", 983 }, 984 Providers: []apiserver.ProviderConfiguration{ 985 { 986 KMS: &apiserver.KMSConfiguration{ 987 Name: "kms", 988 APIVersion: "v1", 989 Timeout: &metav1.Duration{Duration: 3 * time.Second}, 990 Endpoint: "unix:///tmp/testprovider.sock", 991 CacheSize: pointer.Int32(10), 992 }, 993 }, 994 }, 995 }, 996 { 997 Resources: []string{ 998 "*.*", 999 }, 1000 Providers: []apiserver.ProviderConfiguration{ 1001 { 1002 KMS: &apiserver.KMSConfiguration{ 1003 Name: "kms2", 1004 APIVersion: "v1", 1005 Timeout: &metav1.Duration{Duration: 3 * time.Second}, 1006 Endpoint: "unix:///tmp/testprovider.sock", 1007 CacheSize: pointer.Int32(10), 1008 }, 1009 }, 1010 }, 1011 }, 1012 }, 1013 }, 1014 expectedError: "resource \"*.*\" is masked by earlier rule \"*.*\"", 1015 }, 1016 { 1017 desc: "resources masked by *. group in multiple configurations", 1018 config: &apiserver.EncryptionConfiguration{ 1019 Resources: []apiserver.ResourceConfiguration{ 1020 { 1021 Resources: []string{ 1022 "configmaps", 1023 }, 1024 Providers: []apiserver.ProviderConfiguration{ 1025 { 1026 KMS: &apiserver.KMSConfiguration{ 1027 Name: "kms", 1028 APIVersion: "v1", 1029 Timeout: &metav1.Duration{Duration: 3 * time.Second}, 1030 Endpoint: "unix:///tmp/testprovider.sock", 1031 CacheSize: pointer.Int32(10), 1032 }, 1033 }, 1034 }, 1035 }, 1036 { 1037 Resources: []string{ 1038 "*.", 1039 "secrets", 1040 }, 1041 Providers: []apiserver.ProviderConfiguration{ 1042 { 1043 KMS: &apiserver.KMSConfiguration{ 1044 Name: "another-kms", 1045 APIVersion: "v1", 1046 Timeout: &metav1.Duration{Duration: 3 * time.Second}, 1047 Endpoint: "unix:///tmp/another-testprovider.sock", 1048 CacheSize: pointer.Int32(10), 1049 }, 1050 }, 1051 }, 1052 }, 1053 }, 1054 }, 1055 expectedError: "resource \"secrets\" is masked by earlier rule \"*.\"", 1056 }, 1057 { 1058 desc: "resources masked by *.*", 1059 config: &apiserver.EncryptionConfiguration{ 1060 Resources: []apiserver.ResourceConfiguration{ 1061 { 1062 Resources: []string{ 1063 "configmaps", 1064 "*.*", 1065 "secrets", 1066 }, 1067 Providers: []apiserver.ProviderConfiguration{ 1068 { 1069 KMS: &apiserver.KMSConfiguration{ 1070 Name: "kms", 1071 APIVersion: "v1", 1072 Timeout: &metav1.Duration{Duration: 3 * time.Second}, 1073 Endpoint: "unix:///tmp/testprovider.sock", 1074 CacheSize: pointer.Int32(10), 1075 }, 1076 }, 1077 }, 1078 }, 1079 }, 1080 }, 1081 expectedError: "resource \"secrets\" is masked by earlier rule \"*.*\"", 1082 }, 1083 { 1084 desc: "resources masked by *.* in multiple configurations", 1085 config: &apiserver.EncryptionConfiguration{ 1086 Resources: []apiserver.ResourceConfiguration{ 1087 { 1088 Resources: []string{ 1089 "configmaps", 1090 }, 1091 Providers: []apiserver.ProviderConfiguration{ 1092 { 1093 KMS: &apiserver.KMSConfiguration{ 1094 Name: "kms", 1095 APIVersion: "v1", 1096 Timeout: &metav1.Duration{Duration: 3 * time.Second}, 1097 Endpoint: "unix:///tmp/testprovider.sock", 1098 CacheSize: pointer.Int32(10), 1099 }, 1100 }, 1101 }, 1102 }, 1103 { 1104 Resources: []string{ 1105 "*.*", 1106 "secrets", 1107 }, 1108 Providers: []apiserver.ProviderConfiguration{ 1109 { 1110 KMS: &apiserver.KMSConfiguration{ 1111 Name: "another-kms", 1112 APIVersion: "v1", 1113 Timeout: &metav1.Duration{Duration: 3 * time.Second}, 1114 Endpoint: "unix:///tmp/another-testprovider.sock", 1115 CacheSize: pointer.Int32(10), 1116 }, 1117 }, 1118 }, 1119 }, 1120 }, 1121 }, 1122 expectedError: "resource \"secrets\" is masked by earlier rule \"*.*\"", 1123 }, 1124 { 1125 desc: "resources *. masked by *.*", 1126 config: &apiserver.EncryptionConfiguration{ 1127 Resources: []apiserver.ResourceConfiguration{ 1128 { 1129 Resources: []string{ 1130 "configmaps", 1131 "*.*", 1132 "*.", 1133 }, 1134 Providers: []apiserver.ProviderConfiguration{ 1135 { 1136 KMS: &apiserver.KMSConfiguration{ 1137 Name: "kms", 1138 APIVersion: "v1", 1139 Timeout: &metav1.Duration{Duration: 3 * time.Second}, 1140 Endpoint: "unix:///tmp/testprovider.sock", 1141 CacheSize: pointer.Int32(10), 1142 }, 1143 }, 1144 }, 1145 }, 1146 }, 1147 }, 1148 expectedError: "resource \"*.\" is masked by earlier rule \"*.*\"", 1149 }, 1150 { 1151 desc: "resources *. masked by *.* in multiple configurations", 1152 config: &apiserver.EncryptionConfiguration{ 1153 Resources: []apiserver.ResourceConfiguration{ 1154 { 1155 Resources: []string{ 1156 "configmaps", 1157 "*.*", 1158 }, 1159 Providers: []apiserver.ProviderConfiguration{ 1160 { 1161 KMS: &apiserver.KMSConfiguration{ 1162 Name: "kms", 1163 APIVersion: "v1", 1164 Timeout: &metav1.Duration{Duration: 3 * time.Second}, 1165 Endpoint: "unix:///tmp/testprovider.sock", 1166 CacheSize: pointer.Int32(10), 1167 }, 1168 }, 1169 }, 1170 }, 1171 { 1172 Resources: []string{ 1173 "*.", 1174 }, 1175 Providers: []apiserver.ProviderConfiguration{ 1176 { 1177 KMS: &apiserver.KMSConfiguration{ 1178 Name: "another-kms", 1179 APIVersion: "v1", 1180 Timeout: &metav1.Duration{Duration: 3 * time.Second}, 1181 Endpoint: "unix:///tmp/another-testprovider.sock", 1182 CacheSize: pointer.Int32(10), 1183 }, 1184 }, 1185 }, 1186 }, 1187 }, 1188 }, 1189 expectedError: "resource \"*.\" is masked by earlier rule \"*.*\"", 1190 }, 1191 { 1192 desc: "resources not masked by any rule", 1193 config: &apiserver.EncryptionConfiguration{ 1194 Resources: []apiserver.ResourceConfiguration{ 1195 { 1196 Resources: []string{ 1197 "configmaps", 1198 "secrets", 1199 "*.*", 1200 }, 1201 Providers: []apiserver.ProviderConfiguration{ 1202 { 1203 KMS: &apiserver.KMSConfiguration{ 1204 Name: "kms", 1205 APIVersion: "v1", 1206 Timeout: &metav1.Duration{Duration: 3 * time.Second}, 1207 Endpoint: "unix:///tmp/testprovider.sock", 1208 CacheSize: pointer.Int32(10), 1209 }, 1210 }, 1211 }, 1212 }, 1213 }, 1214 }, 1215 }, 1216 { 1217 desc: "resources not masked by any rule in multiple configurations", 1218 config: &apiserver.EncryptionConfiguration{ 1219 Resources: []apiserver.ResourceConfiguration{ 1220 { 1221 Resources: []string{ 1222 "configmaps", 1223 "secrets", 1224 }, 1225 Providers: []apiserver.ProviderConfiguration{ 1226 { 1227 KMS: &apiserver.KMSConfiguration{ 1228 Name: "kms", 1229 APIVersion: "v1", 1230 Timeout: &metav1.Duration{Duration: 3 * time.Second}, 1231 Endpoint: "unix:///tmp/testprovider.sock", 1232 CacheSize: pointer.Int32(10), 1233 }, 1234 }, 1235 }, 1236 }, 1237 { 1238 Resources: []string{ 1239 "*.*", 1240 }, 1241 Providers: []apiserver.ProviderConfiguration{ 1242 { 1243 KMS: &apiserver.KMSConfiguration{ 1244 Name: "another-kms", 1245 APIVersion: "v1", 1246 Timeout: &metav1.Duration{Duration: 3 * time.Second}, 1247 Endpoint: "unix:///tmp/another-testprovider.sock", 1248 CacheSize: pointer.Int32(10), 1249 }, 1250 }, 1251 }, 1252 }, 1253 }, 1254 }, 1255 }, 1256 } 1257 1258 for _, tc := range testCases { 1259 t.Run(tc.desc, func(t *testing.T) { 1260 ctx, cancel := context.WithCancel(context.Background()) 1261 t.Cleanup(cancel) 1262 1263 _, _, _, err := getTransformerOverridesAndKMSPluginProbes(ctx, tc.config, "") 1264 if errString(err) != tc.expectedError { 1265 t.Errorf("expected error %s but got %s", tc.expectedError, errString(err)) 1266 } 1267 }) 1268 } 1269 } 1270 1271 func TestWildcardStructure(t *testing.T) { 1272 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true) 1273 testCases := []struct { 1274 desc string 1275 expectedResourceTransformers map[string]string 1276 config *apiserver.EncryptionConfiguration 1277 errorValue string 1278 }{ 1279 { 1280 desc: "should not result in error", 1281 expectedResourceTransformers: map[string]string{ 1282 "configmaps": "k8s:enc:kms:v1:kms:", 1283 "secrets": "k8s:enc:kms:v1:another-kms:", 1284 "events": "k8s:enc:kms:v1:fancy:", 1285 "deployments.apps": "k8s:enc:kms:v1:kms:", 1286 "pods": "k8s:enc:kms:v1:fancy:", 1287 "pandas": "k8s:enc:kms:v1:fancy:", 1288 "pandas.bears": "k8s:enc:kms:v1:yet-another-provider:", 1289 "jobs.apps": "k8s:enc:kms:v1:kms:", 1290 }, 1291 1292 errorValue: "", 1293 config: &apiserver.EncryptionConfiguration{ 1294 Resources: []apiserver.ResourceConfiguration{ 1295 { 1296 Resources: []string{ 1297 "configmaps", 1298 "*.apps", 1299 }, 1300 Providers: []apiserver.ProviderConfiguration{ 1301 { 1302 KMS: &apiserver.KMSConfiguration{ 1303 Name: "kms", 1304 APIVersion: "v1", 1305 Timeout: &metav1.Duration{Duration: 3 * time.Second}, 1306 Endpoint: "unix:///tmp/testprovider.sock", 1307 CacheSize: pointer.Int32(10), 1308 }, 1309 }, 1310 }, 1311 }, 1312 { 1313 Resources: []string{ 1314 "secrets", 1315 }, 1316 Providers: []apiserver.ProviderConfiguration{ 1317 { 1318 KMS: &apiserver.KMSConfiguration{ 1319 Name: "another-kms", 1320 APIVersion: "v1", 1321 Timeout: &metav1.Duration{Duration: 3 * time.Second}, 1322 Endpoint: "unix:///tmp/testprovider.sock", 1323 CacheSize: pointer.Int32(10), 1324 }, 1325 }, 1326 { 1327 Identity: &apiserver.IdentityConfiguration{}, 1328 }, 1329 }, 1330 }, 1331 { 1332 Resources: []string{ 1333 "*.", 1334 }, 1335 Providers: []apiserver.ProviderConfiguration{ 1336 { 1337 KMS: &apiserver.KMSConfiguration{ 1338 Name: "fancy", 1339 APIVersion: "v1", 1340 Timeout: &metav1.Duration{Duration: 3 * time.Second}, 1341 Endpoint: "unix:///tmp/testprovider.sock", 1342 CacheSize: pointer.Int32(10), 1343 }, 1344 }, 1345 }, 1346 }, 1347 { 1348 Resources: []string{ 1349 "*.*", 1350 }, 1351 Providers: []apiserver.ProviderConfiguration{ 1352 { 1353 KMS: &apiserver.KMSConfiguration{ 1354 Name: "yet-another-provider", 1355 APIVersion: "v1", 1356 Timeout: &metav1.Duration{Duration: 3 * time.Second}, 1357 Endpoint: "unix:///tmp/testprovider.sock", 1358 CacheSize: pointer.Int32(10), 1359 }, 1360 }, 1361 }, 1362 }, 1363 }, 1364 }, 1365 }, 1366 { 1367 desc: "should result in error", 1368 errorValue: "resource \"secrets\" is masked by earlier rule \"*.\"", 1369 config: &apiserver.EncryptionConfiguration{ 1370 Resources: []apiserver.ResourceConfiguration{ 1371 { 1372 Resources: []string{ 1373 "configmaps", 1374 "*.", 1375 }, 1376 Providers: []apiserver.ProviderConfiguration{ 1377 { 1378 KMS: &apiserver.KMSConfiguration{ 1379 Name: "kms", 1380 APIVersion: "v1", 1381 Timeout: &metav1.Duration{Duration: 3 * time.Second}, 1382 Endpoint: "unix:///tmp/testprovider.sock", 1383 CacheSize: pointer.Int32(10), 1384 }, 1385 }, 1386 }, 1387 }, 1388 { 1389 Resources: []string{ 1390 "*.*", 1391 "secrets", 1392 }, 1393 Providers: []apiserver.ProviderConfiguration{ 1394 { 1395 KMS: &apiserver.KMSConfiguration{ 1396 Name: "kms", 1397 APIVersion: "v1", 1398 Timeout: &metav1.Duration{Duration: 3 * time.Second}, 1399 Endpoint: "unix:///tmp/testprovider.sock", 1400 CacheSize: pointer.Int32(10), 1401 }, 1402 }, 1403 { 1404 Identity: &apiserver.IdentityConfiguration{}, 1405 }, 1406 }, 1407 }, 1408 }, 1409 }, 1410 }, 1411 } 1412 1413 for _, tc := range testCases { 1414 t.Run(tc.desc, func(t *testing.T) { 1415 ctx, cancel := context.WithCancel(context.Background()) 1416 t.Cleanup(cancel) 1417 1418 transformers, _, _, err := getTransformerOverridesAndKMSPluginProbes(ctx, tc.config, "") 1419 if errString(err) != tc.errorValue { 1420 t.Errorf("expected error %s but got %s", tc.errorValue, errString(err)) 1421 } 1422 1423 if len(tc.errorValue) > 0 { 1424 return 1425 } 1426 1427 // check if expectedResourceTransformers are present 1428 for resource, expectedTransformerName := range tc.expectedResourceTransformers { 1429 transformer := transformerFromOverrides(transformers, schema.ParseGroupResource(resource)) 1430 transformerName := string( 1431 reflect.ValueOf(transformer).Elem().FieldByName("transformers").Index(0).FieldByName("Prefix").Bytes(), 1432 ) 1433 1434 if transformerName != expectedTransformerName { 1435 t.Errorf("resource %s: expected same transformer name but got %v", resource, cmp.Diff(transformerName, expectedTransformerName)) 1436 } 1437 } 1438 }) 1439 } 1440 } 1441 1442 func TestKMSPluginHealthzTTL(t *testing.T) { 1443 ctx := testContext(t) 1444 1445 service, _ := newMockEnvelopeService(ctx, "unix:///tmp/testprovider.sock", 3*time.Second) 1446 errService, _ := newMockErrorEnvelopeService("unix:///tmp/testprovider.sock", 3*time.Second) 1447 1448 testCases := []struct { 1449 desc string 1450 probe *kmsPluginProbe 1451 wantTTL time.Duration 1452 }{ 1453 { 1454 desc: "kms provider in good state", 1455 probe: &kmsPluginProbe{ 1456 name: "test", 1457 ttl: kmsPluginHealthzNegativeTTL, 1458 service: service, 1459 l: &sync.Mutex{}, 1460 lastResponse: &kmsPluginHealthzResponse{}, 1461 }, 1462 wantTTL: kmsPluginHealthzPositiveTTL, 1463 }, 1464 { 1465 desc: "kms provider in bad state", 1466 probe: &kmsPluginProbe{ 1467 name: "test", 1468 ttl: kmsPluginHealthzPositiveTTL, 1469 service: errService, 1470 l: &sync.Mutex{}, 1471 lastResponse: &kmsPluginHealthzResponse{}, 1472 }, 1473 wantTTL: kmsPluginHealthzNegativeTTL, 1474 }, 1475 } 1476 1477 for _, tt := range testCases { 1478 t.Run(tt.desc, func(t *testing.T) { 1479 _ = tt.probe.check() 1480 if tt.probe.ttl != tt.wantTTL { 1481 t.Fatalf("want ttl %v, got ttl %v", tt.wantTTL, tt.probe.ttl) 1482 } 1483 }) 1484 } 1485 } 1486 1487 func TestKMSv2PluginHealthzTTL(t *testing.T) { 1488 ctx := testContext(t) 1489 1490 service, _ := newMockEnvelopeKMSv2Service(ctx, "unix:///tmp/testprovider.sock", "providerName", 3*time.Second) 1491 errService, _ := newMockErrorEnvelopeKMSv2Service("unix:///tmp/testprovider.sock", 3*time.Second) 1492 1493 testCases := []struct { 1494 desc string 1495 probe *kmsv2PluginProbe 1496 wantTTL time.Duration 1497 }{ 1498 { 1499 desc: "kmsv2 provider in good state", 1500 probe: &kmsv2PluginProbe{ 1501 name: "test", 1502 ttl: kmsPluginHealthzNegativeTTL, 1503 service: service, 1504 l: &sync.Mutex{}, 1505 lastResponse: &kmsPluginHealthzResponse{}, 1506 }, 1507 wantTTL: kmsPluginHealthzPositiveTTL, 1508 }, 1509 { 1510 desc: "kmsv2 provider in bad state", 1511 probe: &kmsv2PluginProbe{ 1512 name: "test", 1513 ttl: kmsPluginHealthzPositiveTTL, 1514 service: errService, 1515 l: &sync.Mutex{}, 1516 lastResponse: &kmsPluginHealthzResponse{}, 1517 }, 1518 wantTTL: kmsPluginHealthzNegativeTTL, 1519 }, 1520 } 1521 1522 for _, tt := range testCases { 1523 t.Run(tt.desc, func(t *testing.T) { 1524 tt.probe.state.Store(&envelopekmsv2.State{}) 1525 _ = tt.probe.check(ctx) 1526 if tt.probe.ttl != tt.wantTTL { 1527 t.Fatalf("want ttl %v, got ttl %v", tt.wantTTL, tt.probe.ttl) 1528 } 1529 }) 1530 } 1531 } 1532 1533 func TestKMSv2InvalidKeyID(t *testing.T) { 1534 ctx := testContext(t) 1535 invalidKeyIDService, _ := newMockInvalidKeyIDEnvelopeKMSv2Service(ctx, "unix:///tmp/testprovider.sock", 3*time.Second, "") 1536 invalidLongKeyIDService, _ := newMockInvalidKeyIDEnvelopeKMSv2Service(ctx, "unix:///tmp/testprovider.sock", 3*time.Second, sampleInvalidKeyID) 1537 service, _ := newMockInvalidKeyIDEnvelopeKMSv2Service(ctx, "unix:///tmp/testprovider.sock", 3*time.Second, "1") 1538 1539 testCases := []struct { 1540 desc string 1541 probe *kmsv2PluginProbe 1542 metrics []string 1543 want string 1544 }{ 1545 { 1546 desc: "kmsv2 provider returns an invalid empty keyID", 1547 probe: &kmsv2PluginProbe{ 1548 name: "test", 1549 ttl: kmsPluginHealthzNegativeTTL, 1550 service: invalidKeyIDService, 1551 l: &sync.Mutex{}, 1552 lastResponse: &kmsPluginHealthzResponse{}, 1553 }, 1554 metrics: []string{ 1555 "apiserver_envelope_encryption_invalid_key_id_from_status_total", 1556 }, 1557 want: ` 1558 # HELP apiserver_envelope_encryption_invalid_key_id_from_status_total [ALPHA] Number of times an invalid keyID is returned by the Status RPC call split by error. 1559 # TYPE apiserver_envelope_encryption_invalid_key_id_from_status_total counter 1560 apiserver_envelope_encryption_invalid_key_id_from_status_total{error="empty",provider_name="test"} 1 1561 `, 1562 }, 1563 { 1564 desc: "kmsv2 provider returns a valid keyID", 1565 probe: &kmsv2PluginProbe{ 1566 name: "test", 1567 ttl: kmsPluginHealthzNegativeTTL, 1568 service: service, 1569 l: &sync.Mutex{}, 1570 lastResponse: &kmsPluginHealthzResponse{}, 1571 }, 1572 metrics: []string{ 1573 "apiserver_envelope_encryption_invalid_key_id_from_status_total", 1574 }, 1575 want: ``, 1576 }, 1577 { 1578 desc: "kmsv2 provider returns an invalid long keyID", 1579 probe: &kmsv2PluginProbe{ 1580 name: "test", 1581 ttl: kmsPluginHealthzNegativeTTL, 1582 service: invalidLongKeyIDService, 1583 l: &sync.Mutex{}, 1584 lastResponse: &kmsPluginHealthzResponse{}, 1585 }, 1586 metrics: []string{ 1587 "apiserver_envelope_encryption_invalid_key_id_from_status_total", 1588 }, 1589 want: ` 1590 # HELP apiserver_envelope_encryption_invalid_key_id_from_status_total [ALPHA] Number of times an invalid keyID is returned by the Status RPC call split by error. 1591 # TYPE apiserver_envelope_encryption_invalid_key_id_from_status_total counter 1592 apiserver_envelope_encryption_invalid_key_id_from_status_total{error="too_long",provider_name="test"} 1 1593 `, 1594 }, 1595 } 1596 1597 metrics.InvalidKeyIDFromStatusTotal.Reset() 1598 metrics.RegisterMetrics() 1599 1600 for _, tt := range testCases { 1601 t.Run(tt.desc, func(t *testing.T) { 1602 defer metrics.InvalidKeyIDFromStatusTotal.Reset() 1603 tt.probe.state.Store(&envelopekmsv2.State{}) 1604 _ = tt.probe.check(ctx) 1605 if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tt.want), tt.metrics...); err != nil { 1606 t.Fatal(err) 1607 } 1608 }) 1609 } 1610 } 1611 1612 func TestCBCKeyRotationWithOverlappingProviders(t *testing.T) { 1613 testCBCKeyRotationWithProviders( 1614 t, 1615 "testdata/valid-configs/aes/aes-cbc-multiple-providers.json", 1616 "k8s:enc:aescbc:v1:1:", 1617 "testdata/valid-configs/aes/aes-cbc-multiple-providers-reversed.json", 1618 "k8s:enc:aescbc:v1:2:", 1619 ) 1620 } 1621 1622 func TestCBCKeyRotationWithoutOverlappingProviders(t *testing.T) { 1623 testCBCKeyRotationWithProviders( 1624 t, 1625 "testdata/valid-configs/aes/aes-cbc-multiple-keys.json", 1626 "k8s:enc:aescbc:v1:A:", 1627 "testdata/valid-configs/aes/aes-cbc-multiple-keys-reversed.json", 1628 "k8s:enc:aescbc:v1:B:", 1629 ) 1630 } 1631 1632 func testCBCKeyRotationWithProviders(t *testing.T, firstEncryptionConfig, firstPrefix, secondEncryptionConfig, secondPrefix string) { 1633 p := getTransformerFromEncryptionConfig(t, firstEncryptionConfig) 1634 1635 ctx := testContext(t) 1636 dataCtx := value.DefaultContext("authenticated_data") 1637 1638 out, err := p.TransformToStorage(ctx, []byte("firstvalue"), dataCtx) 1639 if err != nil { 1640 t.Fatal(err) 1641 } 1642 if !bytes.HasPrefix(out, []byte(firstPrefix)) { 1643 t.Fatalf("unexpected prefix: %q", out) 1644 } 1645 from, stale, err := p.TransformFromStorage(ctx, out, dataCtx) 1646 if err != nil { 1647 t.Fatal(err) 1648 } 1649 if stale || !bytes.Equal([]byte("firstvalue"), from) { 1650 t.Fatalf("unexpected data: %t %q", stale, from) 1651 } 1652 1653 // verify changing the context fails storage 1654 _, _, err = p.TransformFromStorage(ctx, out, value.DefaultContext("incorrect_context")) 1655 if err != nil { 1656 t.Fatalf("CBC mode does not support authentication: %v", err) 1657 } 1658 1659 // reverse the order, use the second key 1660 p = getTransformerFromEncryptionConfig(t, secondEncryptionConfig) 1661 from, stale, err = p.TransformFromStorage(ctx, out, dataCtx) 1662 if err != nil { 1663 t.Fatal(err) 1664 } 1665 if !stale || !bytes.Equal([]byte("firstvalue"), from) { 1666 t.Fatalf("unexpected data: %t %q", stale, from) 1667 } 1668 1669 out, err = p.TransformToStorage(ctx, []byte("firstvalue"), dataCtx) 1670 if err != nil { 1671 t.Fatal(err) 1672 } 1673 if !bytes.HasPrefix(out, []byte(secondPrefix)) { 1674 t.Fatalf("unexpected prefix: %q", out) 1675 } 1676 from, stale, err = p.TransformFromStorage(ctx, out, dataCtx) 1677 if err != nil { 1678 t.Fatal(err) 1679 } 1680 if stale || !bytes.Equal([]byte("firstvalue"), from) { 1681 t.Fatalf("unexpected data: %t %q", stale, from) 1682 } 1683 } 1684 1685 func getTransformerFromEncryptionConfig(t *testing.T, encryptionConfigPath string) value.Transformer { 1686 ctx := testContext(t) 1687 1688 t.Helper() 1689 encryptionConfiguration, err := LoadEncryptionConfig(ctx, encryptionConfigPath, false, "") 1690 if err != nil { 1691 t.Fatal(err) 1692 } 1693 if len(encryptionConfiguration.Transformers) != 1 { 1694 t.Fatalf("input config does not have exactly one resource: %s", encryptionConfigPath) 1695 } 1696 for _, transformer := range encryptionConfiguration.Transformers { 1697 return transformer 1698 } 1699 panic("unreachable") 1700 } 1701 1702 func TestIsKMSv2ProviderHealthyError(t *testing.T) { 1703 probe := &kmsv2PluginProbe{name: "testplugin"} 1704 1705 testCases := []struct { 1706 desc string 1707 expectedErr string 1708 wantMetrics string 1709 statusResponse *kmsservice.StatusResponse 1710 }{ 1711 { 1712 desc: "healthz status is not ok", 1713 statusResponse: &kmsservice.StatusResponse{ 1714 Healthz: "unhealthy", 1715 }, 1716 expectedErr: "got unexpected healthz status: unhealthy, expected KMSv2 API version v2, got , got invalid KMSv2 KeyID ", 1717 wantMetrics: ` 1718 # HELP apiserver_envelope_encryption_invalid_key_id_from_status_total [ALPHA] Number of times an invalid keyID is returned by the Status RPC call split by error. 1719 # TYPE apiserver_envelope_encryption_invalid_key_id_from_status_total counter 1720 apiserver_envelope_encryption_invalid_key_id_from_status_total{error="empty",provider_name="testplugin"} 1 1721 `, 1722 }, 1723 { 1724 desc: "version is not v2", 1725 statusResponse: &kmsservice.StatusResponse{ 1726 Version: "v1beta1", 1727 }, 1728 expectedErr: "got unexpected healthz status: , expected KMSv2 API version v2, got v1beta1, got invalid KMSv2 KeyID ", 1729 wantMetrics: ` 1730 # HELP apiserver_envelope_encryption_invalid_key_id_from_status_total [ALPHA] Number of times an invalid keyID is returned by the Status RPC call split by error. 1731 # TYPE apiserver_envelope_encryption_invalid_key_id_from_status_total counter 1732 apiserver_envelope_encryption_invalid_key_id_from_status_total{error="empty",provider_name="testplugin"} 1 1733 `, 1734 }, 1735 { 1736 desc: "missing keyID", 1737 statusResponse: &kmsservice.StatusResponse{ 1738 Healthz: "ok", 1739 Version: "v2beta1", 1740 }, 1741 expectedErr: "got invalid KMSv2 KeyID ", 1742 wantMetrics: ` 1743 # HELP apiserver_envelope_encryption_invalid_key_id_from_status_total [ALPHA] Number of times an invalid keyID is returned by the Status RPC call split by error. 1744 # TYPE apiserver_envelope_encryption_invalid_key_id_from_status_total counter 1745 apiserver_envelope_encryption_invalid_key_id_from_status_total{error="empty",provider_name="testplugin"} 1 1746 `, 1747 }, 1748 { 1749 desc: "invalid long keyID", 1750 statusResponse: &kmsservice.StatusResponse{ 1751 Healthz: "ok", 1752 Version: "v2", 1753 KeyID: sampleInvalidKeyID, 1754 }, 1755 expectedErr: "got invalid KMSv2 KeyID ", 1756 wantMetrics: ` 1757 # HELP apiserver_envelope_encryption_invalid_key_id_from_status_total [ALPHA] Number of times an invalid keyID is returned by the Status RPC call split by error. 1758 # TYPE apiserver_envelope_encryption_invalid_key_id_from_status_total counter 1759 apiserver_envelope_encryption_invalid_key_id_from_status_total{error="too_long",provider_name="testplugin"} 1 1760 `, 1761 }, 1762 } 1763 1764 for _, tt := range testCases { 1765 t.Run(tt.desc, func(t *testing.T) { 1766 metrics.InvalidKeyIDFromStatusTotal.Reset() 1767 err := probe.isKMSv2ProviderHealthyAndMaybeRotateDEK(testContext(t), tt.statusResponse) 1768 if !strings.Contains(errString(err), tt.expectedErr) { 1769 t.Errorf("expected err %q, got %q", tt.expectedErr, errString(err)) 1770 } 1771 if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tt.wantMetrics), 1772 "apiserver_envelope_encryption_invalid_key_id_from_status_total", 1773 ); err != nil { 1774 t.Fatal(err) 1775 } 1776 }) 1777 } 1778 } 1779 1780 // test to ensure KMSv2 API version is not changed after the first status response 1781 func TestKMSv2SameVersionFromStatus(t *testing.T) { 1782 probe := &kmsv2PluginProbe{name: "testplugin"} 1783 service, _ := newMockEnvelopeKMSv2Service(testContext(t), "unix:///tmp/testprovider.sock", "providerName", 3*time.Second) 1784 probe.l = &sync.Mutex{} 1785 probe.state.Store(&envelopekmsv2.State{}) 1786 probe.service = service 1787 1788 testCases := []struct { 1789 desc string 1790 expectedErr string 1791 newVersion string 1792 }{ 1793 { 1794 desc: "version changed", 1795 newVersion: "v2", 1796 expectedErr: "KMSv2 API version should not change", 1797 }, 1798 { 1799 desc: "version unchanged", 1800 newVersion: "v2beta1", 1801 expectedErr: "", 1802 }, 1803 } 1804 for _, tt := range testCases { 1805 t.Run(tt.desc, func(t *testing.T) { 1806 statusResponse := &kmsservice.StatusResponse{ 1807 Healthz: "ok", 1808 Version: "v2beta1", 1809 KeyID: "1", 1810 } 1811 if err := probe.isKMSv2ProviderHealthyAndMaybeRotateDEK(testContext(t), statusResponse); err != nil { 1812 t.Fatal(err) 1813 } 1814 statusResponse.Version = tt.newVersion 1815 err := probe.isKMSv2ProviderHealthyAndMaybeRotateDEK(testContext(t), statusResponse) 1816 if len(tt.expectedErr) > 0 && !strings.Contains(errString(err), tt.expectedErr) { 1817 t.Errorf("expected err %q, got %q", tt.expectedErr, errString(err)) 1818 } 1819 if len(tt.expectedErr) == 0 && err != nil { 1820 t.Fatal(err) 1821 } 1822 }) 1823 } 1824 } 1825 1826 func testContext(t *testing.T) context.Context { 1827 ctx, cancel := context.WithCancel(context.Background()) 1828 t.Cleanup(cancel) 1829 return ctx 1830 } 1831 1832 func errString(err error) string { 1833 if err == nil { 1834 return "" 1835 } 1836 1837 return err.Error() 1838 } 1839 1840 func TestComputeEncryptionConfigHash(t *testing.T) { 1841 // hash the empty string to be sure that sha256 is being used 1842 expect := "k8s:enc:unstable:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" 1843 sum := computeEncryptionConfigHash([]byte("")) 1844 if expect != sum { 1845 t.Errorf("expected hash %q but got %q", expect, sum) 1846 } 1847 } 1848 1849 func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) { 1850 defaultUseSeed := GetKDF() 1851 1852 origNowFunc := envelopekmsv2.NowFunc 1853 now := origNowFunc() // freeze time 1854 t.Cleanup(func() { envelopekmsv2.NowFunc = origNowFunc }) 1855 envelopekmsv2.NowFunc = func() time.Time { return now } 1856 1857 klog.LogToStderr(false) 1858 var level klog.Level 1859 if err := level.Set("6"); err != nil { 1860 t.Fatal(err) 1861 } 1862 t.Cleanup(func() { 1863 klog.LogToStderr(true) 1864 if err := level.Set("0"); err != nil { 1865 t.Fatal(err) 1866 } 1867 klog.SetOutput(io.Discard) 1868 }) 1869 1870 tests := []struct { 1871 name string 1872 service *testKMSv2EnvelopeService 1873 state envelopekmsv2.State 1874 useSeed bool 1875 statusKeyID string 1876 wantState envelopekmsv2.State 1877 wantEncryptCalls int 1878 wantLogs []string 1879 wantErr string 1880 }{ 1881 { 1882 name: "happy path, no previous state", 1883 service: &testKMSv2EnvelopeService{keyID: "1"}, 1884 state: envelopekmsv2.State{}, 1885 statusKeyID: "1", 1886 wantState: envelopekmsv2.State{ 1887 EncryptedObject: kmstypes.EncryptedObject{KeyID: "1"}, 1888 ExpirationTimestamp: now.Add(3 * time.Minute), 1889 }, 1890 wantEncryptCalls: 1, 1891 wantLogs: []string{ 1892 `"encrypting content using envelope service" uid="panda"`, 1893 fmt.Sprintf(`"successfully rotated DEK" uid="panda" useSeed=false newKeyIDHash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b" oldKeyIDHash="" expirationTimestamp="%s"`, 1894 now.Add(3*time.Minute).Format(time.RFC3339)), 1895 }, 1896 wantErr: "", 1897 }, 1898 { 1899 name: "happy path, with previous state", 1900 service: &testKMSv2EnvelopeService{err: fmt.Errorf("broken")}, // not called 1901 state: validState(t, "2", now, false), 1902 statusKeyID: "2", 1903 wantState: envelopekmsv2.State{ 1904 EncryptedObject: kmstypes.EncryptedObject{KeyID: "2"}, 1905 ExpirationTimestamp: now.Add(3 * time.Minute), 1906 }, 1907 wantEncryptCalls: 0, 1908 wantLogs: nil, 1909 wantErr: "", 1910 }, 1911 { 1912 name: "happy path, with previous state, useSeed=true", 1913 service: &testKMSv2EnvelopeService{keyID: "2"}, 1914 state: validState(t, "2", now, false), 1915 useSeed: true, 1916 statusKeyID: "2", 1917 wantState: envelopekmsv2.State{ 1918 EncryptedObject: kmstypes.EncryptedObject{KeyID: "2", EncryptedDEKSourceType: kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED}, 1919 ExpirationTimestamp: now.Add(3 * time.Minute), 1920 }, 1921 wantEncryptCalls: 1, 1922 wantLogs: []string{ 1923 `"encrypting content using envelope service" uid="panda"`, 1924 fmt.Sprintf(`"successfully rotated DEK" uid="panda" useSeed=true newKeyIDHash="sha256:d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35" oldKeyIDHash="sha256:d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35" expirationTimestamp="%s"`, 1925 now.Add(3*time.Minute).Format(time.RFC3339)), 1926 }, 1927 wantErr: "", 1928 }, 1929 { 1930 name: "happy path, with previous useSeed=true state", 1931 service: &testKMSv2EnvelopeService{keyID: "2"}, 1932 state: validState(t, "2", now, true), 1933 statusKeyID: "2", 1934 wantState: envelopekmsv2.State{ 1935 EncryptedObject: kmstypes.EncryptedObject{KeyID: "2"}, 1936 ExpirationTimestamp: now.Add(3 * time.Minute), 1937 }, 1938 wantEncryptCalls: 1, 1939 wantLogs: []string{ 1940 `"encrypting content using envelope service" uid="panda"`, 1941 fmt.Sprintf(`"successfully rotated DEK" uid="panda" useSeed=false newKeyIDHash="sha256:d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35" oldKeyIDHash="sha256:d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35" expirationTimestamp="%s"`, 1942 now.Add(3*time.Minute).Format(time.RFC3339)), 1943 }, 1944 wantErr: "", 1945 }, 1946 { 1947 name: "happy path, with previous useSeed=true state, useSeed=true", 1948 service: &testKMSv2EnvelopeService{keyID: "2"}, 1949 state: validState(t, "2", now, true), 1950 useSeed: true, 1951 statusKeyID: "2", 1952 wantState: envelopekmsv2.State{ 1953 EncryptedObject: kmstypes.EncryptedObject{KeyID: "2", EncryptedDEKSourceType: kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED}, 1954 ExpirationTimestamp: now.Add(3 * time.Minute), 1955 }, 1956 wantEncryptCalls: 0, 1957 wantLogs: nil, 1958 wantErr: "", 1959 }, 1960 { 1961 name: "happy path, with previous state, useSeed=default", 1962 service: &testKMSv2EnvelopeService{keyID: "2"}, 1963 state: validState(t, "2", now, false), 1964 useSeed: defaultUseSeed, 1965 statusKeyID: "2", 1966 wantState: envelopekmsv2.State{ 1967 EncryptedObject: kmstypes.EncryptedObject{KeyID: "2", EncryptedDEKSourceType: kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED}, 1968 ExpirationTimestamp: now.Add(3 * time.Minute), 1969 }, 1970 wantEncryptCalls: 1, 1971 wantLogs: []string{ 1972 `"encrypting content using envelope service" uid="panda"`, 1973 fmt.Sprintf(`"successfully rotated DEK" uid="panda" useSeed=true newKeyIDHash="sha256:d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35" oldKeyIDHash="sha256:d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35" expirationTimestamp="%s"`, 1974 now.Add(3*time.Minute).Format(time.RFC3339)), 1975 }, 1976 wantErr: "", 1977 }, 1978 { 1979 name: "happy path, with previous useSeed=true state, useSeed=default", 1980 service: &testKMSv2EnvelopeService{keyID: "2"}, 1981 state: validState(t, "2", now, true), 1982 useSeed: defaultUseSeed, 1983 statusKeyID: "2", 1984 wantState: envelopekmsv2.State{ 1985 EncryptedObject: kmstypes.EncryptedObject{KeyID: "2", EncryptedDEKSourceType: kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED}, 1986 ExpirationTimestamp: now.Add(3 * time.Minute), 1987 }, 1988 wantEncryptCalls: 0, 1989 wantLogs: nil, 1990 wantErr: "", 1991 }, 1992 { 1993 name: "previous state expired but key ID matches", 1994 service: &testKMSv2EnvelopeService{err: fmt.Errorf("broken")}, // not called 1995 state: validState(t, "3", now.Add(-time.Hour), false), 1996 statusKeyID: "3", 1997 wantState: envelopekmsv2.State{ 1998 EncryptedObject: kmstypes.EncryptedObject{KeyID: "3"}, 1999 ExpirationTimestamp: now.Add(3 * time.Minute), 2000 }, 2001 wantEncryptCalls: 0, 2002 wantLogs: nil, 2003 wantErr: "", 2004 }, 2005 { 2006 name: "previous state expired but key ID does not match", 2007 service: &testKMSv2EnvelopeService{keyID: "4"}, 2008 state: validState(t, "3", now.Add(-time.Hour), false), 2009 statusKeyID: "4", 2010 wantState: envelopekmsv2.State{ 2011 EncryptedObject: kmstypes.EncryptedObject{KeyID: "4"}, 2012 ExpirationTimestamp: now.Add(3 * time.Minute), 2013 }, 2014 wantEncryptCalls: 1, 2015 wantLogs: []string{ 2016 `"encrypting content using envelope service" uid="panda"`, 2017 fmt.Sprintf(`"successfully rotated DEK" uid="panda" useSeed=false newKeyIDHash="sha256:4b227777d4dd1fc61c6f884f48641d02b4d121d3fd328cb08b5531fcacdabf8a" oldKeyIDHash="sha256:4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce" expirationTimestamp="%s"`, 2018 now.Add(3*time.Minute).Format(time.RFC3339)), 2019 }, 2020 wantErr: "", 2021 }, 2022 { 2023 name: "service down but key ID does not match", 2024 service: &testKMSv2EnvelopeService{err: fmt.Errorf("broken")}, 2025 state: validState(t, "4", now.Add(7*time.Minute), false), 2026 statusKeyID: "5", 2027 wantState: envelopekmsv2.State{ 2028 EncryptedObject: kmstypes.EncryptedObject{KeyID: "4"}, 2029 ExpirationTimestamp: now.Add(7 * time.Minute), 2030 }, 2031 wantEncryptCalls: 1, 2032 wantLogs: []string{ 2033 `"encrypting content using envelope service" uid="panda"`, 2034 }, 2035 wantErr: `failed to rotate DEK uid="panda", useSeed=false, ` + 2036 `errState=<nil>, errGen=failed to encrypt DEK, error: broken, statusKeyIDHash="sha256:ef2d127de37b942baad06145e54b0c619a1f22327b2ebbcfbec78f5564afe39d", ` + 2037 `encryptKeyIDHash="", stateKeyIDHash="sha256:4b227777d4dd1fc61c6f884f48641d02b4d121d3fd328cb08b5531fcacdabf8a", expirationTimestamp=` + now.Add(7*time.Minute).Format(time.RFC3339), 2038 }, 2039 { 2040 name: "invalid service response, no previous state", 2041 service: &testKMSv2EnvelopeService{keyID: "1", encryptAnnotations: map[string][]byte{"panda": nil}}, 2042 state: envelopekmsv2.State{}, 2043 statusKeyID: "1", 2044 wantState: envelopekmsv2.State{}, 2045 wantEncryptCalls: 1, 2046 wantLogs: []string{ 2047 `"encrypting content using envelope service" uid="panda"`, 2048 }, 2049 wantErr: `failed to rotate DEK uid="panda", useSeed=false, ` + 2050 `errState=got unexpected nil transformer, errGen=failed to validate annotations: annotations: Invalid value: "panda": ` + 2051 `should be a domain with at least two segments separated by dots, statusKeyIDHash="sha256:6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b", ` + 2052 `encryptKeyIDHash="", stateKeyIDHash="", expirationTimestamp=` + (time.Time{}).Format(time.RFC3339), 2053 }, 2054 { 2055 name: "invalid service response, with previous state", 2056 service: &testKMSv2EnvelopeService{keyID: "3", encryptAnnotations: map[string][]byte{"panda": nil}}, 2057 state: validState(t, "2", now, false), 2058 statusKeyID: "3", 2059 wantState: envelopekmsv2.State{ 2060 EncryptedObject: kmstypes.EncryptedObject{KeyID: "2"}, 2061 ExpirationTimestamp: now, 2062 }, 2063 wantEncryptCalls: 1, 2064 wantLogs: []string{ 2065 `"encrypting content using envelope service" uid="panda"`, 2066 }, 2067 wantErr: `failed to rotate DEK uid="panda", useSeed=false, ` + 2068 `errState=<nil>, errGen=failed to validate annotations: annotations: Invalid value: "panda": ` + 2069 `should be a domain with at least two segments separated by dots, statusKeyIDHash="sha256:4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce", ` + 2070 `encryptKeyIDHash="", stateKeyIDHash="sha256:d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35", expirationTimestamp=` + now.Format(time.RFC3339), 2071 }, 2072 } 2073 for _, tt := range tests { 2074 t.Run(tt.name, func(t *testing.T) { 2075 defer SetKDFForTests(tt.useSeed)() 2076 2077 var buf bytes.Buffer 2078 klog.SetOutput(&buf) 2079 2080 ctx := testContext(t) 2081 2082 h := &kmsv2PluginProbe{ 2083 name: "panda", 2084 service: tt.service, 2085 } 2086 h.state.Store(&tt.state) 2087 2088 err := h.rotateDEKOnKeyIDChange(ctx, tt.statusKeyID, "panda") 2089 2090 klog.Flush() 2091 klog.SetOutput(io.Discard) // prevent further writes into buf 2092 2093 if diff := cmp.Diff(tt.wantLogs, logLines(buf.String())); len(diff) > 0 { 2094 t.Errorf("log mismatch (-want +got):\n%s", diff) 2095 } 2096 2097 ignoredFields := sets.NewString("Transformer", "EncryptedObject.EncryptedDEKSource", "UID", "CacheKey") 2098 2099 gotState := *h.state.Load() 2100 2101 if diff := cmp.Diff(tt.wantState, gotState, 2102 cmp.FilterPath(func(path cmp.Path) bool { return ignoredFields.Has(path.String()) }, cmp.Ignore()), 2103 ); len(diff) > 0 { 2104 t.Errorf("state mismatch (-want +got):\n%s", diff) 2105 } 2106 2107 if len(cmp.Diff(tt.wantState, gotState)) > 0 { // we only need to run this check when the state changes 2108 validCiphertext := len(gotState.EncryptedObject.EncryptedDEKSource) > 0 2109 if tt.useSeed { 2110 validCiphertext = validCiphertext && gotState.EncryptedObject.EncryptedDEKSourceType == kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED 2111 } else { 2112 validCiphertext = validCiphertext && gotState.EncryptedObject.EncryptedDEKSourceType == kmstypes.EncryptedDEKSourceType_AES_GCM_KEY 2113 } 2114 if !validCiphertext { 2115 t.Errorf("invalid ciphertext with useSeed=%v, encryptedDEKSourceLen=%d, encryptedDEKSourceType=%d", tt.useSeed, 2116 len(gotState.EncryptedObject.EncryptedDEKSource), gotState.EncryptedObject.EncryptedDEKSourceType) 2117 } 2118 } 2119 2120 if tt.wantEncryptCalls != tt.service.encryptCalls { 2121 t.Errorf("want %d encryptCalls, got %d", tt.wantEncryptCalls, tt.service.encryptCalls) 2122 } 2123 2124 if errString(err) != tt.wantErr { 2125 t.Errorf("rotateDEKOnKeyIDChange() error = %v, wantErr %v", err, tt.wantErr) 2126 } 2127 2128 // if the old or new state is valid, we should be able to use it 2129 if _, stateErr := h.getCurrentState(); stateErr == nil || err == nil { 2130 transformer := envelopekmsv2.NewEnvelopeTransformer( 2131 &testKMSv2EnvelopeService{err: fmt.Errorf("broken")}, // not called 2132 "panda", 2133 h.getCurrentState, 2134 "", 2135 ) 2136 2137 dataCtx := value.DefaultContext(sampleContextText) 2138 originalText := []byte(sampleText) 2139 2140 transformedData, err := transformer.TransformToStorage(ctx, originalText, dataCtx) 2141 if err != nil { 2142 t.Fatal(err) 2143 } 2144 2145 untransformedData, stale, err := transformer.TransformFromStorage(ctx, transformedData, dataCtx) 2146 if err != nil { 2147 t.Fatal(err) 2148 } 2149 if stale { 2150 t.Error("unexpected stale data") 2151 } 2152 if !bytes.Equal(untransformedData, originalText) { 2153 t.Fatalf("incorrect transformation, want: %v, got: %v", originalText, untransformedData) 2154 } 2155 } 2156 }) 2157 } 2158 } 2159 2160 func validState(t *testing.T, keyID string, exp time.Time, useSeed bool) envelopekmsv2.State { 2161 t.Helper() 2162 2163 transformer, encObject, cacheKey, err := envelopekmsv2.GenerateTransformer(testContext(t), "", &testKMSv2EnvelopeService{keyID: keyID}, useSeed) 2164 if err != nil { 2165 t.Fatal(err) 2166 } 2167 return envelopekmsv2.State{ 2168 Transformer: transformer, 2169 EncryptedObject: *encObject, 2170 ExpirationTimestamp: exp, 2171 CacheKey: cacheKey, 2172 } 2173 } 2174 2175 func logLines(logs string) []string { 2176 if len(logs) == 0 { 2177 return nil 2178 } 2179 2180 lines := strings.Split(strings.TrimSpace(logs), "\n") 2181 for i, line := range lines { 2182 lines[i] = strings.SplitN(line, "] ", 2)[1] 2183 } 2184 return lines 2185 } 2186 2187 func TestGetEncryptionConfigHash(t *testing.T) { 2188 t.Parallel() 2189 2190 tests := []struct { 2191 name string 2192 filepath string 2193 wantHash string 2194 wantErr string 2195 }{ 2196 { 2197 name: "empty config file content", 2198 filepath: "testdata/invalid-configs/kms/invalid-content.yaml", 2199 wantHash: "", 2200 wantErr: `encryption provider configuration file "testdata/invalid-configs/kms/invalid-content.yaml" is empty`, 2201 }, 2202 { 2203 name: "missing file", 2204 filepath: "testdata/invalid-configs/kms/file-that-does-not-exist.yaml", 2205 wantHash: "", 2206 wantErr: `error reading encryption provider configuration file "testdata/invalid-configs/kms/file-that-does-not-exist.yaml": open testdata/invalid-configs/kms/file-that-does-not-exist.yaml: no such file or directory`, 2207 }, 2208 { 2209 name: "valid file", 2210 filepath: "testdata/valid-configs/secret-box-first.yaml", 2211 wantHash: "k8s:enc:unstable:1:c638c0327dbc3276dd1fcf3e67895d19ebca16b91ae0d19af24ef0759b8e0f66", 2212 wantErr: ``, 2213 }, 2214 } 2215 for _, tt := range tests { 2216 tt := tt 2217 t.Run(tt.name, func(t *testing.T) { 2218 t.Parallel() 2219 2220 got, err := GetEncryptionConfigHash(tt.filepath) 2221 if errString(err) != tt.wantErr { 2222 t.Errorf("GetEncryptionConfigHash() error = %v, wantErr %v", err, tt.wantErr) 2223 return 2224 } 2225 if got != tt.wantHash { 2226 t.Errorf("GetEncryptionConfigHash() got = %v, want %v", got, tt.wantHash) 2227 } 2228 }) 2229 } 2230 }