k8s.io/apiserver@v0.31.1/pkg/server/options/etcd_test.go (about) 1 /* 2 Copyright 2018 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package options 18 19 import ( 20 "strings" 21 "testing" 22 "time" 23 24 "k8s.io/apimachinery/pkg/runtime" 25 "k8s.io/apimachinery/pkg/runtime/schema" 26 "k8s.io/apimachinery/pkg/runtime/serializer" 27 utilerrors "k8s.io/apimachinery/pkg/util/errors" 28 "k8s.io/apimachinery/pkg/util/sets" 29 "k8s.io/apiserver/pkg/features" 30 "k8s.io/apiserver/pkg/server" 31 "k8s.io/apiserver/pkg/server/healthz" 32 "k8s.io/apiserver/pkg/storage/storagebackend" 33 utilfeature "k8s.io/apiserver/pkg/util/feature" 34 featuregatetesting "k8s.io/component-base/featuregate/testing" 35 ) 36 37 func TestEtcdOptionsValidate(t *testing.T) { 38 testCases := []struct { 39 name string 40 testOptions *EtcdOptions 41 expectErr string 42 }{ 43 { 44 name: "test when ServerList is not specified", 45 testOptions: &EtcdOptions{ 46 StorageConfig: storagebackend.Config{ 47 Type: "etcd3", 48 Prefix: "/registry", 49 Transport: storagebackend.TransportConfig{ 50 ServerList: nil, 51 KeyFile: "/var/run/kubernetes/etcd.key", 52 TrustedCAFile: "/var/run/kubernetes/etcdca.crt", 53 CertFile: "/var/run/kubernetes/etcdce.crt", 54 }, 55 CompactionInterval: storagebackend.DefaultCompactInterval, 56 CountMetricPollPeriod: time.Minute, 57 }, 58 DefaultStorageMediaType: "application/vnd.kubernetes.protobuf", 59 DeleteCollectionWorkers: 1, 60 EnableGarbageCollection: true, 61 EnableWatchCache: true, 62 DefaultWatchCacheSize: 100, 63 EtcdServersOverrides: []string{"/events#http://127.0.0.1:4002"}, 64 }, 65 expectErr: "--etcd-servers must be specified", 66 }, 67 { 68 name: "test when storage-backend is invalid", 69 testOptions: &EtcdOptions{ 70 StorageConfig: storagebackend.Config{ 71 Type: "etcd4", 72 Prefix: "/registry", 73 Transport: storagebackend.TransportConfig{ 74 ServerList: []string{"http://127.0.0.1"}, 75 KeyFile: "/var/run/kubernetes/etcd.key", 76 TrustedCAFile: "/var/run/kubernetes/etcdca.crt", 77 CertFile: "/var/run/kubernetes/etcdce.crt", 78 }, 79 CompactionInterval: storagebackend.DefaultCompactInterval, 80 CountMetricPollPeriod: time.Minute, 81 }, 82 DefaultStorageMediaType: "application/vnd.kubernetes.protobuf", 83 DeleteCollectionWorkers: 1, 84 EnableGarbageCollection: true, 85 EnableWatchCache: true, 86 DefaultWatchCacheSize: 100, 87 EtcdServersOverrides: []string{"/events#http://127.0.0.1:4002"}, 88 }, 89 expectErr: "--storage-backend invalid, allowed values: etcd3. If not specified, it will default to 'etcd3'", 90 }, 91 { 92 name: "test when etcd-servers-overrides is invalid", 93 testOptions: &EtcdOptions{ 94 StorageConfig: storagebackend.Config{ 95 Type: "etcd3", 96 Transport: storagebackend.TransportConfig{ 97 ServerList: []string{"http://127.0.0.1"}, 98 KeyFile: "/var/run/kubernetes/etcd.key", 99 TrustedCAFile: "/var/run/kubernetes/etcdca.crt", 100 CertFile: "/var/run/kubernetes/etcdce.crt", 101 }, 102 Prefix: "/registry", 103 CompactionInterval: storagebackend.DefaultCompactInterval, 104 CountMetricPollPeriod: time.Minute, 105 }, 106 DefaultStorageMediaType: "application/vnd.kubernetes.protobuf", 107 DeleteCollectionWorkers: 1, 108 EnableGarbageCollection: true, 109 EnableWatchCache: true, 110 DefaultWatchCacheSize: 100, 111 EtcdServersOverrides: []string{"/events/http://127.0.0.1:4002"}, 112 }, 113 expectErr: "--etcd-servers-overrides invalid, must be of format: group/resource#servers, where servers are URLs, semicolon separated", 114 }, 115 { 116 name: "test when encryption-provider-config-automatic-reload is invalid", 117 testOptions: &EtcdOptions{ 118 StorageConfig: storagebackend.Config{ 119 Type: "etcd3", 120 Prefix: "/registry", 121 Transport: storagebackend.TransportConfig{ 122 ServerList: []string{"http://127.0.0.1"}, 123 KeyFile: "/var/run/kubernetes/etcd.key", 124 TrustedCAFile: "/var/run/kubernetes/etcdca.crt", 125 CertFile: "/var/run/kubernetes/etcdce.crt", 126 }, 127 CompactionInterval: storagebackend.DefaultCompactInterval, 128 CountMetricPollPeriod: time.Minute, 129 }, 130 EncryptionProviderConfigAutomaticReload: true, 131 DefaultStorageMediaType: "application/vnd.kubernetes.protobuf", 132 DeleteCollectionWorkers: 1, 133 EnableGarbageCollection: true, 134 EnableWatchCache: true, 135 DefaultWatchCacheSize: 100, 136 EtcdServersOverrides: []string{"/events#http://127.0.0.1:4002"}, 137 }, 138 expectErr: "--encryption-provider-config-automatic-reload must be set with --encryption-provider-config", 139 }, 140 { 141 name: "test when EtcdOptions is valid", 142 testOptions: &EtcdOptions{ 143 StorageConfig: storagebackend.Config{ 144 Type: "etcd3", 145 Prefix: "/registry", 146 Transport: storagebackend.TransportConfig{ 147 ServerList: []string{"http://127.0.0.1"}, 148 KeyFile: "/var/run/kubernetes/etcd.key", 149 TrustedCAFile: "/var/run/kubernetes/etcdca.crt", 150 CertFile: "/var/run/kubernetes/etcdce.crt", 151 }, 152 CompactionInterval: storagebackend.DefaultCompactInterval, 153 CountMetricPollPeriod: time.Minute, 154 }, 155 DefaultStorageMediaType: "application/vnd.kubernetes.protobuf", 156 DeleteCollectionWorkers: 1, 157 EnableGarbageCollection: true, 158 EnableWatchCache: true, 159 DefaultWatchCacheSize: 100, 160 EtcdServersOverrides: []string{"/events#http://127.0.0.1:4002"}, 161 }, 162 }, 163 { 164 name: "empty storage-media-type", 165 testOptions: &EtcdOptions{ 166 StorageConfig: storagebackend.Config{ 167 Transport: storagebackend.TransportConfig{ 168 ServerList: []string{"http://127.0.0.1"}, 169 }, 170 }, 171 DefaultStorageMediaType: "", 172 }, 173 }, 174 { 175 name: "recognized storage-media-type", 176 testOptions: &EtcdOptions{ 177 StorageConfig: storagebackend.Config{ 178 Transport: storagebackend.TransportConfig{ 179 ServerList: []string{"http://127.0.0.1"}, 180 }, 181 }, 182 DefaultStorageMediaType: "application/json", 183 }, 184 }, 185 { 186 name: "unrecognized storage-media-type", 187 testOptions: &EtcdOptions{ 188 StorageConfig: storagebackend.Config{ 189 Transport: storagebackend.TransportConfig{ 190 ServerList: []string{"http://127.0.0.1"}, 191 }, 192 }, 193 DefaultStorageMediaType: "foo/bar", 194 }, 195 expectErr: `--storage-media-type "foo/bar" invalid, allowed values: application/json, application/vnd.kubernetes.protobuf, application/yaml`, 196 }, 197 } 198 199 for _, testcase := range testCases { 200 t.Run(testcase.name, func(t *testing.T) { 201 errs := testcase.testOptions.Validate() 202 if len(testcase.expectErr) != 0 && !strings.Contains(utilerrors.NewAggregate(errs).Error(), testcase.expectErr) { 203 t.Errorf("got err: %v, expected err: %s", errs, testcase.expectErr) 204 } 205 if len(testcase.expectErr) == 0 && len(errs) != 0 { 206 t.Errorf("got err: %s, expected err nil", errs) 207 } 208 }) 209 } 210 } 211 212 func TestParseWatchCacheSizes(t *testing.T) { 213 testCases := []struct { 214 name string 215 cacheSizes []string 216 expectWatchCacheSizes map[schema.GroupResource]int 217 expectErr string 218 }{ 219 { 220 name: "test when invalid value of watch cache size", 221 cacheSizes: []string{"deployments.apps#65536", "replicasets.extensions"}, 222 expectErr: "invalid value of watch cache size", 223 }, 224 { 225 name: "test when invalid size of watch cache size", 226 cacheSizes: []string{"deployments.apps#65536", "replicasets.extensions#655d1"}, 227 expectErr: "invalid size of watch cache size", 228 }, 229 { 230 name: "test when watch cache size is negative", 231 cacheSizes: []string{"deployments.apps#65536", "replicasets.extensions#-65536"}, 232 expectErr: "watch cache size cannot be negative", 233 }, 234 { 235 name: "test when parse watch cache size success", 236 cacheSizes: []string{"deployments.apps#65536", "replicasets.extensions#65536"}, 237 expectWatchCacheSizes: map[schema.GroupResource]int{ 238 {Group: "apps", Resource: "deployments"}: 65536, 239 {Group: "extensions", Resource: "replicasets"}: 65536, 240 }, 241 }, 242 } 243 244 for _, testcase := range testCases { 245 t.Run(testcase.name, func(t *testing.T) { 246 result, err := ParseWatchCacheSizes(testcase.cacheSizes) 247 if len(testcase.expectErr) != 0 && !strings.Contains(err.Error(), testcase.expectErr) { 248 t.Errorf("got err: %v, expected err: %s", err, testcase.expectErr) 249 } 250 if len(testcase.expectErr) == 0 { 251 if err != nil { 252 t.Errorf("got err: %v, expected err nil", err) 253 } else { 254 for key, expectValue := range testcase.expectWatchCacheSizes { 255 if resultValue, exist := result[key]; !exist || resultValue != expectValue { 256 t.Errorf("got watch cache size: %v, expected watch cache size %v", result, testcase.expectWatchCacheSizes) 257 } 258 } 259 } 260 } 261 }) 262 } 263 } 264 265 func TestKMSHealthzEndpoint(t *testing.T) { 266 featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true) 267 268 testCases := []struct { 269 name string 270 encryptionConfigPath string 271 wantHealthzChecks []string 272 wantReadyzChecks []string 273 wantLivezChecks []string 274 skipHealth bool 275 reload bool 276 }{ 277 { 278 name: "no kms-provider, expect no kms healthz check, no kms livez check", 279 encryptionConfigPath: "testdata/encryption-configs/no-kms-provider.yaml", 280 wantHealthzChecks: []string{"etcd"}, 281 wantReadyzChecks: []string{"etcd", "etcd-readiness"}, 282 wantLivezChecks: []string{"etcd"}, 283 }, 284 { 285 name: "no kms-provider+reload, expect single kms healthz check, no kms livez check", 286 encryptionConfigPath: "testdata/encryption-configs/no-kms-provider.yaml", 287 reload: true, 288 wantHealthzChecks: []string{"etcd", "kms-providers"}, 289 wantReadyzChecks: []string{"etcd", "kms-providers", "etcd-readiness"}, 290 wantLivezChecks: []string{"etcd"}, 291 }, 292 { 293 name: "single kms-provider, expect single kms healthz check, no kms livez check", 294 encryptionConfigPath: "testdata/encryption-configs/single-kms-provider.yaml", 295 wantHealthzChecks: []string{"etcd", "kms-provider-0"}, 296 wantReadyzChecks: []string{"etcd", "kms-provider-0", "etcd-readiness"}, 297 wantLivezChecks: []string{"etcd"}, 298 }, 299 { 300 name: "two kms-providers, expect two kms healthz checks, no kms livez check", 301 encryptionConfigPath: "testdata/encryption-configs/multiple-kms-providers.yaml", 302 wantHealthzChecks: []string{"etcd", "kms-provider-0", "kms-provider-1"}, 303 wantReadyzChecks: []string{"etcd", "kms-provider-0", "kms-provider-1", "etcd-readiness"}, 304 wantLivezChecks: []string{"etcd"}, 305 }, 306 { 307 name: "two kms-providers+reload, expect single kms healthz check, no kms livez check", 308 encryptionConfigPath: "testdata/encryption-configs/multiple-kms-providers.yaml", 309 reload: true, 310 wantHealthzChecks: []string{"etcd", "kms-providers"}, 311 wantReadyzChecks: []string{"etcd", "kms-providers", "etcd-readiness"}, 312 wantLivezChecks: []string{"etcd"}, 313 }, 314 { 315 name: "kms v1+v2, expect three kms healthz checks, no kms livez check", 316 encryptionConfigPath: "testdata/encryption-configs/multiple-kms-providers-with-v2.yaml", 317 wantHealthzChecks: []string{"etcd", "kms-provider-0", "kms-provider-1", "kms-provider-2"}, 318 wantReadyzChecks: []string{"etcd", "kms-provider-0", "kms-provider-1", "kms-provider-2", "etcd-readiness"}, 319 wantLivezChecks: []string{"etcd"}, 320 }, 321 { 322 name: "kms v1+v2+reload, expect single kms healthz check, no kms livez check", 323 encryptionConfigPath: "testdata/encryption-configs/multiple-kms-providers-with-v2.yaml", 324 reload: true, 325 wantHealthzChecks: []string{"etcd", "kms-providers"}, 326 wantReadyzChecks: []string{"etcd", "kms-providers", "etcd-readiness"}, 327 wantLivezChecks: []string{"etcd"}, 328 }, 329 { 330 name: "multiple kms v2, expect single kms healthz check, no kms livez check", 331 encryptionConfigPath: "testdata/encryption-configs/multiple-kms-v2-providers.yaml", 332 wantHealthzChecks: []string{"etcd", "kms-providers"}, 333 wantReadyzChecks: []string{"etcd", "kms-providers", "etcd-readiness"}, 334 wantLivezChecks: []string{"etcd"}, 335 }, 336 { 337 name: "multiple kms v2+reload, expect single kms healthz check, no kms livez check", 338 encryptionConfigPath: "testdata/encryption-configs/multiple-kms-v2-providers.yaml", 339 reload: true, 340 wantHealthzChecks: []string{"etcd", "kms-providers"}, 341 wantReadyzChecks: []string{"etcd", "kms-providers", "etcd-readiness"}, 342 wantLivezChecks: []string{"etcd"}, 343 }, 344 { 345 name: "two kms-providers with skip, expect zero kms healthz checks, no kms livez check", 346 encryptionConfigPath: "testdata/encryption-configs/multiple-kms-providers.yaml", 347 wantHealthzChecks: nil, 348 wantReadyzChecks: nil, 349 wantLivezChecks: nil, 350 skipHealth: true, 351 }, 352 } 353 354 scheme := runtime.NewScheme() 355 codecs := serializer.NewCodecFactory(scheme) 356 357 for _, tc := range testCases { 358 t.Run(tc.name, func(t *testing.T) { 359 serverConfig := server.NewConfig(codecs) 360 etcdOptions := &EtcdOptions{ 361 EncryptionProviderConfigFilepath: tc.encryptionConfigPath, 362 EncryptionProviderConfigAutomaticReload: tc.reload, 363 SkipHealthEndpoints: tc.skipHealth, 364 } 365 if err := etcdOptions.ApplyTo(serverConfig); err != nil { 366 t.Fatalf("Failed to add healthz error: %v", err) 367 } 368 369 healthChecksAreEqual(t, tc.wantHealthzChecks, serverConfig.HealthzChecks, "healthz") 370 healthChecksAreEqual(t, tc.wantReadyzChecks, serverConfig.ReadyzChecks, "readyz") 371 healthChecksAreEqual(t, tc.wantLivezChecks, serverConfig.LivezChecks, "livez") 372 }) 373 } 374 } 375 376 func TestReadinessCheck(t *testing.T) { 377 testCases := []struct { 378 name string 379 wantReadyzChecks []string 380 wantHealthzChecks []string 381 wantLivezChecks []string 382 skipHealth bool 383 }{ 384 { 385 name: "Readyz should have etcd-readiness check", 386 wantReadyzChecks: []string{"etcd", "etcd-readiness"}, 387 wantHealthzChecks: []string{"etcd"}, 388 wantLivezChecks: []string{"etcd"}, 389 }, 390 { 391 name: "skip health, Readyz should not have etcd-readiness check", 392 wantReadyzChecks: nil, 393 wantHealthzChecks: nil, 394 wantLivezChecks: nil, 395 skipHealth: true, 396 }, 397 } 398 399 scheme := runtime.NewScheme() 400 codecs := serializer.NewCodecFactory(scheme) 401 402 for _, tc := range testCases { 403 t.Run(tc.name, func(t *testing.T) { 404 serverConfig := server.NewConfig(codecs) 405 etcdOptions := &EtcdOptions{SkipHealthEndpoints: tc.skipHealth} 406 if err := etcdOptions.ApplyTo(serverConfig); err != nil { 407 t.Fatalf("Failed to add healthz error: %v", err) 408 } 409 410 healthChecksAreEqual(t, tc.wantReadyzChecks, serverConfig.ReadyzChecks, "readyz") 411 healthChecksAreEqual(t, tc.wantHealthzChecks, serverConfig.HealthzChecks, "healthz") 412 healthChecksAreEqual(t, tc.wantLivezChecks, serverConfig.LivezChecks, "livez") 413 }) 414 } 415 } 416 417 func healthChecksAreEqual(t *testing.T, want []string, healthChecks []healthz.HealthChecker, checkerType string) { 418 t.Helper() 419 420 wantSet := sets.NewString(want...) 421 gotSet := sets.NewString() 422 423 for _, h := range healthChecks { 424 gotSet.Insert(h.Name()) 425 } 426 427 gotSet.Delete("log", "ping") // not relevant for our tests 428 429 if !wantSet.Equal(gotSet) { 430 t.Errorf("%s checks are not equal, missing=%q, extra=%q", checkerType, wantSet.Difference(gotSet).List(), gotSet.Difference(wantSet).List()) 431 } 432 } 433 434 func TestRestOptionsStorageObjectCountTracker(t *testing.T) { 435 serverConfig := server.NewConfig(codecs) 436 etcdOptions := &EtcdOptions{} 437 if err := etcdOptions.ApplyTo(serverConfig); err != nil { 438 t.Fatalf("Failed to apply etcd options error: %v", err) 439 } 440 restOptions, err := serverConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: ""}, nil) 441 if err != nil { 442 t.Fatal(err) 443 } 444 if restOptions.StorageConfig.StorageObjectCountTracker != serverConfig.StorageObjectCountTracker { 445 t.Errorf("There are different StorageObjectCountTracker in restOptions and serverConfig") 446 } 447 }