k8s.io/apiserver@v0.31.1/pkg/server/options/etcd_test.go (about)

     1  /*
     2  Copyright 2018 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package options
    18  
    19  import (
    20  	"strings"
    21  	"testing"
    22  	"time"
    23  
    24  	"k8s.io/apimachinery/pkg/runtime"
    25  	"k8s.io/apimachinery/pkg/runtime/schema"
    26  	"k8s.io/apimachinery/pkg/runtime/serializer"
    27  	utilerrors "k8s.io/apimachinery/pkg/util/errors"
    28  	"k8s.io/apimachinery/pkg/util/sets"
    29  	"k8s.io/apiserver/pkg/features"
    30  	"k8s.io/apiserver/pkg/server"
    31  	"k8s.io/apiserver/pkg/server/healthz"
    32  	"k8s.io/apiserver/pkg/storage/storagebackend"
    33  	utilfeature "k8s.io/apiserver/pkg/util/feature"
    34  	featuregatetesting "k8s.io/component-base/featuregate/testing"
    35  )
    36  
    37  func TestEtcdOptionsValidate(t *testing.T) {
    38  	testCases := []struct {
    39  		name        string
    40  		testOptions *EtcdOptions
    41  		expectErr   string
    42  	}{
    43  		{
    44  			name: "test when ServerList is not specified",
    45  			testOptions: &EtcdOptions{
    46  				StorageConfig: storagebackend.Config{
    47  					Type:   "etcd3",
    48  					Prefix: "/registry",
    49  					Transport: storagebackend.TransportConfig{
    50  						ServerList:    nil,
    51  						KeyFile:       "/var/run/kubernetes/etcd.key",
    52  						TrustedCAFile: "/var/run/kubernetes/etcdca.crt",
    53  						CertFile:      "/var/run/kubernetes/etcdce.crt",
    54  					},
    55  					CompactionInterval:    storagebackend.DefaultCompactInterval,
    56  					CountMetricPollPeriod: time.Minute,
    57  				},
    58  				DefaultStorageMediaType: "application/vnd.kubernetes.protobuf",
    59  				DeleteCollectionWorkers: 1,
    60  				EnableGarbageCollection: true,
    61  				EnableWatchCache:        true,
    62  				DefaultWatchCacheSize:   100,
    63  				EtcdServersOverrides:    []string{"/events#http://127.0.0.1:4002"},
    64  			},
    65  			expectErr: "--etcd-servers must be specified",
    66  		},
    67  		{
    68  			name: "test when storage-backend is invalid",
    69  			testOptions: &EtcdOptions{
    70  				StorageConfig: storagebackend.Config{
    71  					Type:   "etcd4",
    72  					Prefix: "/registry",
    73  					Transport: storagebackend.TransportConfig{
    74  						ServerList:    []string{"http://127.0.0.1"},
    75  						KeyFile:       "/var/run/kubernetes/etcd.key",
    76  						TrustedCAFile: "/var/run/kubernetes/etcdca.crt",
    77  						CertFile:      "/var/run/kubernetes/etcdce.crt",
    78  					},
    79  					CompactionInterval:    storagebackend.DefaultCompactInterval,
    80  					CountMetricPollPeriod: time.Minute,
    81  				},
    82  				DefaultStorageMediaType: "application/vnd.kubernetes.protobuf",
    83  				DeleteCollectionWorkers: 1,
    84  				EnableGarbageCollection: true,
    85  				EnableWatchCache:        true,
    86  				DefaultWatchCacheSize:   100,
    87  				EtcdServersOverrides:    []string{"/events#http://127.0.0.1:4002"},
    88  			},
    89  			expectErr: "--storage-backend invalid, allowed values: etcd3. If not specified, it will default to 'etcd3'",
    90  		},
    91  		{
    92  			name: "test when etcd-servers-overrides is invalid",
    93  			testOptions: &EtcdOptions{
    94  				StorageConfig: storagebackend.Config{
    95  					Type: "etcd3",
    96  					Transport: storagebackend.TransportConfig{
    97  						ServerList:    []string{"http://127.0.0.1"},
    98  						KeyFile:       "/var/run/kubernetes/etcd.key",
    99  						TrustedCAFile: "/var/run/kubernetes/etcdca.crt",
   100  						CertFile:      "/var/run/kubernetes/etcdce.crt",
   101  					},
   102  					Prefix:                "/registry",
   103  					CompactionInterval:    storagebackend.DefaultCompactInterval,
   104  					CountMetricPollPeriod: time.Minute,
   105  				},
   106  				DefaultStorageMediaType: "application/vnd.kubernetes.protobuf",
   107  				DeleteCollectionWorkers: 1,
   108  				EnableGarbageCollection: true,
   109  				EnableWatchCache:        true,
   110  				DefaultWatchCacheSize:   100,
   111  				EtcdServersOverrides:    []string{"/events/http://127.0.0.1:4002"},
   112  			},
   113  			expectErr: "--etcd-servers-overrides invalid, must be of format: group/resource#servers, where servers are URLs, semicolon separated",
   114  		},
   115  		{
   116  			name: "test when encryption-provider-config-automatic-reload is invalid",
   117  			testOptions: &EtcdOptions{
   118  				StorageConfig: storagebackend.Config{
   119  					Type:   "etcd3",
   120  					Prefix: "/registry",
   121  					Transport: storagebackend.TransportConfig{
   122  						ServerList:    []string{"http://127.0.0.1"},
   123  						KeyFile:       "/var/run/kubernetes/etcd.key",
   124  						TrustedCAFile: "/var/run/kubernetes/etcdca.crt",
   125  						CertFile:      "/var/run/kubernetes/etcdce.crt",
   126  					},
   127  					CompactionInterval:    storagebackend.DefaultCompactInterval,
   128  					CountMetricPollPeriod: time.Minute,
   129  				},
   130  				EncryptionProviderConfigAutomaticReload: true,
   131  				DefaultStorageMediaType:                 "application/vnd.kubernetes.protobuf",
   132  				DeleteCollectionWorkers:                 1,
   133  				EnableGarbageCollection:                 true,
   134  				EnableWatchCache:                        true,
   135  				DefaultWatchCacheSize:                   100,
   136  				EtcdServersOverrides:                    []string{"/events#http://127.0.0.1:4002"},
   137  			},
   138  			expectErr: "--encryption-provider-config-automatic-reload must be set with --encryption-provider-config",
   139  		},
   140  		{
   141  			name: "test when EtcdOptions is valid",
   142  			testOptions: &EtcdOptions{
   143  				StorageConfig: storagebackend.Config{
   144  					Type:   "etcd3",
   145  					Prefix: "/registry",
   146  					Transport: storagebackend.TransportConfig{
   147  						ServerList:    []string{"http://127.0.0.1"},
   148  						KeyFile:       "/var/run/kubernetes/etcd.key",
   149  						TrustedCAFile: "/var/run/kubernetes/etcdca.crt",
   150  						CertFile:      "/var/run/kubernetes/etcdce.crt",
   151  					},
   152  					CompactionInterval:    storagebackend.DefaultCompactInterval,
   153  					CountMetricPollPeriod: time.Minute,
   154  				},
   155  				DefaultStorageMediaType: "application/vnd.kubernetes.protobuf",
   156  				DeleteCollectionWorkers: 1,
   157  				EnableGarbageCollection: true,
   158  				EnableWatchCache:        true,
   159  				DefaultWatchCacheSize:   100,
   160  				EtcdServersOverrides:    []string{"/events#http://127.0.0.1:4002"},
   161  			},
   162  		},
   163  		{
   164  			name: "empty storage-media-type",
   165  			testOptions: &EtcdOptions{
   166  				StorageConfig: storagebackend.Config{
   167  					Transport: storagebackend.TransportConfig{
   168  						ServerList: []string{"http://127.0.0.1"},
   169  					},
   170  				},
   171  				DefaultStorageMediaType: "",
   172  			},
   173  		},
   174  		{
   175  			name: "recognized storage-media-type",
   176  			testOptions: &EtcdOptions{
   177  				StorageConfig: storagebackend.Config{
   178  					Transport: storagebackend.TransportConfig{
   179  						ServerList: []string{"http://127.0.0.1"},
   180  					},
   181  				},
   182  				DefaultStorageMediaType: "application/json",
   183  			},
   184  		},
   185  		{
   186  			name: "unrecognized storage-media-type",
   187  			testOptions: &EtcdOptions{
   188  				StorageConfig: storagebackend.Config{
   189  					Transport: storagebackend.TransportConfig{
   190  						ServerList: []string{"http://127.0.0.1"},
   191  					},
   192  				},
   193  				DefaultStorageMediaType: "foo/bar",
   194  			},
   195  			expectErr: `--storage-media-type "foo/bar" invalid, allowed values: application/json, application/vnd.kubernetes.protobuf, application/yaml`,
   196  		},
   197  	}
   198  
   199  	for _, testcase := range testCases {
   200  		t.Run(testcase.name, func(t *testing.T) {
   201  			errs := testcase.testOptions.Validate()
   202  			if len(testcase.expectErr) != 0 && !strings.Contains(utilerrors.NewAggregate(errs).Error(), testcase.expectErr) {
   203  				t.Errorf("got err: %v, expected err: %s", errs, testcase.expectErr)
   204  			}
   205  			if len(testcase.expectErr) == 0 && len(errs) != 0 {
   206  				t.Errorf("got err: %s, expected err nil", errs)
   207  			}
   208  		})
   209  	}
   210  }
   211  
   212  func TestParseWatchCacheSizes(t *testing.T) {
   213  	testCases := []struct {
   214  		name                  string
   215  		cacheSizes            []string
   216  		expectWatchCacheSizes map[schema.GroupResource]int
   217  		expectErr             string
   218  	}{
   219  		{
   220  			name:       "test when invalid value of watch cache size",
   221  			cacheSizes: []string{"deployments.apps#65536", "replicasets.extensions"},
   222  			expectErr:  "invalid value of watch cache size",
   223  		},
   224  		{
   225  			name:       "test when invalid size of watch cache size",
   226  			cacheSizes: []string{"deployments.apps#65536", "replicasets.extensions#655d1"},
   227  			expectErr:  "invalid size of watch cache size",
   228  		},
   229  		{
   230  			name:       "test when watch cache size is negative",
   231  			cacheSizes: []string{"deployments.apps#65536", "replicasets.extensions#-65536"},
   232  			expectErr:  "watch cache size cannot be negative",
   233  		},
   234  		{
   235  			name:       "test when parse watch cache size success",
   236  			cacheSizes: []string{"deployments.apps#65536", "replicasets.extensions#65536"},
   237  			expectWatchCacheSizes: map[schema.GroupResource]int{
   238  				{Group: "apps", Resource: "deployments"}:       65536,
   239  				{Group: "extensions", Resource: "replicasets"}: 65536,
   240  			},
   241  		},
   242  	}
   243  
   244  	for _, testcase := range testCases {
   245  		t.Run(testcase.name, func(t *testing.T) {
   246  			result, err := ParseWatchCacheSizes(testcase.cacheSizes)
   247  			if len(testcase.expectErr) != 0 && !strings.Contains(err.Error(), testcase.expectErr) {
   248  				t.Errorf("got err: %v, expected err: %s", err, testcase.expectErr)
   249  			}
   250  			if len(testcase.expectErr) == 0 {
   251  				if err != nil {
   252  					t.Errorf("got err: %v, expected err nil", err)
   253  				} else {
   254  					for key, expectValue := range testcase.expectWatchCacheSizes {
   255  						if resultValue, exist := result[key]; !exist || resultValue != expectValue {
   256  							t.Errorf("got watch cache size: %v, expected watch cache size %v", result, testcase.expectWatchCacheSizes)
   257  						}
   258  					}
   259  				}
   260  			}
   261  		})
   262  	}
   263  }
   264  
   265  func TestKMSHealthzEndpoint(t *testing.T) {
   266  	featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)
   267  
   268  	testCases := []struct {
   269  		name                 string
   270  		encryptionConfigPath string
   271  		wantHealthzChecks    []string
   272  		wantReadyzChecks     []string
   273  		wantLivezChecks      []string
   274  		skipHealth           bool
   275  		reload               bool
   276  	}{
   277  		{
   278  			name:                 "no kms-provider, expect no kms healthz check, no kms livez check",
   279  			encryptionConfigPath: "testdata/encryption-configs/no-kms-provider.yaml",
   280  			wantHealthzChecks:    []string{"etcd"},
   281  			wantReadyzChecks:     []string{"etcd", "etcd-readiness"},
   282  			wantLivezChecks:      []string{"etcd"},
   283  		},
   284  		{
   285  			name:                 "no kms-provider+reload, expect single kms healthz check, no kms livez check",
   286  			encryptionConfigPath: "testdata/encryption-configs/no-kms-provider.yaml",
   287  			reload:               true,
   288  			wantHealthzChecks:    []string{"etcd", "kms-providers"},
   289  			wantReadyzChecks:     []string{"etcd", "kms-providers", "etcd-readiness"},
   290  			wantLivezChecks:      []string{"etcd"},
   291  		},
   292  		{
   293  			name:                 "single kms-provider, expect single kms healthz check, no kms livez check",
   294  			encryptionConfigPath: "testdata/encryption-configs/single-kms-provider.yaml",
   295  			wantHealthzChecks:    []string{"etcd", "kms-provider-0"},
   296  			wantReadyzChecks:     []string{"etcd", "kms-provider-0", "etcd-readiness"},
   297  			wantLivezChecks:      []string{"etcd"},
   298  		},
   299  		{
   300  			name:                 "two kms-providers, expect two kms healthz checks, no kms livez check",
   301  			encryptionConfigPath: "testdata/encryption-configs/multiple-kms-providers.yaml",
   302  			wantHealthzChecks:    []string{"etcd", "kms-provider-0", "kms-provider-1"},
   303  			wantReadyzChecks:     []string{"etcd", "kms-provider-0", "kms-provider-1", "etcd-readiness"},
   304  			wantLivezChecks:      []string{"etcd"},
   305  		},
   306  		{
   307  			name:                 "two kms-providers+reload, expect single kms healthz check, no kms livez check",
   308  			encryptionConfigPath: "testdata/encryption-configs/multiple-kms-providers.yaml",
   309  			reload:               true,
   310  			wantHealthzChecks:    []string{"etcd", "kms-providers"},
   311  			wantReadyzChecks:     []string{"etcd", "kms-providers", "etcd-readiness"},
   312  			wantLivezChecks:      []string{"etcd"},
   313  		},
   314  		{
   315  			name:                 "kms v1+v2, expect three kms healthz checks, no kms livez check",
   316  			encryptionConfigPath: "testdata/encryption-configs/multiple-kms-providers-with-v2.yaml",
   317  			wantHealthzChecks:    []string{"etcd", "kms-provider-0", "kms-provider-1", "kms-provider-2"},
   318  			wantReadyzChecks:     []string{"etcd", "kms-provider-0", "kms-provider-1", "kms-provider-2", "etcd-readiness"},
   319  			wantLivezChecks:      []string{"etcd"},
   320  		},
   321  		{
   322  			name:                 "kms v1+v2+reload, expect single kms healthz check, no kms livez check",
   323  			encryptionConfigPath: "testdata/encryption-configs/multiple-kms-providers-with-v2.yaml",
   324  			reload:               true,
   325  			wantHealthzChecks:    []string{"etcd", "kms-providers"},
   326  			wantReadyzChecks:     []string{"etcd", "kms-providers", "etcd-readiness"},
   327  			wantLivezChecks:      []string{"etcd"},
   328  		},
   329  		{
   330  			name:                 "multiple kms v2, expect single kms healthz check, no kms livez check",
   331  			encryptionConfigPath: "testdata/encryption-configs/multiple-kms-v2-providers.yaml",
   332  			wantHealthzChecks:    []string{"etcd", "kms-providers"},
   333  			wantReadyzChecks:     []string{"etcd", "kms-providers", "etcd-readiness"},
   334  			wantLivezChecks:      []string{"etcd"},
   335  		},
   336  		{
   337  			name:                 "multiple kms v2+reload, expect single kms healthz check, no kms livez check",
   338  			encryptionConfigPath: "testdata/encryption-configs/multiple-kms-v2-providers.yaml",
   339  			reload:               true,
   340  			wantHealthzChecks:    []string{"etcd", "kms-providers"},
   341  			wantReadyzChecks:     []string{"etcd", "kms-providers", "etcd-readiness"},
   342  			wantLivezChecks:      []string{"etcd"},
   343  		},
   344  		{
   345  			name:                 "two kms-providers with skip, expect zero kms healthz checks, no kms livez check",
   346  			encryptionConfigPath: "testdata/encryption-configs/multiple-kms-providers.yaml",
   347  			wantHealthzChecks:    nil,
   348  			wantReadyzChecks:     nil,
   349  			wantLivezChecks:      nil,
   350  			skipHealth:           true,
   351  		},
   352  	}
   353  
   354  	scheme := runtime.NewScheme()
   355  	codecs := serializer.NewCodecFactory(scheme)
   356  
   357  	for _, tc := range testCases {
   358  		t.Run(tc.name, func(t *testing.T) {
   359  			serverConfig := server.NewConfig(codecs)
   360  			etcdOptions := &EtcdOptions{
   361  				EncryptionProviderConfigFilepath:        tc.encryptionConfigPath,
   362  				EncryptionProviderConfigAutomaticReload: tc.reload,
   363  				SkipHealthEndpoints:                     tc.skipHealth,
   364  			}
   365  			if err := etcdOptions.ApplyTo(serverConfig); err != nil {
   366  				t.Fatalf("Failed to add healthz error: %v", err)
   367  			}
   368  
   369  			healthChecksAreEqual(t, tc.wantHealthzChecks, serverConfig.HealthzChecks, "healthz")
   370  			healthChecksAreEqual(t, tc.wantReadyzChecks, serverConfig.ReadyzChecks, "readyz")
   371  			healthChecksAreEqual(t, tc.wantLivezChecks, serverConfig.LivezChecks, "livez")
   372  		})
   373  	}
   374  }
   375  
   376  func TestReadinessCheck(t *testing.T) {
   377  	testCases := []struct {
   378  		name              string
   379  		wantReadyzChecks  []string
   380  		wantHealthzChecks []string
   381  		wantLivezChecks   []string
   382  		skipHealth        bool
   383  	}{
   384  		{
   385  			name:              "Readyz should have etcd-readiness check",
   386  			wantReadyzChecks:  []string{"etcd", "etcd-readiness"},
   387  			wantHealthzChecks: []string{"etcd"},
   388  			wantLivezChecks:   []string{"etcd"},
   389  		},
   390  		{
   391  			name:              "skip health, Readyz should not have etcd-readiness check",
   392  			wantReadyzChecks:  nil,
   393  			wantHealthzChecks: nil,
   394  			wantLivezChecks:   nil,
   395  			skipHealth:        true,
   396  		},
   397  	}
   398  
   399  	scheme := runtime.NewScheme()
   400  	codecs := serializer.NewCodecFactory(scheme)
   401  
   402  	for _, tc := range testCases {
   403  		t.Run(tc.name, func(t *testing.T) {
   404  			serverConfig := server.NewConfig(codecs)
   405  			etcdOptions := &EtcdOptions{SkipHealthEndpoints: tc.skipHealth}
   406  			if err := etcdOptions.ApplyTo(serverConfig); err != nil {
   407  				t.Fatalf("Failed to add healthz error: %v", err)
   408  			}
   409  
   410  			healthChecksAreEqual(t, tc.wantReadyzChecks, serverConfig.ReadyzChecks, "readyz")
   411  			healthChecksAreEqual(t, tc.wantHealthzChecks, serverConfig.HealthzChecks, "healthz")
   412  			healthChecksAreEqual(t, tc.wantLivezChecks, serverConfig.LivezChecks, "livez")
   413  		})
   414  	}
   415  }
   416  
   417  func healthChecksAreEqual(t *testing.T, want []string, healthChecks []healthz.HealthChecker, checkerType string) {
   418  	t.Helper()
   419  
   420  	wantSet := sets.NewString(want...)
   421  	gotSet := sets.NewString()
   422  
   423  	for _, h := range healthChecks {
   424  		gotSet.Insert(h.Name())
   425  	}
   426  
   427  	gotSet.Delete("log", "ping") // not relevant for our tests
   428  
   429  	if !wantSet.Equal(gotSet) {
   430  		t.Errorf("%s checks are not equal, missing=%q, extra=%q", checkerType, wantSet.Difference(gotSet).List(), gotSet.Difference(wantSet).List())
   431  	}
   432  }
   433  
   434  func TestRestOptionsStorageObjectCountTracker(t *testing.T) {
   435  	serverConfig := server.NewConfig(codecs)
   436  	etcdOptions := &EtcdOptions{}
   437  	if err := etcdOptions.ApplyTo(serverConfig); err != nil {
   438  		t.Fatalf("Failed to apply etcd options error: %v", err)
   439  	}
   440  	restOptions, err := serverConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: "", Resource: ""}, nil)
   441  	if err != nil {
   442  		t.Fatal(err)
   443  	}
   444  	if restOptions.StorageConfig.StorageObjectCountTracker != serverConfig.StorageObjectCountTracker {
   445  		t.Errorf("There are different StorageObjectCountTracker in restOptions and serverConfig")
   446  	}
   447  }