k8s.io/kubernetes@v1.29.3/test/integration/framework/test_server.go (about) 1 /* 2 Copyright 2018 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package framework 18 19 import ( 20 "context" 21 "net" 22 "net/http" 23 "os" 24 "path" 25 "strings" 26 "testing" 27 "time" 28 29 "github.com/google/uuid" 30 31 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 32 utilerrors "k8s.io/apimachinery/pkg/util/errors" 33 "k8s.io/apimachinery/pkg/util/wait" 34 genericapiserver "k8s.io/apiserver/pkg/server" 35 genericapiserveroptions "k8s.io/apiserver/pkg/server/options" 36 client "k8s.io/client-go/kubernetes" 37 "k8s.io/client-go/rest" 38 "k8s.io/client-go/util/cert" 39 "k8s.io/kubernetes/cmd/kube-apiserver/app" 40 "k8s.io/kubernetes/cmd/kube-apiserver/app/options" 41 "k8s.io/kubernetes/pkg/controlplane" 42 "k8s.io/kubernetes/test/utils" 43 netutils "k8s.io/utils/net" 44 ) 45 46 // This key is for testing purposes only and is not considered secure. 47 const ecdsaPrivateKey = `-----BEGIN EC PRIVATE KEY----- 48 MHcCAQEEIEZmTmUhuanLjPA2CLquXivuwBDHTt5XYwgIr/kA1LtRoAoGCCqGSM49 49 AwEHoUQDQgAEH6cuzP8XuD5wal6wf9M6xDljTOPLX2i8uIp/C/ASqiIGUeeKQtX0 50 /IR3qCXyThP/dbCiHrF3v1cuhBOHY8CLVg== 51 -----END EC PRIVATE KEY-----` 52 53 // TestServerSetup holds configuration information for a kube-apiserver test server. 54 type TestServerSetup struct { 55 ModifyServerRunOptions func(*options.ServerRunOptions) 56 ModifyServerConfig func(*controlplane.Config) 57 } 58 59 type TearDownFunc func() 60 61 // StartTestServer runs a kube-apiserver, optionally calling out to the setup.ModifyServerRunOptions and setup.ModifyServerConfig functions 62 func StartTestServer(ctx context.Context, t testing.TB, setup TestServerSetup) (client.Interface, *rest.Config, TearDownFunc) { 63 ctx, cancel := context.WithCancel(ctx) 64 65 certDir, err := os.MkdirTemp("", "test-integration-"+strings.ReplaceAll(t.Name(), "/", "_")) 66 if err != nil { 67 t.Fatalf("Couldn't create temp dir: %v", err) 68 } 69 70 var errCh chan error 71 tearDownFn := func() { 72 // Calling cancel function is stopping apiserver and cleaning up 73 // after itself, including shutting down its storage layer. 74 cancel() 75 76 // If the apiserver was started, let's wait for it to 77 // shutdown clearly. 78 if errCh != nil { 79 err, ok := <-errCh 80 if ok && err != nil { 81 t.Error(err) 82 } 83 } 84 if err := os.RemoveAll(certDir); err != nil { 85 t.Log(err) 86 } 87 } 88 89 _, defaultServiceClusterIPRange, _ := netutils.ParseCIDRSloppy("10.0.0.0/24") 90 proxySigningKey, err := utils.NewPrivateKey() 91 if err != nil { 92 t.Fatal(err) 93 } 94 proxySigningCert, err := cert.NewSelfSignedCACert(cert.Config{CommonName: "front-proxy-ca"}, proxySigningKey) 95 if err != nil { 96 t.Fatal(err) 97 } 98 proxyCACertFile, _ := os.CreateTemp(certDir, "proxy-ca.crt") 99 if err := os.WriteFile(proxyCACertFile.Name(), utils.EncodeCertPEM(proxySigningCert), 0644); err != nil { 100 t.Fatal(err) 101 } 102 defer proxyCACertFile.Close() 103 clientSigningKey, err := utils.NewPrivateKey() 104 if err != nil { 105 t.Fatal(err) 106 } 107 clientSigningCert, err := cert.NewSelfSignedCACert(cert.Config{CommonName: "client-ca"}, clientSigningKey) 108 if err != nil { 109 t.Fatal(err) 110 } 111 clientCACertFile, _ := os.CreateTemp(certDir, "client-ca.crt") 112 if err := os.WriteFile(clientCACertFile.Name(), utils.EncodeCertPEM(clientSigningCert), 0644); err != nil { 113 t.Fatal(err) 114 } 115 defer clientCACertFile.Close() 116 listener, _, err := genericapiserveroptions.CreateListener("tcp", "127.0.0.1:0", net.ListenConfig{}) 117 if err != nil { 118 t.Fatal(err) 119 } 120 121 saSigningKeyFile, err := os.CreateTemp("/tmp", "insecure_test_key") 122 if err != nil { 123 t.Fatalf("create temp file failed: %v", err) 124 } 125 defer saSigningKeyFile.Close() 126 if err = os.WriteFile(saSigningKeyFile.Name(), []byte(ecdsaPrivateKey), 0666); err != nil { 127 t.Fatalf("write file %s failed: %v", saSigningKeyFile.Name(), err) 128 } 129 130 opts := options.NewServerRunOptions() 131 opts.SecureServing.Listener = listener 132 opts.SecureServing.BindAddress = netutils.ParseIPSloppy("127.0.0.1") 133 opts.SecureServing.ServerCert.CertDirectory = certDir 134 opts.ServiceAccountSigningKeyFile = saSigningKeyFile.Name() 135 opts.Etcd.StorageConfig.Prefix = path.Join("/", uuid.New().String(), "registry") 136 opts.Etcd.StorageConfig.Transport.ServerList = []string{GetEtcdURL()} 137 opts.ServiceClusterIPRanges = defaultServiceClusterIPRange.String() 138 opts.Authentication.RequestHeader.UsernameHeaders = []string{"X-Remote-User"} 139 opts.Authentication.RequestHeader.GroupHeaders = []string{"X-Remote-Group"} 140 opts.Authentication.RequestHeader.ExtraHeaderPrefixes = []string{"X-Remote-Extra-"} 141 opts.Authentication.RequestHeader.AllowedNames = []string{"kube-aggregator"} 142 opts.Authentication.RequestHeader.ClientCAFile = proxyCACertFile.Name() 143 opts.Authentication.APIAudiences = []string{"https://foo.bar.example.com"} 144 opts.Authentication.ServiceAccounts.Issuers = []string{"https://foo.bar.example.com"} 145 opts.Authentication.ServiceAccounts.KeyFiles = []string{saSigningKeyFile.Name()} 146 opts.Authentication.ClientCert.ClientCA = clientCACertFile.Name() 147 opts.Authorization.Modes = []string{"Node", "RBAC"} 148 149 if setup.ModifyServerRunOptions != nil { 150 setup.ModifyServerRunOptions(opts) 151 } 152 153 completedOptions, err := opts.Complete() 154 if err != nil { 155 t.Fatal(err) 156 } 157 158 if errs := completedOptions.Validate(); len(errs) != 0 { 159 t.Fatalf("failed to validate ServerRunOptions: %v", utilerrors.NewAggregate(errs)) 160 } 161 162 kubeAPIServerConfig, _, _, err := app.CreateKubeAPIServerConfig(completedOptions) 163 if err != nil { 164 t.Fatal(err) 165 } 166 167 if setup.ModifyServerConfig != nil { 168 setup.ModifyServerConfig(kubeAPIServerConfig) 169 } 170 kubeAPIServer, err := kubeAPIServerConfig.Complete().New(genericapiserver.NewEmptyDelegate()) 171 if err != nil { 172 t.Fatal(err) 173 } 174 175 errCh = make(chan error) 176 go func() { 177 defer close(errCh) 178 if err := kubeAPIServer.GenericAPIServer.PrepareRun().Run(ctx.Done()); err != nil { 179 errCh <- err 180 } 181 }() 182 183 // Adjust the loopback config for external use (external server name and CA) 184 kubeAPIServerClientConfig := rest.CopyConfig(kubeAPIServerConfig.GenericConfig.LoopbackClientConfig) 185 kubeAPIServerClientConfig.CAFile = path.Join(certDir, "apiserver.crt") 186 kubeAPIServerClientConfig.CAData = nil 187 kubeAPIServerClientConfig.ServerName = "" 188 189 // wait for health 190 err = wait.PollImmediate(100*time.Millisecond, 10*time.Second, func() (done bool, err error) { 191 select { 192 case err := <-errCh: 193 return false, err 194 default: 195 } 196 197 healthzConfig := rest.CopyConfig(kubeAPIServerClientConfig) 198 healthzConfig.ContentType = "" 199 healthzConfig.AcceptContentTypes = "" 200 kubeClient, err := client.NewForConfig(healthzConfig) 201 if err != nil { 202 // this happens because we race the API server start 203 t.Log(err) 204 return false, nil 205 } 206 207 healthStatus := 0 208 kubeClient.Discovery().RESTClient().Get().AbsPath("/healthz").Do(ctx).StatusCode(&healthStatus) 209 if healthStatus != http.StatusOK { 210 return false, nil 211 } 212 213 if _, err := kubeClient.CoreV1().Namespaces().Get(ctx, "default", metav1.GetOptions{}); err != nil { 214 return false, nil 215 } 216 if _, err := kubeClient.CoreV1().Namespaces().Get(ctx, "kube-system", metav1.GetOptions{}); err != nil { 217 return false, nil 218 } 219 220 return true, nil 221 }) 222 if err != nil { 223 t.Fatal(err) 224 } 225 226 kubeAPIServerClient, err := client.NewForConfig(kubeAPIServerClientConfig) 227 if err != nil { 228 t.Fatal(err) 229 } 230 231 return kubeAPIServerClient, kubeAPIServerClientConfig, tearDownFn 232 }