k8s.io/kubernetes@v1.29.3/test/integration/serving/serving_test.go (about) 1 /* 2 Copyright 2018 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package serving 18 19 import ( 20 "context" 21 "crypto/tls" 22 "crypto/x509" 23 "fmt" 24 "io" 25 "net/http" 26 "os" 27 "path" 28 "strings" 29 "testing" 30 31 "k8s.io/apiserver/pkg/server" 32 "k8s.io/apiserver/pkg/server/options" 33 cloudprovider "k8s.io/cloud-provider" 34 cloudctrlmgrtesting "k8s.io/cloud-provider/app/testing" 35 "k8s.io/cloud-provider/fake" 36 "k8s.io/klog/v2/ktesting" 37 kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" 38 kubectrlmgrtesting "k8s.io/kubernetes/cmd/kube-controller-manager/app/testing" 39 kubeschedulertesting "k8s.io/kubernetes/cmd/kube-scheduler/app/testing" 40 "k8s.io/kubernetes/test/integration/framework" 41 ) 42 43 type componentTester interface { 44 StartTestServer(ctx context.Context, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, func(), error) 45 } 46 47 type kubeControllerManagerTester struct{} 48 49 func (kubeControllerManagerTester) StartTestServer(ctx context.Context, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, func(), error) { 50 // avoid starting any controller loops, we're just testing serving 51 customFlags = append([]string{"--controllers="}, customFlags...) 52 gotResult, err := kubectrlmgrtesting.StartTestServer(ctx, customFlags) 53 if err != nil { 54 return nil, nil, nil, err 55 } 56 return gotResult.Options.SecureServing, gotResult.Config.SecureServing, gotResult.TearDownFn, err 57 } 58 59 type cloudControllerManagerTester struct{} 60 61 func (cloudControllerManagerTester) StartTestServer(ctx context.Context, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, func(), error) { 62 gotResult, err := cloudctrlmgrtesting.StartTestServer(ctx, customFlags) 63 if err != nil { 64 return nil, nil, nil, err 65 } 66 return gotResult.Options.SecureServing, gotResult.Config.SecureServing, gotResult.TearDownFn, err 67 } 68 69 type kubeSchedulerTester struct{} 70 71 func (kubeSchedulerTester) StartTestServer(ctx context.Context, customFlags []string) (*options.SecureServingOptionsWithLoopback, *server.SecureServingInfo, func(), error) { 72 gotResult, err := kubeschedulertesting.StartTestServer(ctx, customFlags) 73 if err != nil { 74 return nil, nil, nil, err 75 } 76 return gotResult.Options.SecureServing, gotResult.Config.SecureServing, gotResult.TearDownFn, err 77 } 78 79 func TestComponentSecureServingAndAuth(t *testing.T) { 80 if !cloudprovider.IsCloudProvider("fake") { 81 cloudprovider.RegisterCloudProvider("fake", fakeCloudProviderFactory) 82 } 83 84 // Insulate this test from picking up in-cluster config when run inside a pod 85 // We can't assume we have permissions to write to /var/run/secrets/... from a unit test to mock in-cluster config for testing 86 if len(os.Getenv("KUBERNETES_SERVICE_HOST")) > 0 { 87 t.Setenv("KUBERNETES_SERVICE_HOST", "") 88 } 89 90 // authenticate to apiserver via bearer token 91 token := "flwqkenfjasasdfmwerasd" // Fake token for testing. 92 tokenFile, err := os.CreateTemp("", "kubeconfig") 93 if err != nil { 94 t.Fatal(err) 95 } 96 tokenFile.WriteString(fmt.Sprintf(` 97 %s,system:kube-controller-manager,system:kube-controller-manager,"" 98 `, token)) 99 tokenFile.Close() 100 101 // start apiserver 102 server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{ 103 "--token-auth-file", tokenFile.Name(), 104 "--authorization-mode", "RBAC", 105 }, framework.SharedEtcd()) 106 defer server.TearDownFn() 107 108 // create kubeconfig for the apiserver 109 apiserverConfig, err := os.CreateTemp("", "kubeconfig") 110 if err != nil { 111 t.Fatal(err) 112 } 113 apiserverConfig.WriteString(fmt.Sprintf(` 114 apiVersion: v1 115 kind: Config 116 clusters: 117 - cluster: 118 server: %s 119 certificate-authority: %s 120 name: integration 121 contexts: 122 - context: 123 cluster: integration 124 user: controller-manager 125 name: default-context 126 current-context: default-context 127 users: 128 - name: controller-manager 129 user: 130 token: %s 131 `, server.ClientConfig.Host, server.ServerOpts.SecureServing.ServerCert.CertKey.CertFile, token)) 132 apiserverConfig.Close() 133 134 // create BROKEN kubeconfig for the apiserver 135 brokenApiserverConfig, err := os.CreateTemp("", "kubeconfig") 136 if err != nil { 137 t.Fatal(err) 138 } 139 brokenApiserverConfig.WriteString(fmt.Sprintf(` 140 apiVersion: v1 141 kind: Config 142 clusters: 143 - cluster: 144 server: %s 145 certificate-authority: %s 146 name: integration 147 contexts: 148 - context: 149 cluster: integration 150 user: controller-manager 151 name: default-context 152 current-context: default-context 153 users: 154 - name: controller-manager 155 user: 156 token: WRONGTOKEN 157 `, server.ClientConfig.Host, server.ServerOpts.SecureServing.ServerCert.CertKey.CertFile)) 158 brokenApiserverConfig.Close() 159 160 tests := []struct { 161 name string 162 tester componentTester 163 extraFlags []string 164 }{ 165 {"kube-controller-manager", kubeControllerManagerTester{}, nil}, 166 {"cloud-controller-manager", cloudControllerManagerTester{}, []string{"--cloud-provider=fake", "--webhook-secure-port=0"}}, 167 {"kube-scheduler", kubeSchedulerTester{}, nil}, 168 } 169 170 for _, tt := range tests { 171 t.Run(tt.name, func(t *testing.T) { 172 testComponentWithSecureServing(t, tt.tester, apiserverConfig.Name(), brokenApiserverConfig.Name(), token, tt.extraFlags) 173 }) 174 } 175 } 176 177 func testComponentWithSecureServing(t *testing.T, tester componentTester, kubeconfig, brokenKubeconfig, token string, extraFlags []string) { 178 tests := []struct { 179 name string 180 flags []string 181 path string 182 anonymous bool // to use the token or not 183 wantErr bool 184 wantSecureCode *int 185 }{ 186 {"no-flags", nil, "/healthz", false, true, nil}, 187 {"/healthz without authn/authz", []string{ 188 "--kubeconfig", kubeconfig, 189 "--leader-elect=false", 190 }, "/healthz", true, false, intPtr(http.StatusOK)}, 191 {"/metrics without authn/authz", []string{ 192 "--kubeconfig", kubeconfig, 193 "--leader-elect=false", 194 }, "/metrics", true, false, intPtr(http.StatusForbidden)}, 195 {"authorization skipped for /healthz with authn/authz", []string{ 196 "--authentication-kubeconfig", kubeconfig, 197 "--authorization-kubeconfig", kubeconfig, 198 "--kubeconfig", kubeconfig, 199 "--leader-elect=false", 200 }, "/healthz", false, false, intPtr(http.StatusOK)}, 201 {"authorization skipped for /healthz with BROKEN authn/authz", []string{ 202 "--authentication-skip-lookup", // to survive inaccessible extensions-apiserver-authentication configmap 203 "--authentication-kubeconfig", brokenKubeconfig, 204 "--authorization-kubeconfig", brokenKubeconfig, 205 "--kubeconfig", kubeconfig, 206 "--leader-elect=false", 207 }, "/healthz", false, false, intPtr(http.StatusOK)}, 208 {"not authorized /metrics with BROKEN authn/authz", []string{ 209 "--authentication-kubeconfig", kubeconfig, 210 "--authorization-kubeconfig", brokenKubeconfig, 211 "--kubeconfig", kubeconfig, 212 "--leader-elect=false", 213 }, "/metrics", false, false, intPtr(http.StatusInternalServerError)}, 214 {"always-allowed /metrics with BROKEN authn/authz", []string{ 215 "--authentication-skip-lookup", // to survive inaccessible extensions-apiserver-authentication configmap 216 "--authentication-kubeconfig", brokenKubeconfig, 217 "--authorization-kubeconfig", brokenKubeconfig, 218 "--authorization-always-allow-paths", "/healthz,/metrics", 219 "--kubeconfig", kubeconfig, 220 "--leader-elect=false", 221 }, "/metrics", false, false, intPtr(http.StatusOK)}, 222 } 223 for _, tt := range tests { 224 t.Run(tt.name, func(t *testing.T) { 225 _, ctx := ktesting.NewTestContext(t) 226 secureOptions, secureInfo, tearDownFn, err := tester.StartTestServer(ctx, append(append([]string{}, tt.flags...), extraFlags...)) 227 if tearDownFn != nil { 228 defer tearDownFn() 229 } 230 if (err != nil) != tt.wantErr { 231 t.Fatalf("StartTestServer() error = %v, wantErr %v", err, tt.wantErr) 232 } 233 if err != nil { 234 return 235 } 236 237 if want, got := tt.wantSecureCode != nil, secureInfo != nil; want != got { 238 t.Errorf("SecureServing enabled: expected=%v got=%v", want, got) 239 } else if want { 240 url := fmt.Sprintf("https://%s%s", secureInfo.Listener.Addr().String(), tt.path) 241 url = strings.Replace(url, "[::]", "127.0.0.1", -1) // switch to IPv4 because the self-signed cert does not support [::] 242 243 // read self-signed server cert disk 244 pool := x509.NewCertPool() 245 serverCertPath := path.Join(secureOptions.ServerCert.CertDirectory, secureOptions.ServerCert.PairName+".crt") 246 serverCert, err := os.ReadFile(serverCertPath) 247 if err != nil { 248 t.Fatalf("Failed to read component server cert %q: %v", serverCertPath, err) 249 } 250 pool.AppendCertsFromPEM(serverCert) 251 tr := &http.Transport{ 252 TLSClientConfig: &tls.Config{ 253 RootCAs: pool, 254 }, 255 } 256 257 client := &http.Client{Transport: tr} 258 req, err := http.NewRequest("GET", url, nil) 259 if err != nil { 260 t.Fatal(err) 261 } 262 if !tt.anonymous { 263 req.Header.Add("Authorization", fmt.Sprintf("Token %s", token)) 264 } 265 r, err := client.Do(req) 266 if err != nil { 267 t.Fatalf("failed to GET %s from component: %v", tt.path, err) 268 } 269 270 body, err := io.ReadAll(r.Body) 271 if err != nil { 272 t.Fatalf("failed to read response body: %v", err) 273 } 274 defer r.Body.Close() 275 if got, expected := r.StatusCode, *tt.wantSecureCode; got != expected { 276 t.Fatalf("expected http %d at %s of component, got: %d %q", expected, tt.path, got, string(body)) 277 } 278 } 279 }) 280 } 281 } 282 283 func intPtr(x int) *int { 284 return &x 285 } 286 287 func fakeCloudProviderFactory(io.Reader) (cloudprovider.Interface, error) { 288 return &fake.Cloud{ 289 DisableRoutes: true, // disable routes for server tests, otherwise --cluster-cidr is required 290 }, nil 291 }