k8s.io/kubernetes@v1.29.3/test/integration/examples/apiserver_test.go (about) 1 /* 2 Copyright 2016 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package apiserver 18 19 import ( 20 "context" 21 "encoding/json" 22 "fmt" 23 "net" 24 "net/http" 25 "net/http/httptest" 26 "net/url" 27 "os" 28 "path" 29 "reflect" 30 "sort" 31 "strings" 32 "testing" 33 "time" 34 35 "github.com/stretchr/testify/assert" 36 37 corev1 "k8s.io/api/core/v1" 38 apierrors "k8s.io/apimachinery/pkg/api/errors" 39 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 40 "k8s.io/apimachinery/pkg/util/wait" 41 "k8s.io/apiserver/pkg/server/dynamiccertificates" 42 genericapiserveroptions "k8s.io/apiserver/pkg/server/options" 43 client "k8s.io/client-go/kubernetes" 44 "k8s.io/client-go/rest" 45 "k8s.io/client-go/tools/clientcmd" 46 clientcmdapi "k8s.io/client-go/tools/clientcmd/api" 47 "k8s.io/client-go/util/cert" 48 apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" 49 aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" 50 "k8s.io/kubernetes/cmd/kube-apiserver/app" 51 kastesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" 52 "k8s.io/kubernetes/test/integration" 53 "k8s.io/kubernetes/test/integration/framework" 54 wardlev1alpha1 "k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1" 55 wardlev1beta1 "k8s.io/sample-apiserver/pkg/apis/wardle/v1beta1" 56 sampleserver "k8s.io/sample-apiserver/pkg/cmd/server" 57 wardlev1alpha1client "k8s.io/sample-apiserver/pkg/generated/clientset/versioned/typed/wardle/v1alpha1" 58 netutils "k8s.io/utils/net" 59 ) 60 61 func TestAPIServiceWaitOnStart(t *testing.T) { 62 ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) 63 t.Cleanup(cancel) 64 65 stopCh := make(chan struct{}) 66 defer close(stopCh) 67 68 etcdConfig := framework.SharedEtcd() 69 70 etcd3Client, _, err := integration.GetEtcdClients(etcdConfig.Transport) 71 if err != nil { 72 t.Fatal(err) 73 } 74 t.Cleanup(func() { etcd3Client.Close() }) 75 76 t.Log("Pollute CRD path in etcd so CRD lists cannot succeed and the informer cannot sync") 77 bogusCRDEtcdPath := path.Join("/", etcdConfig.Prefix, "apiextensions.k8s.io/customresourcedefinitions/bogus") 78 if _, err := etcd3Client.KV.Put(ctx, bogusCRDEtcdPath, `bogus data`); err != nil { 79 t.Fatal(err) 80 } 81 82 t.Log("Populate a valid CRD and managed APIService in etcd") 83 if _, err := etcd3Client.KV.Put( 84 ctx, 85 path.Join("/", etcdConfig.Prefix, "apiextensions.k8s.io/customresourcedefinitions/widgets.valid.example.com"), 86 `{ 87 "apiVersion":"apiextensions.k8s.io/v1beta1", 88 "kind":"CustomResourceDefinition", 89 "metadata":{ 90 "name":"widgets.valid.example.com", 91 "uid":"mycrd", 92 "creationTimestamp": "2022-06-08T23:46:32Z" 93 }, 94 "spec":{ 95 "scope": "Namespaced", 96 "group":"valid.example.com", 97 "version":"v1", 98 "names":{ 99 "kind": "Widget", 100 "listKind": "WidgetList", 101 "plural": "widgets", 102 "singular": "widget" 103 } 104 }, 105 "status": { 106 "acceptedNames": { 107 "kind": "Widget", 108 "listKind": "WidgetList", 109 "plural": "widgets", 110 "singular": "widget" 111 }, 112 "conditions": [ 113 { 114 "lastTransitionTime": "2023-05-18T15:03:57Z", 115 "message": "no conflicts found", 116 "reason": "NoConflicts", 117 "status": "True", 118 "type": "NamesAccepted" 119 }, 120 { 121 "lastTransitionTime": "2023-05-18T15:03:57Z", 122 "message": "the initial names have been accepted", 123 "reason": "InitialNamesAccepted", 124 "status": "True", 125 "type": "Established" 126 } 127 ], 128 "storedVersions": [ 129 "v1" 130 ] 131 } 132 }`); err != nil { 133 t.Fatal(err) 134 } 135 if _, err := etcd3Client.KV.Put( 136 ctx, 137 path.Join("/", etcdConfig.Prefix, "apiregistration.k8s.io/apiservices/v1.valid.example.com"), 138 `{ 139 "apiVersion":"apiregistration.k8s.io/v1", 140 "kind":"APIService", 141 "metadata": { 142 "name": "v1.valid.example.com", 143 "uid":"foo", 144 "creationTimestamp": "2022-06-08T23:46:32Z", 145 "labels":{"kube-aggregator.kubernetes.io/automanaged":"true"} 146 }, 147 "spec": { 148 "group": "valid.example.com", 149 "version": "v1", 150 "groupPriorityMinimum":100, 151 "versionPriority":10 152 } 153 }`, 154 ); err != nil { 155 t.Fatal(err) 156 } 157 158 t.Log("Populate a stale managed APIService in etcd") 159 if _, err := etcd3Client.KV.Put( 160 ctx, 161 path.Join("/", etcdConfig.Prefix, "apiregistration.k8s.io/apiservices/v1.stale.example.com"), 162 `{ 163 "apiVersion":"apiregistration.k8s.io/v1", 164 "kind":"APIService", 165 "metadata": { 166 "name": "v1.stale.example.com", 167 "uid":"foo", 168 "creationTimestamp": "2022-06-08T23:46:32Z", 169 "labels":{"kube-aggregator.kubernetes.io/automanaged":"true"} 170 }, 171 "spec": { 172 "group": "stale.example.com", 173 "version": "v1", 174 "groupPriorityMinimum":100, 175 "versionPriority":10 176 } 177 }`, 178 ); err != nil { 179 t.Fatal(err) 180 } 181 182 t.Log("Starting server") 183 options := kastesting.NewDefaultTestServerOptions() 184 options.SkipHealthzCheck = true 185 testServer := kastesting.StartTestServerOrDie(t, options, nil, etcdConfig) 186 defer testServer.TearDownFn() 187 188 kubeClientConfig := rest.CopyConfig(testServer.ClientConfig) 189 aggregatorClient := aggregatorclient.NewForConfigOrDie(kubeClientConfig) 190 191 t.Log("Ensure both APIService objects remain") 192 for i := 0; i < 10; i++ { 193 if _, err := aggregatorClient.ApiregistrationV1().APIServices().Get(ctx, "v1.valid.example.com", metav1.GetOptions{}); err != nil { 194 t.Fatal(err) 195 } 196 if _, err := aggregatorClient.ApiregistrationV1().APIServices().Get(ctx, "v1.stale.example.com", metav1.GetOptions{}); err != nil { 197 t.Fatal(err) 198 } 199 time.Sleep(time.Second) 200 } 201 202 t.Log("Clear the bogus CRD data so the informer can sync") 203 if _, err := etcd3Client.KV.Delete(ctx, bogusCRDEtcdPath); err != nil { 204 t.Fatal(err) 205 } 206 207 t.Log("Ensure the stale APIService object is cleaned up") 208 if err := wait.Poll(time.Second, wait.ForeverTestTimeout, func() (bool, error) { 209 _, err := aggregatorClient.ApiregistrationV1().APIServices().Get(ctx, "v1.stale.example.com", metav1.GetOptions{}) 210 if err == nil { 211 t.Log("stale APIService still exists, waiting...") 212 return false, nil 213 } 214 if !apierrors.IsNotFound(err) { 215 return false, err 216 } 217 return true, nil 218 }); err != nil { 219 t.Fatal(err) 220 } 221 222 t.Log("Ensure the valid APIService object remains") 223 for i := 0; i < 5; i++ { 224 time.Sleep(time.Second) 225 if _, err := aggregatorClient.ApiregistrationV1().APIServices().Get(ctx, "v1.valid.example.com", metav1.GetOptions{}); err != nil { 226 t.Fatal(err) 227 } 228 } 229 } 230 231 func TestAggregatedAPIServer(t *testing.T) { 232 ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) 233 t.Cleanup(cancel) 234 235 // makes the kube-apiserver very responsive. it's normally a minute 236 dynamiccertificates.FileRefreshDuration = 1 * time.Second 237 238 stopCh := make(chan struct{}) 239 defer close(stopCh) 240 241 // we need the wardle port information first to set up the service resolver 242 listener, wardlePort, err := genericapiserveroptions.CreateListener("tcp", "127.0.0.1:0", net.ListenConfig{}) 243 if err != nil { 244 t.Fatal(err) 245 } 246 // endpoints cannot have loopback IPs so we need to override the resolver itself 247 t.Cleanup(app.SetServiceResolverForTests(staticURLServiceResolver(fmt.Sprintf("https://127.0.0.1:%d", wardlePort)))) 248 249 testServer := kastesting.StartTestServerOrDie(t, &kastesting.TestServerInstanceOptions{EnableCertAuth: true}, nil, framework.SharedEtcd()) 250 defer testServer.TearDownFn() 251 kubeClientConfig := rest.CopyConfig(testServer.ClientConfig) 252 // force json because everything speaks it 253 kubeClientConfig.ContentType = "" 254 kubeClientConfig.AcceptContentTypes = "" 255 kubeClient := client.NewForConfigOrDie(kubeClientConfig) 256 aggregatorClient := aggregatorclient.NewForConfigOrDie(kubeClientConfig) 257 wardleClient := wardlev1alpha1client.NewForConfigOrDie(kubeClientConfig) 258 259 // create the bare minimum resources required to be able to get the API service into an available state 260 _, err = kubeClient.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ 261 ObjectMeta: metav1.ObjectMeta{ 262 Name: "kube-wardle", 263 }, 264 }, metav1.CreateOptions{}) 265 if err != nil { 266 t.Fatal(err) 267 } 268 _, err = kubeClient.CoreV1().Services("kube-wardle").Create(ctx, &corev1.Service{ 269 ObjectMeta: metav1.ObjectMeta{ 270 Name: "api", 271 }, 272 Spec: corev1.ServiceSpec{ 273 ExternalName: "needs-to-be-non-empty", 274 Type: corev1.ServiceTypeExternalName, 275 }, 276 }, metav1.CreateOptions{}) 277 if err != nil { 278 t.Fatal(err) 279 } 280 281 // start the wardle server to prove we can aggregate it 282 wardleToKASKubeConfigFile := writeKubeConfigForWardleServerToKASConnection(t, rest.CopyConfig(kubeClientConfig)) 283 defer os.Remove(wardleToKASKubeConfigFile) 284 wardleCertDir, _ := os.MkdirTemp("", "test-integration-wardle-server") 285 defer os.RemoveAll(wardleCertDir) 286 go func() { 287 o := sampleserver.NewWardleServerOptions(os.Stdout, os.Stderr) 288 // ensure this is a SAN on the generated cert for service FQDN 289 o.AlternateDNS = []string{ 290 "api.kube-wardle.svc", 291 } 292 o.RecommendedOptions.SecureServing.Listener = listener 293 o.RecommendedOptions.SecureServing.BindAddress = netutils.ParseIPSloppy("127.0.0.1") 294 wardleCmd := sampleserver.NewCommandStartWardleServer(o, stopCh) 295 wardleCmd.SetArgs([]string{ 296 "--authentication-kubeconfig", wardleToKASKubeConfigFile, 297 "--authorization-kubeconfig", wardleToKASKubeConfigFile, 298 "--etcd-servers", framework.GetEtcdURL(), 299 "--cert-dir", wardleCertDir, 300 "--kubeconfig", wardleToKASKubeConfigFile, 301 }) 302 if err := wardleCmd.Execute(); err != nil { 303 t.Error(err) 304 } 305 }() 306 directWardleClientConfig, err := waitForWardleRunning(ctx, t, kubeClientConfig, wardleCertDir, wardlePort) 307 if err != nil { 308 t.Fatal(err) 309 } 310 311 // now we're finally ready to test. These are what's run by default now 312 wardleDirectClient := client.NewForConfigOrDie(directWardleClientConfig) 313 testAPIGroupList(ctx, t, wardleDirectClient.Discovery().RESTClient()) 314 testAPIGroup(ctx, t, wardleDirectClient.Discovery().RESTClient()) 315 testAPIResourceList(ctx, t, wardleDirectClient.Discovery().RESTClient()) 316 317 wardleCA, err := os.ReadFile(directWardleClientConfig.CAFile) 318 if err != nil { 319 t.Fatal(err) 320 } 321 _, err = aggregatorClient.ApiregistrationV1().APIServices().Create(ctx, &apiregistrationv1.APIService{ 322 ObjectMeta: metav1.ObjectMeta{Name: "v1alpha1.wardle.example.com"}, 323 Spec: apiregistrationv1.APIServiceSpec{ 324 Service: &apiregistrationv1.ServiceReference{ 325 Namespace: "kube-wardle", 326 Name: "api", 327 }, 328 Group: "wardle.example.com", 329 Version: "v1alpha1", 330 CABundle: wardleCA, 331 GroupPriorityMinimum: 200, 332 VersionPriority: 200, 333 }, 334 }, metav1.CreateOptions{}) 335 if err != nil { 336 t.Fatal(err) 337 } 338 339 // wait for the API service to be available 340 err = wait.Poll(time.Second, wait.ForeverTestTimeout, func() (done bool, err error) { 341 apiService, err := aggregatorClient.ApiregistrationV1().APIServices().Get(ctx, "v1alpha1.wardle.example.com", metav1.GetOptions{}) 342 if err != nil { 343 return false, err 344 } 345 var available bool 346 for _, condition := range apiService.Status.Conditions { 347 if condition.Type == apiregistrationv1.Available && condition.Status == apiregistrationv1.ConditionTrue { 348 available = true 349 break 350 } 351 } 352 if !available { 353 t.Log("api service is not available", apiService.Status.Conditions) 354 return false, nil 355 } 356 357 // make sure discovery is healthy overall 358 _, _, err = kubeClient.Discovery().ServerGroupsAndResources() 359 if err != nil { 360 t.Log("discovery failed", err) 361 return false, nil 362 } 363 364 // make sure we have the wardle resources in discovery 365 apiResources, err := kubeClient.Discovery().ServerResourcesForGroupVersion("wardle.example.com/v1alpha1") 366 if err != nil { 367 t.Log("wardle discovery failed", err) 368 return false, nil 369 } 370 if len(apiResources.APIResources) != 2 { 371 t.Log("wardle discovery has wrong resources", apiResources.APIResources) 372 return false, nil 373 } 374 resources := make([]string, 0, 2) 375 for _, resource := range apiResources.APIResources { 376 resource := resource 377 resources = append(resources, resource.Name) 378 } 379 sort.Strings(resources) 380 if !reflect.DeepEqual([]string{"fischers", "flunders"}, resources) { 381 return false, fmt.Errorf("unexpected resources: %v", resources) 382 } 383 384 return true, nil 385 }) 386 if err != nil { 387 t.Fatal(err) 388 } 389 390 // perform simple CRUD operations against the wardle resources 391 _, err = wardleClient.Fischers().Create(ctx, &wardlev1alpha1.Fischer{ 392 ObjectMeta: metav1.ObjectMeta{ 393 Name: "panda", 394 }, 395 }, metav1.CreateOptions{}) 396 if err != nil { 397 t.Fatal(err) 398 } 399 fischersList, err := wardleClient.Fischers().List(ctx, metav1.ListOptions{}) 400 if err != nil { 401 t.Fatal(err) 402 } 403 if len(fischersList.Items) != 1 { 404 t.Errorf("expected one fischer: %#v", fischersList.Items) 405 } 406 if len(fischersList.ResourceVersion) == 0 { 407 t.Error("expected non-empty resource version for fischer list") 408 } 409 410 _, err = wardleClient.Flunders(metav1.NamespaceSystem).Create(ctx, &wardlev1alpha1.Flunder{ 411 ObjectMeta: metav1.ObjectMeta{ 412 Name: "panda", 413 }, 414 }, metav1.CreateOptions{}) 415 flunderList, err := wardleClient.Flunders(metav1.NamespaceSystem).List(ctx, metav1.ListOptions{}) 416 if err != nil { 417 t.Fatal(err) 418 } 419 if len(flunderList.Items) != 1 { 420 t.Errorf("expected one flunder: %#v", flunderList.Items) 421 } 422 if len(flunderList.ResourceVersion) == 0 { 423 t.Error("expected non-empty resource version for flunder list") 424 } 425 426 // Since ClientCAs are provided by "client-ca::kube-system::extension-apiserver-authentication::client-ca-file" controller 427 // we need to wait until it picks up the configmap (via a lister) otherwise the response might contain an empty result. 428 // The following code waits up to ForeverTestTimeout seconds for ClientCA to show up otherwise it fails 429 // maybe in the future this could be wired into the /readyz EP 430 431 // Now we want to verify that the client CA bundles properly reflect the values for the cluster-authentication 432 var firstKubeCANames []string 433 err = wait.Poll(1*time.Second, wait.ForeverTestTimeout, func() (done bool, err error) { 434 firstKubeCANames, err = cert.GetClientCANamesForURL(kubeClientConfig.Host) 435 if err != nil { 436 return false, err 437 } 438 return len(firstKubeCANames) != 0, nil 439 }) 440 if err != nil { 441 t.Fatal(err) 442 } 443 t.Log(firstKubeCANames) 444 var firstWardleCANames []string 445 err = wait.Poll(1*time.Second, wait.ForeverTestTimeout, func() (done bool, err error) { 446 firstWardleCANames, err = cert.GetClientCANamesForURL(directWardleClientConfig.Host) 447 if err != nil { 448 return false, err 449 } 450 return len(firstWardleCANames) != 0, nil 451 }) 452 if err != nil { 453 t.Fatal(err) 454 } 455 t.Log(firstWardleCANames) 456 // Now we want to verify that the client CA bundles properly reflect the values for the cluster-authentication 457 if !reflect.DeepEqual(firstKubeCANames, firstWardleCANames) { 458 t.Fatal("names don't match") 459 } 460 461 // now we update the client-ca nd request-header-client-ca-file and the kas will consume it, update the configmap 462 // and then the wardle server will detect and update too. 463 if err := os.WriteFile(path.Join(testServer.TmpDir, "client-ca.crt"), differentClientCA, 0644); err != nil { 464 t.Fatal(err) 465 } 466 if err := os.WriteFile(path.Join(testServer.TmpDir, "proxy-ca.crt"), differentFrontProxyCA, 0644); err != nil { 467 t.Fatal(err) 468 } 469 // wait for it to be picked up. there's a test in certreload_test.go that ensure this works 470 time.Sleep(4 * time.Second) 471 472 // Now we want to verify that the client CA bundles properly updated to reflect the new values written for the kube-apiserver 473 secondKubeCANames, err := cert.GetClientCANamesForURL(kubeClientConfig.Host) 474 if err != nil { 475 t.Fatal(err) 476 } 477 t.Log(secondKubeCANames) 478 for i := range firstKubeCANames { 479 if firstKubeCANames[i] == secondKubeCANames[i] { 480 t.Errorf("ca bundles should change") 481 } 482 } 483 secondWardleCANames, err := cert.GetClientCANamesForURL(directWardleClientConfig.Host) 484 if err != nil { 485 t.Fatal(err) 486 } 487 t.Log(secondWardleCANames) 488 489 // second wardle should contain all the certs, first and last 490 numMatches := 0 491 for _, needle := range firstKubeCANames { 492 for _, haystack := range secondWardleCANames { 493 if needle == haystack { 494 numMatches++ 495 break 496 } 497 } 498 } 499 for _, needle := range secondKubeCANames { 500 for _, haystack := range secondWardleCANames { 501 if needle == haystack { 502 numMatches++ 503 break 504 } 505 } 506 } 507 if numMatches != 4 { 508 t.Fatal("names don't match") 509 } 510 } 511 512 func waitForWardleRunning(ctx context.Context, t *testing.T, wardleToKASKubeConfig *rest.Config, wardleCertDir string, wardlePort int) (*rest.Config, error) { 513 directWardleClientConfig := rest.AnonymousClientConfig(rest.CopyConfig(wardleToKASKubeConfig)) 514 directWardleClientConfig.CAFile = path.Join(wardleCertDir, "apiserver.crt") 515 directWardleClientConfig.CAData = nil 516 directWardleClientConfig.ServerName = "" 517 directWardleClientConfig.BearerToken = wardleToKASKubeConfig.BearerToken 518 var wardleClient client.Interface 519 lastHealthContent := []byte{} 520 var lastHealthErr error 521 err := wait.PollImmediate(100*time.Millisecond, 10*time.Second, func() (done bool, err error) { 522 if _, err := os.Stat(directWardleClientConfig.CAFile); os.IsNotExist(err) { // wait until the file trust is created 523 lastHealthErr = err 524 return false, nil 525 } 526 directWardleClientConfig.Host = fmt.Sprintf("https://127.0.0.1:%d", wardlePort) 527 wardleClient, err = client.NewForConfig(directWardleClientConfig) 528 if err != nil { 529 // this happens because we race the API server start 530 t.Log(err) 531 return false, nil 532 } 533 healthStatus := 0 534 result := wardleClient.Discovery().RESTClient().Get().AbsPath("/healthz").Do(ctx).StatusCode(&healthStatus) 535 lastHealthContent, lastHealthErr = result.Raw() 536 if healthStatus != http.StatusOK { 537 return false, nil 538 } 539 return true, nil 540 }) 541 if err != nil { 542 t.Log(string(lastHealthContent)) 543 t.Log(lastHealthErr) 544 return nil, err 545 } 546 547 return directWardleClientConfig, nil 548 } 549 550 func TestAggregatedAPIServerRejectRedirectResponse(t *testing.T) { 551 ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) 552 t.Cleanup(cancel) 553 554 backendServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 555 w.WriteHeader(http.StatusOK) 556 if strings.HasSuffix(r.URL.Path, "redirectTarget") { 557 t.Errorf("backend called unexpectedly") 558 } 559 })) 560 defer backendServer.Close() 561 562 redirectedURL := backendServer.URL 563 redirectServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 564 if strings.HasSuffix(r.URL.Path, "tryRedirect") { 565 http.Redirect(w, r, redirectedURL+"/redirectTarget", http.StatusMovedPermanently) 566 } else { 567 w.WriteHeader(http.StatusOK) 568 } 569 })) 570 defer redirectServer.Close() 571 572 // endpoints cannot have loopback IPs so we need to override the resolver itself 573 t.Cleanup(app.SetServiceResolverForTests(staticURLServiceResolver(fmt.Sprintf("https://%s", redirectServer.Listener.Addr().String())))) 574 575 // start the server after resolver is overwritten 576 redirectServer.StartTLS() 577 578 testServer := kastesting.StartTestServerOrDie(t, &kastesting.TestServerInstanceOptions{EnableCertAuth: false}, nil, framework.SharedEtcd()) 579 defer testServer.TearDownFn() 580 kubeClientConfig := rest.CopyConfig(testServer.ClientConfig) 581 // force json because everything speaks it 582 kubeClientConfig.ContentType = "" 583 kubeClientConfig.AcceptContentTypes = "" 584 kubeClient := client.NewForConfigOrDie(kubeClientConfig) 585 aggregatorClient := aggregatorclient.NewForConfigOrDie(kubeClientConfig) 586 587 // create the bare minimum resources required to be able to get the API service into an available state 588 _, err := kubeClient.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ 589 ObjectMeta: metav1.ObjectMeta{ 590 Name: "kube-redirect", 591 }, 592 }, metav1.CreateOptions{}) 593 if err != nil { 594 t.Fatal(err) 595 } 596 _, err = kubeClient.CoreV1().Services("kube-redirect").Create(ctx, &corev1.Service{ 597 ObjectMeta: metav1.ObjectMeta{ 598 Name: "api", 599 }, 600 Spec: corev1.ServiceSpec{ 601 ExternalName: "needs-to-be-non-empty", 602 Type: corev1.ServiceTypeExternalName, 603 }, 604 }, metav1.CreateOptions{}) 605 if err != nil { 606 t.Fatal(err) 607 } 608 609 _, err = aggregatorClient.ApiregistrationV1().APIServices().Create(ctx, &apiregistrationv1.APIService{ 610 ObjectMeta: metav1.ObjectMeta{Name: "v1alpha1.reject.redirect.example.com"}, 611 Spec: apiregistrationv1.APIServiceSpec{ 612 Service: &apiregistrationv1.ServiceReference{ 613 Namespace: "kube-redirect", 614 Name: "api", 615 }, 616 Group: "reject.redirect.example.com", 617 Version: "v1alpha1", 618 GroupPriorityMinimum: 200, 619 VersionPriority: 200, 620 InsecureSkipTLSVerify: true, 621 }, 622 }, metav1.CreateOptions{}) 623 if err != nil { 624 t.Fatal(err) 625 } 626 627 // wait for the API service to be available 628 err = wait.Poll(time.Second, wait.ForeverTestTimeout, func() (done bool, err error) { 629 apiService, err := aggregatorClient.ApiregistrationV1().APIServices().Get(ctx, "v1alpha1.reject.redirect.example.com", metav1.GetOptions{}) 630 if err != nil { 631 return false, err 632 } 633 var available bool 634 for _, condition := range apiService.Status.Conditions { 635 if condition.Type == apiregistrationv1.Available && condition.Status == apiregistrationv1.ConditionTrue { 636 available = true 637 break 638 } 639 } 640 if !available { 641 t.Log("api service is not available", apiService.Status.Conditions) 642 return false, nil 643 } 644 return available, nil 645 }) 646 if err != nil { 647 t.Errorf("%v", err) 648 } 649 650 // get raw response to check the original error and msg 651 expectedMsg := "the backend attempted to redirect this request, which is not permitted" 652 // add specific request path suffix to discriminate between request from client and generic pings from the aggregator 653 url := url.URL{ 654 Path: "/apis/reject.redirect.example.com/v1alpha1/tryRedirect", 655 } 656 bytes, err := kubeClient.RESTClient().Get().AbsPath(url.String()).DoRaw(context.TODO()) 657 if err == nil { 658 t.Errorf("expect server to reject redirect response, but forwarded") 659 } else if !strings.Contains(string(bytes), expectedMsg) { 660 t.Errorf("expect response contains %s, got %s", expectedMsg, string(bytes)) 661 } 662 } 663 664 func writeKubeConfigForWardleServerToKASConnection(t *testing.T, kubeClientConfig *rest.Config) string { 665 // write a kubeconfig out for starting other API servers with delegated auth. remember, no in-cluster config 666 // the loopback client config uses a loopback cert with different SNI. We need to use the "real" 667 // cert, so we'll hope we aren't hacked during a unit test and instead load it from the server we started. 668 wardleToKASKubeClientConfig := rest.CopyConfig(kubeClientConfig) 669 670 servingCerts, _, err := cert.GetServingCertificatesForURL(wardleToKASKubeClientConfig.Host, "") 671 if err != nil { 672 t.Fatal(err) 673 } 674 encodedServing, err := cert.EncodeCertificates(servingCerts...) 675 if err != nil { 676 t.Fatal(err) 677 } 678 wardleToKASKubeClientConfig.CAData = encodedServing 679 680 for _, v := range servingCerts { 681 t.Logf("Client: Server public key is %v\n", dynamiccertificates.GetHumanCertDetail(v)) 682 } 683 certs, err := cert.ParseCertsPEM(wardleToKASKubeClientConfig.CAData) 684 if err != nil { 685 t.Fatal(err) 686 } 687 for _, curr := range certs { 688 t.Logf("CA bundle %v\n", dynamiccertificates.GetHumanCertDetail(curr)) 689 } 690 691 adminKubeConfig := createKubeConfig(wardleToKASKubeClientConfig) 692 wardleToKASKubeConfigFile, _ := os.CreateTemp("", "") 693 if err := clientcmd.WriteToFile(*adminKubeConfig, wardleToKASKubeConfigFile.Name()); err != nil { 694 t.Fatal(err) 695 } 696 697 defer wardleToKASKubeConfigFile.Close() 698 return wardleToKASKubeConfigFile.Name() 699 } 700 701 func createKubeConfig(clientCfg *rest.Config) *clientcmdapi.Config { 702 clusterNick := "cluster" 703 userNick := "user" 704 contextNick := "context" 705 706 config := clientcmdapi.NewConfig() 707 708 credentials := clientcmdapi.NewAuthInfo() 709 credentials.Token = clientCfg.BearerToken 710 credentials.ClientCertificate = clientCfg.TLSClientConfig.CertFile 711 if len(credentials.ClientCertificate) == 0 { 712 credentials.ClientCertificateData = clientCfg.TLSClientConfig.CertData 713 } 714 credentials.ClientKey = clientCfg.TLSClientConfig.KeyFile 715 if len(credentials.ClientKey) == 0 { 716 credentials.ClientKeyData = clientCfg.TLSClientConfig.KeyData 717 } 718 config.AuthInfos[userNick] = credentials 719 720 cluster := clientcmdapi.NewCluster() 721 cluster.Server = clientCfg.Host 722 cluster.CertificateAuthority = clientCfg.CAFile 723 if len(cluster.CertificateAuthority) == 0 { 724 cluster.CertificateAuthorityData = clientCfg.CAData 725 } 726 cluster.InsecureSkipTLSVerify = clientCfg.Insecure 727 config.Clusters[clusterNick] = cluster 728 729 context := clientcmdapi.NewContext() 730 context.Cluster = clusterNick 731 context.AuthInfo = userNick 732 config.Contexts[contextNick] = context 733 config.CurrentContext = contextNick 734 735 return config 736 } 737 738 func readResponse(ctx context.Context, client rest.Interface, location string) ([]byte, error) { 739 return client.Get().AbsPath(location).DoRaw(ctx) 740 } 741 742 func testAPIGroupList(ctx context.Context, t *testing.T, client rest.Interface) { 743 contents, err := readResponse(ctx, client, "/apis") 744 if err != nil { 745 t.Fatalf("%v", err) 746 } 747 t.Log(string(contents)) 748 var apiGroupList metav1.APIGroupList 749 err = json.Unmarshal(contents, &apiGroupList) 750 if err != nil { 751 t.Fatalf("Error in unmarshalling response from server %s: %v", "/apis", err) 752 } 753 assert.Equal(t, 1, len(apiGroupList.Groups)) 754 assert.Equal(t, wardlev1alpha1.GroupName, apiGroupList.Groups[0].Name) 755 assert.Equal(t, 2, len(apiGroupList.Groups[0].Versions)) 756 757 v1alpha1 := metav1.GroupVersionForDiscovery{ 758 GroupVersion: wardlev1alpha1.SchemeGroupVersion.String(), 759 Version: wardlev1alpha1.SchemeGroupVersion.Version, 760 } 761 v1beta1 := metav1.GroupVersionForDiscovery{ 762 GroupVersion: wardlev1beta1.SchemeGroupVersion.String(), 763 Version: wardlev1beta1.SchemeGroupVersion.Version, 764 } 765 766 assert.Equal(t, v1beta1, apiGroupList.Groups[0].Versions[0]) 767 assert.Equal(t, v1alpha1, apiGroupList.Groups[0].Versions[1]) 768 assert.Equal(t, v1beta1, apiGroupList.Groups[0].PreferredVersion) 769 } 770 771 func testAPIGroup(ctx context.Context, t *testing.T, client rest.Interface) { 772 contents, err := readResponse(ctx, client, "/apis/wardle.example.com") 773 if err != nil { 774 t.Fatalf("%v", err) 775 } 776 t.Log(string(contents)) 777 var apiGroup metav1.APIGroup 778 err = json.Unmarshal(contents, &apiGroup) 779 if err != nil { 780 t.Fatalf("Error in unmarshalling response from server %s: %v", "/apis/wardle.example.com", err) 781 } 782 assert.Equal(t, wardlev1alpha1.SchemeGroupVersion.Group, apiGroup.Name) 783 assert.Equal(t, 2, len(apiGroup.Versions)) 784 assert.Equal(t, wardlev1alpha1.SchemeGroupVersion.String(), apiGroup.Versions[1].GroupVersion) 785 assert.Equal(t, wardlev1alpha1.SchemeGroupVersion.Version, apiGroup.Versions[1].Version) 786 assert.Equal(t, apiGroup.PreferredVersion, apiGroup.Versions[0]) 787 } 788 789 func testAPIResourceList(ctx context.Context, t *testing.T, client rest.Interface) { 790 contents, err := readResponse(ctx, client, "/apis/wardle.example.com/v1alpha1") 791 if err != nil { 792 t.Fatalf("%v", err) 793 } 794 t.Log(string(contents)) 795 var apiResourceList metav1.APIResourceList 796 err = json.Unmarshal(contents, &apiResourceList) 797 if err != nil { 798 t.Fatalf("Error in unmarshalling response from server %s: %v", "/apis/wardle.example.com/v1alpha1", err) 799 } 800 assert.Equal(t, wardlev1alpha1.SchemeGroupVersion.String(), apiResourceList.GroupVersion) 801 assert.Equal(t, 2, len(apiResourceList.APIResources)) 802 assert.Equal(t, "fischers", apiResourceList.APIResources[0].Name) 803 assert.False(t, apiResourceList.APIResources[0].Namespaced) 804 assert.Equal(t, "flunders", apiResourceList.APIResources[1].Name) 805 assert.True(t, apiResourceList.APIResources[1].Namespaced) 806 } 807 808 var ( 809 // I have no idea what these certs are, they just need to be different 810 differentClientCA = []byte(`-----BEGIN CERTIFICATE----- 811 MIIDQDCCAiigAwIBAgIJANWw74P5KJk2MA0GCSqGSIb3DQEBCwUAMDQxMjAwBgNV 812 BAMMKWdlbmVyaWNfd2ViaG9va19hZG1pc3Npb25fcGx1Z2luX3Rlc3RzX2NhMCAX 813 DTE3MTExNjAwMDUzOVoYDzIyOTEwOTAxMDAwNTM5WjAjMSEwHwYDVQQDExh3ZWJo 814 b29rLXRlc3QuZGVmYXVsdC5zdmMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK 815 AoIBAQDXd/nQ89a5H8ifEsigmMd01Ib6NVR3bkJjtkvYnTbdfYEBj7UzqOQtHoLa 816 dIVmefny5uIHvj93WD8WDVPB3jX2JHrXkDTXd/6o6jIXHcsUfFTVLp6/bZ+Anqe0 817 r/7hAPkzA2A7APyTWM3ZbEeo1afXogXhOJ1u/wz0DflgcB21gNho4kKTONXO3NHD 818 XLpspFqSkxfEfKVDJaYAoMnYZJtFNsa2OvsmLnhYF8bjeT3i07lfwrhUZvP+7Gsp 819 7UgUwc06WuNHjfx1s5e6ySzH0QioMD1rjYneqOvk0pKrMIhuAEWXqq7jlXcDtx1E 820 j+wnYbVqqVYheHZ8BCJoVAAQGs9/AgMBAAGjZDBiMAkGA1UdEwQCMAAwCwYDVR0P 821 BAQDAgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATApBgNVHREEIjAg 822 hwR/AAABghh3ZWJob29rLXRlc3QuZGVmYXVsdC5zdmMwDQYJKoZIhvcNAQELBQAD 823 ggEBAD/GKSPNyQuAOw/jsYZesb+RMedbkzs18sSwlxAJQMUrrXwlVdHrA8q5WhE6 824 ABLqU1b8lQ8AWun07R8k5tqTmNvCARrAPRUqls/ryER+3Y9YEcxEaTc3jKNZFLbc 825 T6YtcnkdhxsiO136wtiuatpYL91RgCmuSpR8+7jEHhuFU01iaASu7ypFrUzrKHTF 826 bKwiLRQi1cMzVcLErq5CDEKiKhUkoDucyARFszrGt9vNIl/YCcBOkcNvM3c05Hn3 827 M++C29JwS3Hwbubg6WO3wjFjoEhpCwU6qRYUz3MRp4tHO4kxKXx+oQnUiFnR7vW0 828 YkNtGc1RUDHwecCTFpJtPb7Yu/E= 829 -----END CERTIFICATE----- 830 `) 831 differentFrontProxyCA = []byte(`-----BEGIN CERTIFICATE----- 832 MIIBqDCCAU2gAwIBAgIUfbqeieihh/oERbfvRm38XvS/xHAwCgYIKoZIzj0EAwIw 833 GjEYMBYGA1UEAxMPSW50ZXJtZWRpYXRlLUNBMCAXDTE2MTAxMTA1MDYwMFoYDzIx 834 MTYwOTE3MDUwNjAwWjAUMRIwEAYDVQQDEwlNeSBDbGllbnQwWTATBgcqhkjOPQIB 835 BggqhkjOPQMBBwNCAARv6N4R/sjMR65iMFGNLN1GC/vd7WhDW6J4X/iAjkRLLnNb 836 KbRG/AtOUZ+7upJ3BWIRKYbOabbQGQe2BbKFiap4o3UwczAOBgNVHQ8BAf8EBAMC 837 BaAwEwYDVR0lBAwwCgYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU 838 K/pZOWpNcYai6eHFpmJEeFpeQlEwHwYDVR0jBBgwFoAUX6nQlxjfWnP6aM1meO/Q 839 a6b3a9kwCgYIKoZIzj0EAwIDSQAwRgIhAIWTKw/sjJITqeuNzJDAKU4xo1zL+xJ5 840 MnVCuBwfwDXCAiEAw/1TA+CjPq9JC5ek1ifR0FybTURjeQqYkKpve1dveps= 841 -----END CERTIFICATE----- 842 843 `) 844 ) 845 846 type staticURLServiceResolver string 847 848 func (u staticURLServiceResolver) ResolveEndpoint(namespace, name string, port int32) (*url.URL, error) { 849 return url.Parse(string(u)) 850 }