sigs.k8s.io/cluster-api@v1.7.1/internal/test/envtest/environment.go (about) 1 /* 2 Copyright 2020 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package envtest 18 19 import ( 20 "context" 21 "fmt" 22 "net" 23 "os" 24 "path" 25 "path/filepath" 26 goruntime "runtime" 27 "strconv" 28 "strings" 29 "testing" 30 "time" 31 32 "github.com/onsi/ginkgo/v2" 33 "github.com/pkg/errors" 34 admissionv1 "k8s.io/api/admissionregistration/v1" 35 corev1 "k8s.io/api/core/v1" 36 apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" 37 apierrors "k8s.io/apimachinery/pkg/api/errors" 38 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 39 kerrors "k8s.io/apimachinery/pkg/util/errors" 40 utilruntime "k8s.io/apimachinery/pkg/util/runtime" 41 "k8s.io/apimachinery/pkg/util/wait" 42 "k8s.io/client-go/kubernetes/scheme" 43 "k8s.io/client-go/rest" 44 "k8s.io/klog/v2" 45 ctrl "sigs.k8s.io/controller-runtime" 46 "sigs.k8s.io/controller-runtime/pkg/client" 47 "sigs.k8s.io/controller-runtime/pkg/envtest" 48 "sigs.k8s.io/controller-runtime/pkg/manager" 49 metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" 50 "sigs.k8s.io/controller-runtime/pkg/webhook" 51 52 clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" 53 bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" 54 bootstrapwebhooks "sigs.k8s.io/cluster-api/bootstrap/kubeadm/webhooks" 55 "sigs.k8s.io/cluster-api/cmd/clusterctl/log" 56 controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" 57 controlplanewebhooks "sigs.k8s.io/cluster-api/controlplane/kubeadm/webhooks" 58 addonsv1 "sigs.k8s.io/cluster-api/exp/addons/api/v1beta1" 59 addonswebhooks "sigs.k8s.io/cluster-api/exp/addons/webhooks" 60 expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" 61 ipamv1 "sigs.k8s.io/cluster-api/exp/ipam/api/v1beta1" 62 expipamwebhooks "sigs.k8s.io/cluster-api/exp/ipam/webhooks" 63 runtimev1 "sigs.k8s.io/cluster-api/exp/runtime/api/v1alpha1" 64 expapiwebhooks "sigs.k8s.io/cluster-api/exp/webhooks" 65 "sigs.k8s.io/cluster-api/internal/test/builder" 66 internalwebhooks "sigs.k8s.io/cluster-api/internal/webhooks" 67 runtimewebhooks "sigs.k8s.io/cluster-api/internal/webhooks/runtime" 68 "sigs.k8s.io/cluster-api/util/kubeconfig" 69 "sigs.k8s.io/cluster-api/version" 70 "sigs.k8s.io/cluster-api/webhooks" 71 ) 72 73 func init() { 74 logger := klog.Background() 75 // Use klog as the internal logger for this envtest environment. 76 log.SetLogger(logger) 77 // Additionally force all controllers to use the Ginkgo logger. 78 ctrl.SetLogger(logger) 79 // Add logger for ginkgo. 80 klog.SetOutput(ginkgo.GinkgoWriter) 81 82 // Calculate the scheme. 83 utilruntime.Must(apiextensionsv1.AddToScheme(scheme.Scheme)) 84 utilruntime.Must(admissionv1.AddToScheme(scheme.Scheme)) 85 utilruntime.Must(clusterv1.AddToScheme(scheme.Scheme)) 86 utilruntime.Must(bootstrapv1.AddToScheme(scheme.Scheme)) 87 utilruntime.Must(expv1.AddToScheme(scheme.Scheme)) 88 utilruntime.Must(addonsv1.AddToScheme(scheme.Scheme)) 89 utilruntime.Must(controlplanev1.AddToScheme(scheme.Scheme)) 90 utilruntime.Must(admissionv1.AddToScheme(scheme.Scheme)) 91 utilruntime.Must(runtimev1.AddToScheme(scheme.Scheme)) 92 utilruntime.Must(ipamv1.AddToScheme(scheme.Scheme)) 93 } 94 95 // RunInput is the input for Run. 96 type RunInput struct { 97 M *testing.M 98 ManagerUncachedObjs []client.Object 99 SetupIndexes func(ctx context.Context, mgr ctrl.Manager) 100 SetupReconcilers func(ctx context.Context, mgr ctrl.Manager) 101 SetupEnv func(e *Environment) 102 MinK8sVersion string 103 } 104 105 // Run executes the tests of the given testing.M in a test environment. 106 // Note: The environment will be created in this func and should not be created before. This func takes a *Environment 107 // 108 // because our tests require access to the *Environment. We use this field to make the created Environment available 109 // to the consumer. 110 // 111 // Note: Test environment creation can be skipped by setting the environment variable `CAPI_DISABLE_TEST_ENV` 112 // to a non-empty value. This only makes sense when executing tests which don't require the test environment, 113 // e.g. tests using only the fake client. 114 // Note: It's possible to write a kubeconfig for the test environment to a file by setting `CAPI_TEST_ENV_KUBECONFIG`. 115 // Note: It's possible to skip stopping the test env after the tests have been run by setting `CAPI_TEST_ENV_SKIP_STOP` 116 // to a non-empty value. 117 func Run(ctx context.Context, input RunInput) int { 118 if os.Getenv("CAPI_DISABLE_TEST_ENV") != "" { 119 klog.Info("Skipping test env start as CAPI_DISABLE_TEST_ENV is set") 120 return input.M.Run() 121 } 122 123 // Bootstrapping test environment 124 env := newEnvironment(input.ManagerUncachedObjs...) 125 126 if input.SetupIndexes != nil { 127 input.SetupIndexes(ctx, env.Manager) 128 } 129 if input.SetupReconcilers != nil { 130 input.SetupReconcilers(ctx, env.Manager) 131 } 132 133 // Start the environment. 134 env.start(ctx) 135 136 if kubeconfigPath := os.Getenv("CAPI_TEST_ENV_KUBECONFIG"); kubeconfigPath != "" { 137 klog.Infof("Writing test env kubeconfig to %q", kubeconfigPath) 138 config := kubeconfig.FromEnvTestConfig(env.Config, &clusterv1.Cluster{ 139 ObjectMeta: metav1.ObjectMeta{Name: "test"}, 140 }) 141 if err := os.WriteFile(kubeconfigPath, config, 0600); err != nil { 142 panic(errors.Wrapf(err, "failed to write the test env kubeconfig")) 143 } 144 } 145 146 if input.MinK8sVersion != "" { 147 if err := version.CheckKubernetesVersion(env.Config, input.MinK8sVersion); err != nil { 148 fmt.Printf("[ERROR] Cannot run tests after failing version check: %v\n", err) 149 if err := env.stop(); err != nil { 150 fmt.Println("[WARNING] Failed to stop the test environment") 151 } 152 return 1 153 } 154 } 155 156 // Expose the environment. 157 input.SetupEnv(env) 158 159 // Run tests 160 code := input.M.Run() 161 162 if skipStop := os.Getenv("CAPI_TEST_ENV_SKIP_STOP"); skipStop != "" { 163 klog.Info("Skipping test env stop as CAPI_TEST_ENV_SKIP_STOP is set") 164 return code 165 } 166 167 // Tearing down the test environment 168 if err := env.stop(); err != nil { 169 panic(fmt.Sprintf("Failed to stop the test environment: %v", err)) 170 } 171 return code 172 } 173 174 var ( 175 cacheSyncBackoff = wait.Backoff{ 176 Duration: 100 * time.Millisecond, 177 Factor: 1.5, 178 Steps: 8, 179 Jitter: 0.4, 180 } 181 ) 182 183 // Environment encapsulates a Kubernetes local test environment. 184 type Environment struct { 185 manager.Manager 186 client.Client 187 Config *rest.Config 188 189 env *envtest.Environment 190 cancelManager context.CancelFunc 191 } 192 193 // newEnvironment creates a new environment spinning up a local api-server. 194 // 195 // This function should be called only once for each package you're running tests within, 196 // usually the environment is initialized in a suite_test.go file within a `BeforeSuite` ginkgo block. 197 func newEnvironment(uncachedObjs ...client.Object) *Environment { 198 // Get the root of the current file to use in CRD paths. 199 _, filename, _, _ := goruntime.Caller(0) //nolint:dogsled 200 root := path.Join(path.Dir(filename), "..", "..", "..") 201 202 // Create the test environment. 203 env := &envtest.Environment{ 204 ErrorIfCRDPathMissing: true, 205 CRDDirectoryPaths: []string{ 206 filepath.Join(root, "config", "crd", "bases"), 207 filepath.Join(root, "controlplane", "kubeadm", "config", "crd", "bases"), 208 filepath.Join(root, "bootstrap", "kubeadm", "config", "crd", "bases"), 209 }, 210 CRDs: []*apiextensionsv1.CustomResourceDefinition{ 211 builder.GenericBootstrapConfigCRD.DeepCopy(), 212 builder.GenericBootstrapConfigTemplateCRD.DeepCopy(), 213 builder.GenericControlPlaneCRD.DeepCopy(), 214 builder.GenericControlPlaneTemplateCRD.DeepCopy(), 215 builder.GenericInfrastructureMachineCRD.DeepCopy(), 216 builder.GenericInfrastructureMachineTemplateCRD.DeepCopy(), 217 builder.GenericInfrastructureMachinePoolCRD.DeepCopy(), 218 builder.GenericInfrastructureMachinePoolTemplateCRD.DeepCopy(), 219 builder.GenericInfrastructureClusterCRD.DeepCopy(), 220 builder.GenericInfrastructureClusterTemplateCRD.DeepCopy(), 221 builder.GenericRemediationCRD.DeepCopy(), 222 builder.GenericRemediationTemplateCRD.DeepCopy(), 223 builder.TestInfrastructureClusterTemplateCRD.DeepCopy(), 224 builder.TestInfrastructureClusterCRD.DeepCopy(), 225 builder.TestInfrastructureMachineTemplateCRD.DeepCopy(), 226 builder.TestInfrastructureMachinePoolCRD.DeepCopy(), 227 builder.TestInfrastructureMachinePoolTemplateCRD.DeepCopy(), 228 builder.TestInfrastructureMachineCRD.DeepCopy(), 229 builder.TestBootstrapConfigTemplateCRD.DeepCopy(), 230 builder.TestBootstrapConfigCRD.DeepCopy(), 231 builder.TestControlPlaneTemplateCRD.DeepCopy(), 232 builder.TestControlPlaneCRD.DeepCopy(), 233 }, 234 // initialize webhook here to be able to test the envtest install via webhookOptions 235 // This should set LocalServingCertDir and LocalServingPort that are used below. 236 WebhookInstallOptions: initWebhookInstallOptions(), 237 } 238 239 if _, err := env.Start(); err != nil { 240 err = kerrors.NewAggregate([]error{err, env.Stop()}) 241 panic(err) 242 } 243 244 // Localhost is used on MacOS to avoid Firewall warning popups. 245 host := "localhost" 246 if strings.EqualFold(os.Getenv("USE_EXISTING_CLUSTER"), "true") { 247 // 0.0.0.0 is required on Linux when using kind because otherwise the kube-apiserver running in kind 248 // is unable to reach the webhook, because the webhook would be only listening on 127.0.0.1. 249 // Somehow that's not an issue on MacOS. 250 if goruntime.GOOS == "linux" { 251 host = "0.0.0.0" 252 } 253 } 254 255 options := manager.Options{ 256 Scheme: scheme.Scheme, 257 Metrics: metricsserver.Options{ 258 BindAddress: "0", 259 }, 260 Client: client.Options{ 261 Cache: &client.CacheOptions{ 262 DisableFor: uncachedObjs, 263 }, 264 }, 265 WebhookServer: webhook.NewServer( 266 webhook.Options{ 267 Port: env.WebhookInstallOptions.LocalServingPort, 268 CertDir: env.WebhookInstallOptions.LocalServingCertDir, 269 Host: host, 270 }, 271 ), 272 } 273 274 mgr, err := ctrl.NewManager(env.Config, options) 275 if err != nil { 276 klog.Fatalf("Failed to start testenv manager: %v", err) 277 } 278 279 // Set minNodeStartupTimeout for Test, so it does not need to be at least 30s 280 internalwebhooks.SetMinNodeStartupTimeout(metav1.Duration{Duration: 1 * time.Millisecond}) 281 282 if err := (&webhooks.Cluster{Client: mgr.GetClient()}).SetupWebhookWithManager(mgr); err != nil { 283 klog.Fatalf("unable to create webhook: %+v", err) 284 } 285 if err := (&webhooks.ClusterClass{Client: mgr.GetClient()}).SetupWebhookWithManager(mgr); err != nil { 286 klog.Fatalf("unable to create webhook: %+v", err) 287 } 288 if err := (&webhooks.Machine{}).SetupWebhookWithManager(mgr); err != nil { 289 klog.Fatalf("unable to create webhook: %+v", err) 290 } 291 if err := (&webhooks.MachineHealthCheck{}).SetupWebhookWithManager(mgr); err != nil { 292 klog.Fatalf("unable to create webhook: %+v", err) 293 } 294 if err := (&webhooks.Machine{}).SetupWebhookWithManager(mgr); err != nil { 295 klog.Fatalf("unable to create webhook: %+v", err) 296 } 297 if err := (&webhooks.MachineSet{}).SetupWebhookWithManager(mgr); err != nil { 298 klog.Fatalf("unable to create webhook: %+v", err) 299 } 300 if err := (&webhooks.MachineDeployment{}).SetupWebhookWithManager(mgr); err != nil { 301 klog.Fatalf("unable to create webhook: %+v", err) 302 } 303 if err := (&bootstrapwebhooks.KubeadmConfig{}).SetupWebhookWithManager(mgr); err != nil { 304 klog.Fatalf("unable to create webhook: %+v", err) 305 } 306 if err := (&bootstrapwebhooks.KubeadmConfigTemplate{}).SetupWebhookWithManager(mgr); err != nil { 307 klog.Fatalf("unable to create webhook: %+v", err) 308 } 309 if err := (&controlplanewebhooks.KubeadmControlPlaneTemplate{}).SetupWebhookWithManager(mgr); err != nil { 310 klog.Fatalf("unable to create webhook: %+v", err) 311 } 312 if err := (&controlplanewebhooks.KubeadmControlPlane{}).SetupWebhookWithManager(mgr); err != nil { 313 klog.Fatalf("unable to create webhook: %+v", err) 314 } 315 if err := (&addonswebhooks.ClusterResourceSet{}).SetupWebhookWithManager(mgr); err != nil { 316 klog.Fatalf("unable to create webhook for crs: %+v", err) 317 } 318 if err := (&addonswebhooks.ClusterResourceSetBinding{}).SetupWebhookWithManager(mgr); err != nil { 319 klog.Fatalf("unable to create webhook for ClusterResourceSetBinding: %+v", err) 320 } 321 if err := (&expapiwebhooks.MachinePool{}).SetupWebhookWithManager(mgr); err != nil { 322 klog.Fatalf("unable to create webhook for machinepool: %+v", err) 323 } 324 if err := (&runtimewebhooks.ExtensionConfig{}).SetupWebhookWithManager(mgr); err != nil { 325 klog.Fatalf("unable to create webhook for extensionconfig: %+v", err) 326 } 327 if err := (&expipamwebhooks.IPAddress{}).SetupWebhookWithManager(mgr); err != nil { 328 klog.Fatalf("unable to create webhook for ipaddress: %v", err) 329 } 330 if err := (&expipamwebhooks.IPAddressClaim{}).SetupWebhookWithManager(mgr); err != nil { 331 klog.Fatalf("unable to create webhook for ipaddressclaim: %v", err) 332 } 333 334 return &Environment{ 335 Manager: mgr, 336 Client: mgr.GetClient(), 337 Config: mgr.GetConfig(), 338 env: env, 339 } 340 } 341 342 // start starts the manager. 343 func (e *Environment) start(ctx context.Context) { 344 ctx, cancel := context.WithCancel(ctx) 345 e.cancelManager = cancel 346 347 go func() { 348 fmt.Println("Starting the test environment manager") 349 if err := e.Manager.Start(ctx); err != nil { 350 panic(fmt.Sprintf("Failed to start the test environment manager: %v", err)) 351 } 352 }() 353 <-e.Manager.Elected() 354 e.waitForWebhooks() 355 } 356 357 // stop stops the test environment. 358 func (e *Environment) stop() error { 359 fmt.Println("Stopping the test environment") 360 e.cancelManager() 361 return e.env.Stop() 362 } 363 364 // waitForWebhooks waits for the webhook server to be available. 365 func (e *Environment) waitForWebhooks() { 366 port := e.env.WebhookInstallOptions.LocalServingPort 367 368 klog.V(2).Infof("Waiting for webhook port %d to be open prior to running tests", port) 369 timeout := 1 * time.Second 370 for { 371 time.Sleep(1 * time.Second) 372 conn, err := net.DialTimeout("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(port)), timeout) 373 if err != nil { 374 klog.V(2).Infof("Webhook port is not ready, will retry in %v: %s", timeout, err) 375 continue 376 } 377 if err := conn.Close(); err != nil { 378 klog.V(2).Infof("Closing connection when testing if webhook port is ready failed: %v", err) 379 } 380 klog.V(2).Info("Webhook port is now open. Continuing with tests...") 381 return 382 } 383 } 384 385 // CreateKubeconfigSecret generates a new Kubeconfig secret from the envtest config. 386 func (e *Environment) CreateKubeconfigSecret(ctx context.Context, cluster *clusterv1.Cluster) error { 387 return e.Create(ctx, kubeconfig.GenerateSecret(cluster, kubeconfig.FromEnvTestConfig(e.Config, cluster))) 388 } 389 390 // Cleanup deletes all the given objects. 391 func (e *Environment) Cleanup(ctx context.Context, objs ...client.Object) error { 392 errs := []error{} 393 for _, o := range objs { 394 err := e.Client.Delete(ctx, o) 395 if apierrors.IsNotFound(err) { 396 continue 397 } 398 errs = append(errs, err) 399 } 400 return kerrors.NewAggregate(errs) 401 } 402 403 // CleanupAndWait deletes all the given objects and waits for the cache to be updated accordingly. 404 // 405 // NOTE: Waiting for the cache to be updated helps in preventing test flakes due to the cache sync delays. 406 func (e *Environment) CleanupAndWait(ctx context.Context, objs ...client.Object) error { 407 if err := e.Cleanup(ctx, objs...); err != nil { 408 return err 409 } 410 411 // Makes sure the cache is updated with the deleted object 412 errs := []error{} 413 for _, o := range objs { 414 // Ignoring namespaces because in testenv the namespace cleaner is not running. 415 if o.GetObjectKind().GroupVersionKind().GroupKind() == corev1.SchemeGroupVersion.WithKind("Namespace").GroupKind() { 416 continue 417 } 418 419 oCopy := o.DeepCopyObject().(client.Object) 420 key := client.ObjectKeyFromObject(o) 421 err := wait.ExponentialBackoff( 422 cacheSyncBackoff, 423 func() (done bool, err error) { 424 if err := e.Get(ctx, key, oCopy); err != nil { 425 if apierrors.IsNotFound(err) { 426 return true, nil 427 } 428 return false, err 429 } 430 return false, nil 431 }) 432 errs = append(errs, errors.Wrapf(err, "key %s, %s is not being deleted from the testenv client cache", o.GetObjectKind().GroupVersionKind().String(), key)) 433 } 434 return kerrors.NewAggregate(errs) 435 } 436 437 // CreateAndWait creates the given object and waits for the cache to be updated accordingly. 438 // 439 // NOTE: Waiting for the cache to be updated helps in preventing test flakes due to the cache sync delays. 440 func (e *Environment) CreateAndWait(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { 441 if err := e.Client.Create(ctx, obj, opts...); err != nil { 442 return err 443 } 444 445 // Makes sure the cache is updated with the new object 446 objCopy := obj.DeepCopyObject().(client.Object) 447 key := client.ObjectKeyFromObject(obj) 448 if err := wait.ExponentialBackoff( 449 cacheSyncBackoff, 450 func() (done bool, err error) { 451 if err := e.Get(ctx, key, objCopy); err != nil { 452 if apierrors.IsNotFound(err) { 453 return false, nil 454 } 455 return false, err 456 } 457 return true, nil 458 }); err != nil { 459 return errors.Wrapf(err, "object %s, %s is not being added to the testenv client cache", obj.GetObjectKind().GroupVersionKind().String(), key) 460 } 461 return nil 462 } 463 464 // PatchAndWait creates or updates the given object using server-side apply and waits for the cache to be updated accordingly. 465 // 466 // NOTE: Waiting for the cache to be updated helps in preventing test flakes due to the cache sync delays. 467 func (e *Environment) PatchAndWait(ctx context.Context, obj client.Object, opts ...client.PatchOption) error { 468 key := client.ObjectKeyFromObject(obj) 469 objCopy := obj.DeepCopyObject().(client.Object) 470 if err := e.GetAPIReader().Get(ctx, key, objCopy); err != nil { 471 if !apierrors.IsNotFound(err) { 472 return err 473 } 474 } 475 // Store old resource version, empty string if not found. 476 oldResourceVersion := objCopy.GetResourceVersion() 477 478 if err := e.Client.Patch(ctx, obj, client.Apply, opts...); err != nil { 479 return err 480 } 481 482 // Makes sure the cache is updated with the new object 483 if err := wait.ExponentialBackoff( 484 cacheSyncBackoff, 485 func() (done bool, err error) { 486 if err := e.Get(ctx, key, objCopy); err != nil { 487 if apierrors.IsNotFound(err) { 488 return false, nil 489 } 490 return false, err 491 } 492 if objCopy.GetResourceVersion() == oldResourceVersion { 493 return false, nil 494 } 495 return true, nil 496 }); err != nil { 497 return errors.Wrapf(err, "object %s, %s is not being added to or did not get updated in the testenv client cache", obj.GetObjectKind().GroupVersionKind().String(), key) 498 } 499 return nil 500 } 501 502 // CreateNamespace creates a new namespace with a generated name. 503 func (e *Environment) CreateNamespace(ctx context.Context, generateName string) (*corev1.Namespace, error) { 504 ns := &corev1.Namespace{ 505 ObjectMeta: metav1.ObjectMeta{ 506 GenerateName: fmt.Sprintf("%s-", generateName), 507 Labels: map[string]string{ 508 "testenv/original-name": generateName, 509 }, 510 }, 511 } 512 if err := e.Client.Create(ctx, ns); err != nil { 513 return nil, err 514 } 515 516 return ns, nil 517 }