sigs.k8s.io/cluster-api@v1.7.1/internal/test/envtest/environment.go (about)

     1  /*
     2  Copyright 2020 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package envtest
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"net"
    23  	"os"
    24  	"path"
    25  	"path/filepath"
    26  	goruntime "runtime"
    27  	"strconv"
    28  	"strings"
    29  	"testing"
    30  	"time"
    31  
    32  	"github.com/onsi/ginkgo/v2"
    33  	"github.com/pkg/errors"
    34  	admissionv1 "k8s.io/api/admissionregistration/v1"
    35  	corev1 "k8s.io/api/core/v1"
    36  	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
    37  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    38  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    39  	kerrors "k8s.io/apimachinery/pkg/util/errors"
    40  	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
    41  	"k8s.io/apimachinery/pkg/util/wait"
    42  	"k8s.io/client-go/kubernetes/scheme"
    43  	"k8s.io/client-go/rest"
    44  	"k8s.io/klog/v2"
    45  	ctrl "sigs.k8s.io/controller-runtime"
    46  	"sigs.k8s.io/controller-runtime/pkg/client"
    47  	"sigs.k8s.io/controller-runtime/pkg/envtest"
    48  	"sigs.k8s.io/controller-runtime/pkg/manager"
    49  	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
    50  	"sigs.k8s.io/controller-runtime/pkg/webhook"
    51  
    52  	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
    53  	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
    54  	bootstrapwebhooks "sigs.k8s.io/cluster-api/bootstrap/kubeadm/webhooks"
    55  	"sigs.k8s.io/cluster-api/cmd/clusterctl/log"
    56  	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
    57  	controlplanewebhooks "sigs.k8s.io/cluster-api/controlplane/kubeadm/webhooks"
    58  	addonsv1 "sigs.k8s.io/cluster-api/exp/addons/api/v1beta1"
    59  	addonswebhooks "sigs.k8s.io/cluster-api/exp/addons/webhooks"
    60  	expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1"
    61  	ipamv1 "sigs.k8s.io/cluster-api/exp/ipam/api/v1beta1"
    62  	expipamwebhooks "sigs.k8s.io/cluster-api/exp/ipam/webhooks"
    63  	runtimev1 "sigs.k8s.io/cluster-api/exp/runtime/api/v1alpha1"
    64  	expapiwebhooks "sigs.k8s.io/cluster-api/exp/webhooks"
    65  	"sigs.k8s.io/cluster-api/internal/test/builder"
    66  	internalwebhooks "sigs.k8s.io/cluster-api/internal/webhooks"
    67  	runtimewebhooks "sigs.k8s.io/cluster-api/internal/webhooks/runtime"
    68  	"sigs.k8s.io/cluster-api/util/kubeconfig"
    69  	"sigs.k8s.io/cluster-api/version"
    70  	"sigs.k8s.io/cluster-api/webhooks"
    71  )
    72  
    73  func init() {
    74  	logger := klog.Background()
    75  	// Use klog as the internal logger for this envtest environment.
    76  	log.SetLogger(logger)
    77  	// Additionally force all controllers to use the Ginkgo logger.
    78  	ctrl.SetLogger(logger)
    79  	// Add logger for ginkgo.
    80  	klog.SetOutput(ginkgo.GinkgoWriter)
    81  
    82  	// Calculate the scheme.
    83  	utilruntime.Must(apiextensionsv1.AddToScheme(scheme.Scheme))
    84  	utilruntime.Must(admissionv1.AddToScheme(scheme.Scheme))
    85  	utilruntime.Must(clusterv1.AddToScheme(scheme.Scheme))
    86  	utilruntime.Must(bootstrapv1.AddToScheme(scheme.Scheme))
    87  	utilruntime.Must(expv1.AddToScheme(scheme.Scheme))
    88  	utilruntime.Must(addonsv1.AddToScheme(scheme.Scheme))
    89  	utilruntime.Must(controlplanev1.AddToScheme(scheme.Scheme))
    90  	utilruntime.Must(admissionv1.AddToScheme(scheme.Scheme))
    91  	utilruntime.Must(runtimev1.AddToScheme(scheme.Scheme))
    92  	utilruntime.Must(ipamv1.AddToScheme(scheme.Scheme))
    93  }
    94  
    95  // RunInput is the input for Run.
    96  type RunInput struct {
    97  	M                   *testing.M
    98  	ManagerUncachedObjs []client.Object
    99  	SetupIndexes        func(ctx context.Context, mgr ctrl.Manager)
   100  	SetupReconcilers    func(ctx context.Context, mgr ctrl.Manager)
   101  	SetupEnv            func(e *Environment)
   102  	MinK8sVersion       string
   103  }
   104  
   105  // Run executes the tests of the given testing.M in a test environment.
   106  // Note: The environment will be created in this func and should not be created before. This func takes a *Environment
   107  //
   108  //	because our tests require access to the *Environment. We use this field to make the created Environment available
   109  //	to the consumer.
   110  //
   111  // Note: Test environment creation can be skipped by setting the environment variable `CAPI_DISABLE_TEST_ENV`
   112  // to a non-empty value. This only makes sense when executing tests which don't require the test environment,
   113  // e.g. tests using only the fake client.
   114  // Note: It's possible to write a kubeconfig for the test environment to a file by setting `CAPI_TEST_ENV_KUBECONFIG`.
   115  // Note: It's possible to skip stopping the test env after the tests have been run by setting `CAPI_TEST_ENV_SKIP_STOP`
   116  // to a non-empty value.
   117  func Run(ctx context.Context, input RunInput) int {
   118  	if os.Getenv("CAPI_DISABLE_TEST_ENV") != "" {
   119  		klog.Info("Skipping test env start as CAPI_DISABLE_TEST_ENV is set")
   120  		return input.M.Run()
   121  	}
   122  
   123  	// Bootstrapping test environment
   124  	env := newEnvironment(input.ManagerUncachedObjs...)
   125  
   126  	if input.SetupIndexes != nil {
   127  		input.SetupIndexes(ctx, env.Manager)
   128  	}
   129  	if input.SetupReconcilers != nil {
   130  		input.SetupReconcilers(ctx, env.Manager)
   131  	}
   132  
   133  	// Start the environment.
   134  	env.start(ctx)
   135  
   136  	if kubeconfigPath := os.Getenv("CAPI_TEST_ENV_KUBECONFIG"); kubeconfigPath != "" {
   137  		klog.Infof("Writing test env kubeconfig to %q", kubeconfigPath)
   138  		config := kubeconfig.FromEnvTestConfig(env.Config, &clusterv1.Cluster{
   139  			ObjectMeta: metav1.ObjectMeta{Name: "test"},
   140  		})
   141  		if err := os.WriteFile(kubeconfigPath, config, 0600); err != nil {
   142  			panic(errors.Wrapf(err, "failed to write the test env kubeconfig"))
   143  		}
   144  	}
   145  
   146  	if input.MinK8sVersion != "" {
   147  		if err := version.CheckKubernetesVersion(env.Config, input.MinK8sVersion); err != nil {
   148  			fmt.Printf("[ERROR] Cannot run tests after failing version check: %v\n", err)
   149  			if err := env.stop(); err != nil {
   150  				fmt.Println("[WARNING] Failed to stop the test environment")
   151  			}
   152  			return 1
   153  		}
   154  	}
   155  
   156  	// Expose the environment.
   157  	input.SetupEnv(env)
   158  
   159  	// Run tests
   160  	code := input.M.Run()
   161  
   162  	if skipStop := os.Getenv("CAPI_TEST_ENV_SKIP_STOP"); skipStop != "" {
   163  		klog.Info("Skipping test env stop as CAPI_TEST_ENV_SKIP_STOP is set")
   164  		return code
   165  	}
   166  
   167  	// Tearing down the test environment
   168  	if err := env.stop(); err != nil {
   169  		panic(fmt.Sprintf("Failed to stop the test environment: %v", err))
   170  	}
   171  	return code
   172  }
   173  
   174  var (
   175  	cacheSyncBackoff = wait.Backoff{
   176  		Duration: 100 * time.Millisecond,
   177  		Factor:   1.5,
   178  		Steps:    8,
   179  		Jitter:   0.4,
   180  	}
   181  )
   182  
   183  // Environment encapsulates a Kubernetes local test environment.
   184  type Environment struct {
   185  	manager.Manager
   186  	client.Client
   187  	Config *rest.Config
   188  
   189  	env           *envtest.Environment
   190  	cancelManager context.CancelFunc
   191  }
   192  
   193  // newEnvironment creates a new environment spinning up a local api-server.
   194  //
   195  // This function should be called only once for each package you're running tests within,
   196  // usually the environment is initialized in a suite_test.go file within a `BeforeSuite` ginkgo block.
   197  func newEnvironment(uncachedObjs ...client.Object) *Environment {
   198  	// Get the root of the current file to use in CRD paths.
   199  	_, filename, _, _ := goruntime.Caller(0) //nolint:dogsled
   200  	root := path.Join(path.Dir(filename), "..", "..", "..")
   201  
   202  	// Create the test environment.
   203  	env := &envtest.Environment{
   204  		ErrorIfCRDPathMissing: true,
   205  		CRDDirectoryPaths: []string{
   206  			filepath.Join(root, "config", "crd", "bases"),
   207  			filepath.Join(root, "controlplane", "kubeadm", "config", "crd", "bases"),
   208  			filepath.Join(root, "bootstrap", "kubeadm", "config", "crd", "bases"),
   209  		},
   210  		CRDs: []*apiextensionsv1.CustomResourceDefinition{
   211  			builder.GenericBootstrapConfigCRD.DeepCopy(),
   212  			builder.GenericBootstrapConfigTemplateCRD.DeepCopy(),
   213  			builder.GenericControlPlaneCRD.DeepCopy(),
   214  			builder.GenericControlPlaneTemplateCRD.DeepCopy(),
   215  			builder.GenericInfrastructureMachineCRD.DeepCopy(),
   216  			builder.GenericInfrastructureMachineTemplateCRD.DeepCopy(),
   217  			builder.GenericInfrastructureMachinePoolCRD.DeepCopy(),
   218  			builder.GenericInfrastructureMachinePoolTemplateCRD.DeepCopy(),
   219  			builder.GenericInfrastructureClusterCRD.DeepCopy(),
   220  			builder.GenericInfrastructureClusterTemplateCRD.DeepCopy(),
   221  			builder.GenericRemediationCRD.DeepCopy(),
   222  			builder.GenericRemediationTemplateCRD.DeepCopy(),
   223  			builder.TestInfrastructureClusterTemplateCRD.DeepCopy(),
   224  			builder.TestInfrastructureClusterCRD.DeepCopy(),
   225  			builder.TestInfrastructureMachineTemplateCRD.DeepCopy(),
   226  			builder.TestInfrastructureMachinePoolCRD.DeepCopy(),
   227  			builder.TestInfrastructureMachinePoolTemplateCRD.DeepCopy(),
   228  			builder.TestInfrastructureMachineCRD.DeepCopy(),
   229  			builder.TestBootstrapConfigTemplateCRD.DeepCopy(),
   230  			builder.TestBootstrapConfigCRD.DeepCopy(),
   231  			builder.TestControlPlaneTemplateCRD.DeepCopy(),
   232  			builder.TestControlPlaneCRD.DeepCopy(),
   233  		},
   234  		// initialize webhook here to be able to test the envtest install via webhookOptions
   235  		// This should set LocalServingCertDir and LocalServingPort that are used below.
   236  		WebhookInstallOptions: initWebhookInstallOptions(),
   237  	}
   238  
   239  	if _, err := env.Start(); err != nil {
   240  		err = kerrors.NewAggregate([]error{err, env.Stop()})
   241  		panic(err)
   242  	}
   243  
   244  	// Localhost is used on MacOS to avoid Firewall warning popups.
   245  	host := "localhost"
   246  	if strings.EqualFold(os.Getenv("USE_EXISTING_CLUSTER"), "true") {
   247  		// 0.0.0.0 is required on Linux when using kind because otherwise the kube-apiserver running in kind
   248  		// is unable to reach the webhook, because the webhook would be only listening on 127.0.0.1.
   249  		// Somehow that's not an issue on MacOS.
   250  		if goruntime.GOOS == "linux" {
   251  			host = "0.0.0.0"
   252  		}
   253  	}
   254  
   255  	options := manager.Options{
   256  		Scheme: scheme.Scheme,
   257  		Metrics: metricsserver.Options{
   258  			BindAddress: "0",
   259  		},
   260  		Client: client.Options{
   261  			Cache: &client.CacheOptions{
   262  				DisableFor: uncachedObjs,
   263  			},
   264  		},
   265  		WebhookServer: webhook.NewServer(
   266  			webhook.Options{
   267  				Port:    env.WebhookInstallOptions.LocalServingPort,
   268  				CertDir: env.WebhookInstallOptions.LocalServingCertDir,
   269  				Host:    host,
   270  			},
   271  		),
   272  	}
   273  
   274  	mgr, err := ctrl.NewManager(env.Config, options)
   275  	if err != nil {
   276  		klog.Fatalf("Failed to start testenv manager: %v", err)
   277  	}
   278  
   279  	// Set minNodeStartupTimeout for Test, so it does not need to be at least 30s
   280  	internalwebhooks.SetMinNodeStartupTimeout(metav1.Duration{Duration: 1 * time.Millisecond})
   281  
   282  	if err := (&webhooks.Cluster{Client: mgr.GetClient()}).SetupWebhookWithManager(mgr); err != nil {
   283  		klog.Fatalf("unable to create webhook: %+v", err)
   284  	}
   285  	if err := (&webhooks.ClusterClass{Client: mgr.GetClient()}).SetupWebhookWithManager(mgr); err != nil {
   286  		klog.Fatalf("unable to create webhook: %+v", err)
   287  	}
   288  	if err := (&webhooks.Machine{}).SetupWebhookWithManager(mgr); err != nil {
   289  		klog.Fatalf("unable to create webhook: %+v", err)
   290  	}
   291  	if err := (&webhooks.MachineHealthCheck{}).SetupWebhookWithManager(mgr); err != nil {
   292  		klog.Fatalf("unable to create webhook: %+v", err)
   293  	}
   294  	if err := (&webhooks.Machine{}).SetupWebhookWithManager(mgr); err != nil {
   295  		klog.Fatalf("unable to create webhook: %+v", err)
   296  	}
   297  	if err := (&webhooks.MachineSet{}).SetupWebhookWithManager(mgr); err != nil {
   298  		klog.Fatalf("unable to create webhook: %+v", err)
   299  	}
   300  	if err := (&webhooks.MachineDeployment{}).SetupWebhookWithManager(mgr); err != nil {
   301  		klog.Fatalf("unable to create webhook: %+v", err)
   302  	}
   303  	if err := (&bootstrapwebhooks.KubeadmConfig{}).SetupWebhookWithManager(mgr); err != nil {
   304  		klog.Fatalf("unable to create webhook: %+v", err)
   305  	}
   306  	if err := (&bootstrapwebhooks.KubeadmConfigTemplate{}).SetupWebhookWithManager(mgr); err != nil {
   307  		klog.Fatalf("unable to create webhook: %+v", err)
   308  	}
   309  	if err := (&controlplanewebhooks.KubeadmControlPlaneTemplate{}).SetupWebhookWithManager(mgr); err != nil {
   310  		klog.Fatalf("unable to create webhook: %+v", err)
   311  	}
   312  	if err := (&controlplanewebhooks.KubeadmControlPlane{}).SetupWebhookWithManager(mgr); err != nil {
   313  		klog.Fatalf("unable to create webhook: %+v", err)
   314  	}
   315  	if err := (&addonswebhooks.ClusterResourceSet{}).SetupWebhookWithManager(mgr); err != nil {
   316  		klog.Fatalf("unable to create webhook for crs: %+v", err)
   317  	}
   318  	if err := (&addonswebhooks.ClusterResourceSetBinding{}).SetupWebhookWithManager(mgr); err != nil {
   319  		klog.Fatalf("unable to create webhook for ClusterResourceSetBinding: %+v", err)
   320  	}
   321  	if err := (&expapiwebhooks.MachinePool{}).SetupWebhookWithManager(mgr); err != nil {
   322  		klog.Fatalf("unable to create webhook for machinepool: %+v", err)
   323  	}
   324  	if err := (&runtimewebhooks.ExtensionConfig{}).SetupWebhookWithManager(mgr); err != nil {
   325  		klog.Fatalf("unable to create webhook for extensionconfig: %+v", err)
   326  	}
   327  	if err := (&expipamwebhooks.IPAddress{}).SetupWebhookWithManager(mgr); err != nil {
   328  		klog.Fatalf("unable to create webhook for ipaddress: %v", err)
   329  	}
   330  	if err := (&expipamwebhooks.IPAddressClaim{}).SetupWebhookWithManager(mgr); err != nil {
   331  		klog.Fatalf("unable to create webhook for ipaddressclaim: %v", err)
   332  	}
   333  
   334  	return &Environment{
   335  		Manager: mgr,
   336  		Client:  mgr.GetClient(),
   337  		Config:  mgr.GetConfig(),
   338  		env:     env,
   339  	}
   340  }
   341  
   342  // start starts the manager.
   343  func (e *Environment) start(ctx context.Context) {
   344  	ctx, cancel := context.WithCancel(ctx)
   345  	e.cancelManager = cancel
   346  
   347  	go func() {
   348  		fmt.Println("Starting the test environment manager")
   349  		if err := e.Manager.Start(ctx); err != nil {
   350  			panic(fmt.Sprintf("Failed to start the test environment manager: %v", err))
   351  		}
   352  	}()
   353  	<-e.Manager.Elected()
   354  	e.waitForWebhooks()
   355  }
   356  
   357  // stop stops the test environment.
   358  func (e *Environment) stop() error {
   359  	fmt.Println("Stopping the test environment")
   360  	e.cancelManager()
   361  	return e.env.Stop()
   362  }
   363  
   364  // waitForWebhooks waits for the webhook server to be available.
   365  func (e *Environment) waitForWebhooks() {
   366  	port := e.env.WebhookInstallOptions.LocalServingPort
   367  
   368  	klog.V(2).Infof("Waiting for webhook port %d to be open prior to running tests", port)
   369  	timeout := 1 * time.Second
   370  	for {
   371  		time.Sleep(1 * time.Second)
   372  		conn, err := net.DialTimeout("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(port)), timeout)
   373  		if err != nil {
   374  			klog.V(2).Infof("Webhook port is not ready, will retry in %v: %s", timeout, err)
   375  			continue
   376  		}
   377  		if err := conn.Close(); err != nil {
   378  			klog.V(2).Infof("Closing connection when testing if webhook port is ready failed: %v", err)
   379  		}
   380  		klog.V(2).Info("Webhook port is now open. Continuing with tests...")
   381  		return
   382  	}
   383  }
   384  
   385  // CreateKubeconfigSecret generates a new Kubeconfig secret from the envtest config.
   386  func (e *Environment) CreateKubeconfigSecret(ctx context.Context, cluster *clusterv1.Cluster) error {
   387  	return e.Create(ctx, kubeconfig.GenerateSecret(cluster, kubeconfig.FromEnvTestConfig(e.Config, cluster)))
   388  }
   389  
   390  // Cleanup deletes all the given objects.
   391  func (e *Environment) Cleanup(ctx context.Context, objs ...client.Object) error {
   392  	errs := []error{}
   393  	for _, o := range objs {
   394  		err := e.Client.Delete(ctx, o)
   395  		if apierrors.IsNotFound(err) {
   396  			continue
   397  		}
   398  		errs = append(errs, err)
   399  	}
   400  	return kerrors.NewAggregate(errs)
   401  }
   402  
   403  // CleanupAndWait deletes all the given objects and waits for the cache to be updated accordingly.
   404  //
   405  // NOTE: Waiting for the cache to be updated helps in preventing test flakes due to the cache sync delays.
   406  func (e *Environment) CleanupAndWait(ctx context.Context, objs ...client.Object) error {
   407  	if err := e.Cleanup(ctx, objs...); err != nil {
   408  		return err
   409  	}
   410  
   411  	// Makes sure the cache is updated with the deleted object
   412  	errs := []error{}
   413  	for _, o := range objs {
   414  		// Ignoring namespaces because in testenv the namespace cleaner is not running.
   415  		if o.GetObjectKind().GroupVersionKind().GroupKind() == corev1.SchemeGroupVersion.WithKind("Namespace").GroupKind() {
   416  			continue
   417  		}
   418  
   419  		oCopy := o.DeepCopyObject().(client.Object)
   420  		key := client.ObjectKeyFromObject(o)
   421  		err := wait.ExponentialBackoff(
   422  			cacheSyncBackoff,
   423  			func() (done bool, err error) {
   424  				if err := e.Get(ctx, key, oCopy); err != nil {
   425  					if apierrors.IsNotFound(err) {
   426  						return true, nil
   427  					}
   428  					return false, err
   429  				}
   430  				return false, nil
   431  			})
   432  		errs = append(errs, errors.Wrapf(err, "key %s, %s is not being deleted from the testenv client cache", o.GetObjectKind().GroupVersionKind().String(), key))
   433  	}
   434  	return kerrors.NewAggregate(errs)
   435  }
   436  
   437  // CreateAndWait creates the given object and waits for the cache to be updated accordingly.
   438  //
   439  // NOTE: Waiting for the cache to be updated helps in preventing test flakes due to the cache sync delays.
   440  func (e *Environment) CreateAndWait(ctx context.Context, obj client.Object, opts ...client.CreateOption) error {
   441  	if err := e.Client.Create(ctx, obj, opts...); err != nil {
   442  		return err
   443  	}
   444  
   445  	// Makes sure the cache is updated with the new object
   446  	objCopy := obj.DeepCopyObject().(client.Object)
   447  	key := client.ObjectKeyFromObject(obj)
   448  	if err := wait.ExponentialBackoff(
   449  		cacheSyncBackoff,
   450  		func() (done bool, err error) {
   451  			if err := e.Get(ctx, key, objCopy); err != nil {
   452  				if apierrors.IsNotFound(err) {
   453  					return false, nil
   454  				}
   455  				return false, err
   456  			}
   457  			return true, nil
   458  		}); err != nil {
   459  		return errors.Wrapf(err, "object %s, %s is not being added to the testenv client cache", obj.GetObjectKind().GroupVersionKind().String(), key)
   460  	}
   461  	return nil
   462  }
   463  
   464  // PatchAndWait creates or updates the given object using server-side apply and waits for the cache to be updated accordingly.
   465  //
   466  // NOTE: Waiting for the cache to be updated helps in preventing test flakes due to the cache sync delays.
   467  func (e *Environment) PatchAndWait(ctx context.Context, obj client.Object, opts ...client.PatchOption) error {
   468  	key := client.ObjectKeyFromObject(obj)
   469  	objCopy := obj.DeepCopyObject().(client.Object)
   470  	if err := e.GetAPIReader().Get(ctx, key, objCopy); err != nil {
   471  		if !apierrors.IsNotFound(err) {
   472  			return err
   473  		}
   474  	}
   475  	// Store old resource version, empty string if not found.
   476  	oldResourceVersion := objCopy.GetResourceVersion()
   477  
   478  	if err := e.Client.Patch(ctx, obj, client.Apply, opts...); err != nil {
   479  		return err
   480  	}
   481  
   482  	// Makes sure the cache is updated with the new object
   483  	if err := wait.ExponentialBackoff(
   484  		cacheSyncBackoff,
   485  		func() (done bool, err error) {
   486  			if err := e.Get(ctx, key, objCopy); err != nil {
   487  				if apierrors.IsNotFound(err) {
   488  					return false, nil
   489  				}
   490  				return false, err
   491  			}
   492  			if objCopy.GetResourceVersion() == oldResourceVersion {
   493  				return false, nil
   494  			}
   495  			return true, nil
   496  		}); err != nil {
   497  		return errors.Wrapf(err, "object %s, %s is not being added to or did not get updated in the testenv client cache", obj.GetObjectKind().GroupVersionKind().String(), key)
   498  	}
   499  	return nil
   500  }
   501  
   502  // CreateNamespace creates a new namespace with a generated name.
   503  func (e *Environment) CreateNamespace(ctx context.Context, generateName string) (*corev1.Namespace, error) {
   504  	ns := &corev1.Namespace{
   505  		ObjectMeta: metav1.ObjectMeta{
   506  			GenerateName: fmt.Sprintf("%s-", generateName),
   507  			Labels: map[string]string{
   508  				"testenv/original-name": generateName,
   509  			},
   510  		},
   511  	}
   512  	if err := e.Client.Create(ctx, ns); err != nil {
   513  		return nil, err
   514  	}
   515  
   516  	return ns, nil
   517  }