github.com/juju/juju@v0.0.0-20240430160146-1752b71fcf00/caas/kubernetes/provider/admissionregistration_test.go (about)

     1  // Copyright 2020 Canonical Ltd.
     2  // Licensed under the AGPLv3, see LICENCE file for details.
     3  
     4  package provider_test
     5  
     6  import (
     7  	"encoding/base64"
     8  
     9  	jc "github.com/juju/testing/checkers"
    10  	"go.uber.org/mock/gomock"
    11  	gc "gopkg.in/check.v1"
    12  	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
    13  	admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1"
    14  	apps "k8s.io/api/apps/v1"
    15  	appsv1 "k8s.io/api/apps/v1"
    16  	core "k8s.io/api/core/v1"
    17  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    18  	k8sversion "k8s.io/apimachinery/pkg/version"
    19  	"k8s.io/utils/pointer"
    20  
    21  	"github.com/juju/juju/caas"
    22  	"github.com/juju/juju/caas/kubernetes/provider"
    23  	k8sspecs "github.com/juju/juju/caas/kubernetes/provider/specs"
    24  	"github.com/juju/juju/core/config"
    25  	"github.com/juju/juju/core/resources"
    26  	"github.com/juju/juju/core/status"
    27  	"github.com/juju/juju/testing"
    28  )
    29  
    30  func (s *K8sBrokerSuite) assertMutatingWebhookConfigurations(c *gc.C, cfgs []k8sspecs.K8sMutatingWebhook, assertCalls ...any) {
    31  
    32  	basicPodSpec := getBasicPodspec()
    33  	basicPodSpec.ProviderPod = &k8sspecs.K8sPodSpec{
    34  		KubernetesResources: &k8sspecs.KubernetesResources{
    35  			MutatingWebhookConfigurations: cfgs,
    36  		},
    37  	}
    38  	workloadSpec, err := provider.PrepareWorkloadSpec(
    39  		"app-name", "app-name", basicPodSpec, resources.DockerImageDetails{RegistryPath: "operator/image-path"},
    40  	)
    41  	c.Assert(err, jc.ErrorIsNil)
    42  	podSpec := provider.Pod(workloadSpec).PodSpec
    43  
    44  	numUnits := int32(2)
    45  	statefulSetArg := &appsv1.StatefulSet{
    46  		ObjectMeta: metav1.ObjectMeta{
    47  			Name:   "app-name",
    48  			Labels: map[string]string{"app.kubernetes.io/managed-by": "juju", "app.kubernetes.io/name": "app-name"},
    49  			Annotations: map[string]string{
    50  				"app.juju.is/uuid":               "appuuid",
    51  				"controller.juju.is/id":          testing.ControllerTag.Id(),
    52  				"charm.juju.is/modified-version": "0",
    53  			},
    54  		},
    55  		Spec: appsv1.StatefulSetSpec{
    56  			Replicas: &numUnits,
    57  			Selector: &metav1.LabelSelector{
    58  				MatchLabels: map[string]string{"app.kubernetes.io/name": "app-name"},
    59  			},
    60  			RevisionHistoryLimit: pointer.Int32Ptr(0),
    61  			Template: core.PodTemplateSpec{
    62  				ObjectMeta: metav1.ObjectMeta{
    63  					Labels: map[string]string{"app.kubernetes.io/name": "app-name"},
    64  					Annotations: map[string]string{
    65  						"apparmor.security.beta.kubernetes.io/pod": "runtime/default",
    66  						"seccomp.security.beta.kubernetes.io/pod":  "docker/default",
    67  						"controller.juju.is/id":                    testing.ControllerTag.Id(),
    68  						"charm.juju.is/modified-version":           "0",
    69  					},
    70  				},
    71  				Spec: podSpec,
    72  			},
    73  			PodManagementPolicy: apps.ParallelPodManagement,
    74  			ServiceName:         "app-name-endpoints",
    75  		},
    76  	}
    77  
    78  	serviceArg := *basicServiceArg
    79  	serviceArg.Spec.Type = core.ServiceTypeClusterIP
    80  
    81  	assertCalls = append(
    82  		[]any{
    83  			s.mockStatefulSets.EXPECT().Get(gomock.Any(), "juju-operator-app-name", metav1.GetOptions{}).
    84  				Return(nil, s.k8sNotFoundError()),
    85  		},
    86  		assertCalls...,
    87  	)
    88  
    89  	ociImageSecret := s.getOCIImageSecret(c, nil)
    90  	assertCalls = append(assertCalls, []any{
    91  		s.mockSecrets.EXPECT().Create(gomock.Any(), ociImageSecret, metav1.CreateOptions{}).
    92  			Return(ociImageSecret, nil),
    93  		s.mockServices.EXPECT().Get(gomock.Any(), "app-name", metav1.GetOptions{}).
    94  			Return(nil, s.k8sNotFoundError()),
    95  		s.mockServices.EXPECT().Update(gomock.Any(), &serviceArg, metav1.UpdateOptions{}).
    96  			Return(nil, s.k8sNotFoundError()),
    97  		s.mockServices.EXPECT().Create(gomock.Any(), &serviceArg, metav1.CreateOptions{}).
    98  			Return(nil, nil),
    99  		s.mockServices.EXPECT().Get(gomock.Any(), "app-name-endpoints", metav1.GetOptions{}).
   100  			Return(nil, s.k8sNotFoundError()),
   101  		s.mockServices.EXPECT().Update(gomock.Any(), basicHeadlessServiceArg, metav1.UpdateOptions{}).
   102  			Return(nil, s.k8sNotFoundError()),
   103  		s.mockServices.EXPECT().Create(gomock.Any(), basicHeadlessServiceArg, metav1.CreateOptions{}).
   104  			Return(nil, nil),
   105  		s.mockStatefulSets.EXPECT().Get(gomock.Any(), "app-name", metav1.GetOptions{}).
   106  			Return(statefulSetArg, nil),
   107  		s.mockStatefulSets.EXPECT().Create(gomock.Any(), statefulSetArg, metav1.CreateOptions{}).
   108  			Return(nil, s.k8sAlreadyExistsError()),
   109  		s.mockStatefulSets.EXPECT().Get(gomock.Any(), "app-name", metav1.GetOptions{}).
   110  			Return(statefulSetArg, nil),
   111  		s.mockStatefulSets.EXPECT().Update(gomock.Any(), statefulSetArg, metav1.UpdateOptions{}).
   112  			Return(nil, nil),
   113  	}...)
   114  	gomock.InOrder(assertCalls...)
   115  
   116  	params := &caas.ServiceParams{
   117  		PodSpec: basicPodSpec,
   118  		Deployment: caas.DeploymentParams{
   119  			DeploymentType: caas.DeploymentStateful,
   120  		},
   121  		ImageDetails: resources.DockerImageDetails{RegistryPath: "operator/image-path"},
   122  		ResourceTags: map[string]string{"juju-controller-uuid": testing.ControllerTag.Id()},
   123  	}
   124  	err = s.broker.EnsureService("app-name", func(_ string, _ status.Status, e string, _ map[string]interface{}) error {
   125  		c.Logf("EnsureService error -> %q", e)
   126  		return nil
   127  	}, params, 2, config.ConfigAttributes{
   128  		"kubernetes-service-loadbalancer-ip": "10.0.0.1",
   129  		"kubernetes-service-externalname":    "ext-name",
   130  	})
   131  	c.Assert(err, jc.ErrorIsNil)
   132  }
   133  
   134  func (s *K8sBrokerSuite) TestEnsureMutatingWebhookConfigurationsCreateV1Beta1(c *gc.C) {
   135  	ctrl := s.setupController(c)
   136  	defer ctrl.Finish()
   137  
   138  	s.mockDiscovery.EXPECT().ServerVersion().AnyTimes().Return(&k8sversion.Info{
   139  		Major: "1", Minor: "15",
   140  	}, nil)
   141  
   142  	webhook1Rule1 := admissionregistrationv1beta1.Rule{
   143  		APIGroups:   []string{""},
   144  		APIVersions: []string{"v1"},
   145  		Resources:   []string{"pods"},
   146  	}
   147  	webhookRuleWithOperations1 := admissionregistrationv1beta1.RuleWithOperations{
   148  		Operations: []admissionregistrationv1beta1.OperationType{
   149  			admissionregistrationv1beta1.Create,
   150  			admissionregistrationv1beta1.Update,
   151  		},
   152  	}
   153  	webhookRuleWithOperations1.Rule = webhook1Rule1
   154  	CABundle, err := base64.StdEncoding.DecodeString("YXBwbGVz")
   155  	c.Assert(err, jc.ErrorIsNil)
   156  	webhook1FailurePolicy := admissionregistrationv1beta1.Ignore
   157  	webhook1 := admissionregistrationv1beta1.MutatingWebhook{
   158  		Name:          "example.mutatingwebhookconfiguration.com",
   159  		FailurePolicy: &webhook1FailurePolicy,
   160  		ClientConfig: admissionregistrationv1beta1.WebhookClientConfig{
   161  			Service: &admissionregistrationv1beta1.ServiceReference{
   162  				Name:      "apple-service",
   163  				Namespace: "apples",
   164  				Path:      pointer.StringPtr("/apple"),
   165  			},
   166  			CABundle: CABundle,
   167  		},
   168  		NamespaceSelector: &metav1.LabelSelector{
   169  			MatchExpressions: []metav1.LabelSelectorRequirement{
   170  				{Key: "production", Operator: metav1.LabelSelectorOpDoesNotExist},
   171  			},
   172  		},
   173  		Rules: []admissionregistrationv1beta1.RuleWithOperations{webhookRuleWithOperations1},
   174  	}
   175  
   176  	cfgs := []k8sspecs.K8sMutatingWebhook{
   177  		{
   178  			Meta: k8sspecs.Meta{Name: "example-mutatingwebhookconfiguration"},
   179  			Webhooks: []k8sspecs.K8sMutatingWebhookSpec{
   180  				{
   181  					Version:     k8sspecs.K8sWebhookV1Beta1,
   182  					SpecV1Beta1: webhook1,
   183  				},
   184  			},
   185  		},
   186  	}
   187  
   188  	cfg1 := &admissionregistrationv1beta1.MutatingWebhookConfiguration{
   189  		ObjectMeta: metav1.ObjectMeta{
   190  			Name:        "test-example-mutatingwebhookconfiguration",
   191  			Namespace:   "test",
   192  			Labels:      map[string]string{"app.kubernetes.io/managed-by": "juju", "app.kubernetes.io/name": "app-name", "model.juju.is/name": "test"},
   193  			Annotations: map[string]string{"controller.juju.is/id": testing.ControllerTag.Id()},
   194  		},
   195  		Webhooks: []admissionregistrationv1beta1.MutatingWebhook{webhook1},
   196  	}
   197  
   198  	s.assertMutatingWebhookConfigurations(
   199  		c, cfgs,
   200  		s.mockMutatingWebhookConfigurationV1Beta1.EXPECT().Create(gomock.Any(), cfg1, metav1.CreateOptions{}).Return(cfg1, nil),
   201  	)
   202  }
   203  
   204  func (s *K8sBrokerSuite) TestEnsureMutatingWebhookConfigurationsCreateV1Beta1Upgrade(c *gc.C) {
   205  	ctrl := s.setupController(c)
   206  	defer ctrl.Finish()
   207  
   208  	s.mockDiscovery.EXPECT().ServerVersion().AnyTimes().Return(&k8sversion.Info{
   209  		Major: "1", Minor: "16",
   210  	}, nil)
   211  
   212  	webhook1Rule1 := admissionregistrationv1beta1.Rule{
   213  		APIGroups:   []string{""},
   214  		APIVersions: []string{"v1"},
   215  		Resources:   []string{"pods"},
   216  	}
   217  	webhookRuleWithOperations1 := admissionregistrationv1beta1.RuleWithOperations{
   218  		Operations: []admissionregistrationv1beta1.OperationType{
   219  			admissionregistrationv1beta1.Create,
   220  			admissionregistrationv1beta1.Update,
   221  		},
   222  	}
   223  	webhookRuleWithOperations1.Rule = webhook1Rule1
   224  	CABundle, err := base64.StdEncoding.DecodeString("YXBwbGVz")
   225  	c.Assert(err, jc.ErrorIsNil)
   226  	webhook1FailurePolicy := admissionregistrationv1beta1.Ignore
   227  	webhook1 := admissionregistrationv1beta1.MutatingWebhook{
   228  		Name:          "example.mutatingwebhookconfiguration.com",
   229  		FailurePolicy: &webhook1FailurePolicy,
   230  		ClientConfig: admissionregistrationv1beta1.WebhookClientConfig{
   231  			Service: &admissionregistrationv1beta1.ServiceReference{
   232  				Name:      "apple-service",
   233  				Namespace: "apples",
   234  				Path:      pointer.StringPtr("/apple"),
   235  			},
   236  			CABundle: CABundle,
   237  		},
   238  		NamespaceSelector: &metav1.LabelSelector{
   239  			MatchExpressions: []metav1.LabelSelectorRequirement{
   240  				{Key: "production", Operator: metav1.LabelSelectorOpDoesNotExist},
   241  			},
   242  		},
   243  		Rules: []admissionregistrationv1beta1.RuleWithOperations{webhookRuleWithOperations1},
   244  	}
   245  
   246  	cfgs := []k8sspecs.K8sMutatingWebhook{
   247  		{
   248  			Meta: k8sspecs.Meta{Name: "example-mutatingwebhookconfiguration"},
   249  			Webhooks: []k8sspecs.K8sMutatingWebhookSpec{
   250  				{
   251  					Version:     k8sspecs.K8sWebhookV1Beta1,
   252  					SpecV1Beta1: webhook1,
   253  				},
   254  			},
   255  		},
   256  	}
   257  
   258  	cfg2 := func() *admissionregistrationv1.MutatingWebhookConfiguration {
   259  		webhook2Rule1 := admissionregistrationv1.Rule{
   260  			APIGroups:   []string{""},
   261  			APIVersions: []string{"v1"},
   262  			Resources:   []string{"pods"},
   263  		}
   264  		webhookRuleWithOperations2 := admissionregistrationv1.RuleWithOperations{
   265  			Operations: []admissionregistrationv1.OperationType{
   266  				admissionregistrationv1.Create,
   267  				admissionregistrationv1.Update,
   268  			},
   269  		}
   270  		webhookRuleWithOperations2.Rule = webhook2Rule1
   271  		CABundle, err := base64.StdEncoding.DecodeString("YXBwbGVz")
   272  		c.Assert(err, jc.ErrorIsNil)
   273  		webhook2FailurePolicy := admissionregistrationv1.Ignore
   274  		sideEffects := admissionregistrationv1.SideEffectClassNoneOnDryRun
   275  		webhook2 := admissionregistrationv1.MutatingWebhook{
   276  			Name:          "example.mutatingwebhookconfiguration.com",
   277  			FailurePolicy: &webhook2FailurePolicy,
   278  			ClientConfig: admissionregistrationv1.WebhookClientConfig{
   279  				Service: &admissionregistrationv1.ServiceReference{
   280  					Name:      "apple-service",
   281  					Namespace: "apples",
   282  					Path:      pointer.StringPtr("/apple"),
   283  				},
   284  				CABundle: CABundle,
   285  			},
   286  			NamespaceSelector: &metav1.LabelSelector{
   287  				MatchExpressions: []metav1.LabelSelectorRequirement{
   288  					{Key: "production", Operator: metav1.LabelSelectorOpDoesNotExist},
   289  				},
   290  			},
   291  			Rules:                   []admissionregistrationv1.RuleWithOperations{webhookRuleWithOperations2},
   292  			AdmissionReviewVersions: []string{"v1beta1"},
   293  			SideEffects:             &sideEffects,
   294  		}
   295  
   296  		return &admissionregistrationv1.MutatingWebhookConfiguration{
   297  			ObjectMeta: metav1.ObjectMeta{
   298  				Name:        "test-example-mutatingwebhookconfiguration",
   299  				Namespace:   "test",
   300  				Labels:      map[string]string{"app.kubernetes.io/managed-by": "juju", "app.kubernetes.io/name": "app-name", "model.juju.is/name": "test"},
   301  				Annotations: map[string]string{"controller.juju.is/id": testing.ControllerTag.Id()},
   302  			},
   303  			Webhooks: []admissionregistrationv1.MutatingWebhook{webhook2},
   304  		}
   305  	}()
   306  
   307  	s.assertMutatingWebhookConfigurations(
   308  		c, cfgs,
   309  		s.mockMutatingWebhookConfigurationV1.EXPECT().Create(gomock.Any(), cfg2, gomock.Any()).Return(cfg2, nil),
   310  	)
   311  }
   312  
   313  func (s *K8sBrokerSuite) TestEnsureMutatingWebhookConfigurationsCreateKeepNameV1Beta1(c *gc.C) {
   314  	ctrl := s.setupController(c)
   315  	defer ctrl.Finish()
   316  
   317  	s.mockDiscovery.EXPECT().ServerVersion().AnyTimes().Return(&k8sversion.Info{
   318  		Major: "1", Minor: "15",
   319  	}, nil)
   320  
   321  	webhook1Rule1 := admissionregistrationv1beta1.Rule{
   322  		APIGroups:   []string{""},
   323  		APIVersions: []string{"v1"},
   324  		Resources:   []string{"pods"},
   325  	}
   326  	webhookRuleWithOperations1 := admissionregistrationv1beta1.RuleWithOperations{
   327  		Operations: []admissionregistrationv1beta1.OperationType{
   328  			admissionregistrationv1beta1.Create,
   329  			admissionregistrationv1beta1.Update,
   330  		},
   331  	}
   332  	webhookRuleWithOperations1.Rule = webhook1Rule1
   333  	CABundle, err := base64.StdEncoding.DecodeString("YXBwbGVz")
   334  	c.Assert(err, jc.ErrorIsNil)
   335  	webhook1FailurePolicy := admissionregistrationv1beta1.Ignore
   336  	webhook1 := admissionregistrationv1beta1.MutatingWebhook{
   337  		Name:          "example.mutatingwebhookconfiguration.com",
   338  		FailurePolicy: &webhook1FailurePolicy,
   339  		ClientConfig: admissionregistrationv1beta1.WebhookClientConfig{
   340  			Service: &admissionregistrationv1beta1.ServiceReference{
   341  				Name:      "apple-service",
   342  				Namespace: "apples",
   343  				Path:      pointer.StringPtr("/apple"),
   344  			},
   345  			CABundle: CABundle,
   346  		},
   347  		NamespaceSelector: &metav1.LabelSelector{
   348  			MatchExpressions: []metav1.LabelSelectorRequirement{
   349  				{Key: "production", Operator: metav1.LabelSelectorOpDoesNotExist},
   350  			},
   351  		},
   352  		Rules: []admissionregistrationv1beta1.RuleWithOperations{webhookRuleWithOperations1},
   353  	}
   354  
   355  	cfgs := []k8sspecs.K8sMutatingWebhook{
   356  		{
   357  			Meta: k8sspecs.Meta{
   358  				Name:        "example-mutatingwebhookconfiguration",
   359  				Annotations: map[string]string{"model.juju.is/disable-prefix": "true"},
   360  			},
   361  			Webhooks: []k8sspecs.K8sMutatingWebhookSpec{
   362  				{
   363  					Version:     k8sspecs.K8sWebhookV1Beta1,
   364  					SpecV1Beta1: webhook1,
   365  				},
   366  			},
   367  		},
   368  	}
   369  
   370  	cfg1 := &admissionregistrationv1beta1.MutatingWebhookConfiguration{
   371  		ObjectMeta: metav1.ObjectMeta{
   372  			Name:        "example-mutatingwebhookconfiguration", // This name kept no change.
   373  			Namespace:   "test",
   374  			Labels:      map[string]string{"app.kubernetes.io/managed-by": "juju", "app.kubernetes.io/name": "app-name", "model.juju.is/name": "test"},
   375  			Annotations: map[string]string{"controller.juju.is/id": testing.ControllerTag.Id(), "model.juju.is/disable-prefix": "true"},
   376  		},
   377  		Webhooks: []admissionregistrationv1beta1.MutatingWebhook{webhook1},
   378  	}
   379  
   380  	s.assertMutatingWebhookConfigurations(
   381  		c, cfgs,
   382  		s.mockMutatingWebhookConfigurationV1Beta1.EXPECT().Create(gomock.Any(), cfg1, gomock.Any()).Return(cfg1, nil),
   383  	)
   384  }
   385  
   386  func (s *K8sBrokerSuite) TestEnsureMutatingWebhookConfigurationsUpdateV1Beta1(c *gc.C) {
   387  	ctrl := s.setupController(c)
   388  	defer ctrl.Finish()
   389  
   390  	s.mockDiscovery.EXPECT().ServerVersion().AnyTimes().Return(&k8sversion.Info{
   391  		Major: "1", Minor: "15",
   392  	}, nil)
   393  
   394  	webhook1Rule1 := admissionregistrationv1beta1.Rule{
   395  		APIGroups:   []string{""},
   396  		APIVersions: []string{"v1"},
   397  		Resources:   []string{"pods"},
   398  	}
   399  	webhookRuleWithOperations1 := admissionregistrationv1beta1.RuleWithOperations{
   400  		Operations: []admissionregistrationv1beta1.OperationType{
   401  			admissionregistrationv1beta1.Create,
   402  			admissionregistrationv1beta1.Update,
   403  		},
   404  	}
   405  	webhookRuleWithOperations1.Rule = webhook1Rule1
   406  	CABundle, err := base64.StdEncoding.DecodeString("YXBwbGVz")
   407  	c.Assert(err, jc.ErrorIsNil)
   408  	webhook1FailurePolicy := admissionregistrationv1beta1.Ignore
   409  	webhook1 := admissionregistrationv1beta1.MutatingWebhook{
   410  		Name:          "example.mutatingwebhookconfiguration.com",
   411  		FailurePolicy: &webhook1FailurePolicy,
   412  		ClientConfig: admissionregistrationv1beta1.WebhookClientConfig{
   413  			Service: &admissionregistrationv1beta1.ServiceReference{
   414  				Name:      "apple-service",
   415  				Namespace: "apples",
   416  				Path:      pointer.StringPtr("/apple"),
   417  			},
   418  			CABundle: CABundle,
   419  		},
   420  		NamespaceSelector: &metav1.LabelSelector{
   421  			MatchExpressions: []metav1.LabelSelectorRequirement{
   422  				{Key: "production", Operator: metav1.LabelSelectorOpDoesNotExist},
   423  			},
   424  		},
   425  		Rules: []admissionregistrationv1beta1.RuleWithOperations{webhookRuleWithOperations1},
   426  	}
   427  
   428  	cfgs := []k8sspecs.K8sMutatingWebhook{
   429  		{
   430  			Meta: k8sspecs.Meta{Name: "example-mutatingwebhookconfiguration"},
   431  			Webhooks: []k8sspecs.K8sMutatingWebhookSpec{
   432  				{
   433  					Version:     k8sspecs.K8sWebhookV1Beta1,
   434  					SpecV1Beta1: webhook1,
   435  				},
   436  			},
   437  		},
   438  	}
   439  
   440  	cfg1 := &admissionregistrationv1beta1.MutatingWebhookConfiguration{
   441  		ObjectMeta: metav1.ObjectMeta{
   442  			Name:        "test-example-mutatingwebhookconfiguration",
   443  			Namespace:   "test",
   444  			Labels:      map[string]string{"app.kubernetes.io/managed-by": "juju", "app.kubernetes.io/name": "app-name", "model.juju.is/name": "test"},
   445  			Annotations: map[string]string{"controller.juju.is/id": testing.ControllerTag.Id()},
   446  		},
   447  		Webhooks: []admissionregistrationv1beta1.MutatingWebhook{webhook1},
   448  	}
   449  
   450  	s.assertMutatingWebhookConfigurations(
   451  		c, cfgs,
   452  		s.mockMutatingWebhookConfigurationV1Beta1.EXPECT().Create(gomock.Any(), cfg1, gomock.Any()).Return(cfg1, s.k8sAlreadyExistsError()),
   453  		s.mockMutatingWebhookConfigurationV1Beta1.EXPECT().
   454  			List(gomock.Any(), metav1.ListOptions{LabelSelector: "app.kubernetes.io/managed-by=juju,app.kubernetes.io/name=app-name,model.juju.is/name=test"}).
   455  			Return(&admissionregistrationv1beta1.MutatingWebhookConfigurationList{Items: []admissionregistrationv1beta1.MutatingWebhookConfiguration{*cfg1}}, nil),
   456  		s.mockMutatingWebhookConfigurationV1Beta1.EXPECT().
   457  			Get(gomock.Any(), "test-example-mutatingwebhookconfiguration", metav1.GetOptions{}).
   458  			Return(cfg1, nil),
   459  		s.mockMutatingWebhookConfigurationV1Beta1.EXPECT().Update(gomock.Any(), cfg1, gomock.Any()).Return(cfg1, nil),
   460  	)
   461  }
   462  
   463  func (s *K8sBrokerSuite) TestEnsureMutatingWebhookConfigurationsCreateV1(c *gc.C) {
   464  	ctrl := s.setupController(c)
   465  	defer ctrl.Finish()
   466  
   467  	s.mockDiscovery.EXPECT().ServerVersion().AnyTimes().Return(&k8sversion.Info{
   468  		Major: "1", Minor: "16",
   469  	}, nil)
   470  
   471  	webhook1Rule1 := admissionregistrationv1.Rule{
   472  		APIGroups:   []string{""},
   473  		APIVersions: []string{"v1"},
   474  		Resources:   []string{"pods"},
   475  	}
   476  	webhookRuleWithOperations1 := admissionregistrationv1.RuleWithOperations{
   477  		Operations: []admissionregistrationv1.OperationType{
   478  			admissionregistrationv1.Create,
   479  			admissionregistrationv1.Update,
   480  		},
   481  	}
   482  	webhookRuleWithOperations1.Rule = webhook1Rule1
   483  	CABundle, err := base64.StdEncoding.DecodeString("YXBwbGVz")
   484  	c.Assert(err, jc.ErrorIsNil)
   485  	webhook1FailurePolicy := admissionregistrationv1.Ignore
   486  	webhook1 := admissionregistrationv1.MutatingWebhook{
   487  		Name:          "example.mutatingwebhookconfiguration.com",
   488  		FailurePolicy: &webhook1FailurePolicy,
   489  		ClientConfig: admissionregistrationv1.WebhookClientConfig{
   490  			Service: &admissionregistrationv1.ServiceReference{
   491  				Name:      "apple-service",
   492  				Namespace: "apples",
   493  				Path:      pointer.StringPtr("/apple"),
   494  			},
   495  			CABundle: CABundle,
   496  		},
   497  		NamespaceSelector: &metav1.LabelSelector{
   498  			MatchExpressions: []metav1.LabelSelectorRequirement{
   499  				{Key: "production", Operator: metav1.LabelSelectorOpDoesNotExist},
   500  			},
   501  		},
   502  		Rules: []admissionregistrationv1.RuleWithOperations{webhookRuleWithOperations1},
   503  	}
   504  
   505  	cfgs := []k8sspecs.K8sMutatingWebhook{
   506  		{
   507  			Meta: k8sspecs.Meta{Name: "example-mutatingwebhookconfiguration"},
   508  			Webhooks: []k8sspecs.K8sMutatingWebhookSpec{
   509  				{
   510  					Version: k8sspecs.K8sWebhookV1,
   511  					SpecV1:  webhook1,
   512  				},
   513  			},
   514  		},
   515  	}
   516  
   517  	cfg1 := &admissionregistrationv1.MutatingWebhookConfiguration{
   518  		ObjectMeta: metav1.ObjectMeta{
   519  			Name:        "test-example-mutatingwebhookconfiguration",
   520  			Namespace:   "test",
   521  			Labels:      map[string]string{"app.kubernetes.io/managed-by": "juju", "app.kubernetes.io/name": "app-name", "model.juju.is/name": "test"},
   522  			Annotations: map[string]string{"controller.juju.is/id": testing.ControllerTag.Id()},
   523  		},
   524  		Webhooks: []admissionregistrationv1.MutatingWebhook{webhook1},
   525  	}
   526  
   527  	s.assertMutatingWebhookConfigurations(
   528  		c, cfgs,
   529  		s.mockMutatingWebhookConfigurationV1.EXPECT().Create(gomock.Any(), cfg1, gomock.Any()).Return(cfg1, nil),
   530  	)
   531  }
   532  
   533  func (s *K8sBrokerSuite) TestEnsureMutatingWebhookConfigurationsCreateKeepNameV1(c *gc.C) {
   534  	ctrl := s.setupController(c)
   535  	defer ctrl.Finish()
   536  
   537  	s.mockDiscovery.EXPECT().ServerVersion().AnyTimes().Return(&k8sversion.Info{
   538  		Major: "1", Minor: "16",
   539  	}, nil)
   540  
   541  	webhook1Rule1 := admissionregistrationv1.Rule{
   542  		APIGroups:   []string{""},
   543  		APIVersions: []string{"v1"},
   544  		Resources:   []string{"pods"},
   545  	}
   546  	webhookRuleWithOperations1 := admissionregistrationv1.RuleWithOperations{
   547  		Operations: []admissionregistrationv1.OperationType{
   548  			admissionregistrationv1.Create,
   549  			admissionregistrationv1.Update,
   550  		},
   551  	}
   552  	webhookRuleWithOperations1.Rule = webhook1Rule1
   553  	CABundle, err := base64.StdEncoding.DecodeString("YXBwbGVz")
   554  	c.Assert(err, jc.ErrorIsNil)
   555  	webhook1FailurePolicy := admissionregistrationv1.Ignore
   556  	webhook1 := admissionregistrationv1.MutatingWebhook{
   557  		Name:          "example.mutatingwebhookconfiguration.com",
   558  		FailurePolicy: &webhook1FailurePolicy,
   559  		ClientConfig: admissionregistrationv1.WebhookClientConfig{
   560  			Service: &admissionregistrationv1.ServiceReference{
   561  				Name:      "apple-service",
   562  				Namespace: "apples",
   563  				Path:      pointer.StringPtr("/apple"),
   564  			},
   565  			CABundle: CABundle,
   566  		},
   567  		NamespaceSelector: &metav1.LabelSelector{
   568  			MatchExpressions: []metav1.LabelSelectorRequirement{
   569  				{Key: "production", Operator: metav1.LabelSelectorOpDoesNotExist},
   570  			},
   571  		},
   572  		Rules: []admissionregistrationv1.RuleWithOperations{webhookRuleWithOperations1},
   573  	}
   574  
   575  	cfgs := []k8sspecs.K8sMutatingWebhook{
   576  		{
   577  			Meta: k8sspecs.Meta{
   578  				Name:        "example-mutatingwebhookconfiguration",
   579  				Annotations: map[string]string{"model.juju.is/disable-prefix": "true"},
   580  			},
   581  			Webhooks: []k8sspecs.K8sMutatingWebhookSpec{
   582  				{
   583  					Version: k8sspecs.K8sWebhookV1,
   584  					SpecV1:  webhook1,
   585  				},
   586  			},
   587  		},
   588  	}
   589  
   590  	cfg1 := &admissionregistrationv1.MutatingWebhookConfiguration{
   591  		ObjectMeta: metav1.ObjectMeta{
   592  			Name:        "example-mutatingwebhookconfiguration", // This name kept no change.
   593  			Namespace:   "test",
   594  			Labels:      map[string]string{"app.kubernetes.io/managed-by": "juju", "app.kubernetes.io/name": "app-name", "model.juju.is/name": "test"},
   595  			Annotations: map[string]string{"controller.juju.is/id": testing.ControllerTag.Id(), "model.juju.is/disable-prefix": "true"},
   596  		},
   597  		Webhooks: []admissionregistrationv1.MutatingWebhook{webhook1},
   598  	}
   599  
   600  	s.assertMutatingWebhookConfigurations(
   601  		c, cfgs,
   602  		s.mockMutatingWebhookConfigurationV1.EXPECT().Create(gomock.Any(), cfg1, gomock.Any()).Return(cfg1, nil),
   603  	)
   604  }
   605  
   606  func (s *K8sBrokerSuite) TestEnsureMutatingWebhookConfigurationsUpdateV1(c *gc.C) {
   607  	ctrl := s.setupController(c)
   608  	defer ctrl.Finish()
   609  
   610  	s.mockDiscovery.EXPECT().ServerVersion().AnyTimes().Return(&k8sversion.Info{
   611  		Major: "1", Minor: "16",
   612  	}, nil)
   613  
   614  	webhook1Rule1 := admissionregistrationv1.Rule{
   615  		APIGroups:   []string{""},
   616  		APIVersions: []string{"v1"},
   617  		Resources:   []string{"pods"},
   618  	}
   619  	webhookRuleWithOperations1 := admissionregistrationv1.RuleWithOperations{
   620  		Operations: []admissionregistrationv1.OperationType{
   621  			admissionregistrationv1.Create,
   622  			admissionregistrationv1.Update,
   623  		},
   624  	}
   625  	webhookRuleWithOperations1.Rule = webhook1Rule1
   626  	CABundle, err := base64.StdEncoding.DecodeString("YXBwbGVz")
   627  	c.Assert(err, jc.ErrorIsNil)
   628  	webhook1FailurePolicy := admissionregistrationv1.Ignore
   629  	webhook1 := admissionregistrationv1.MutatingWebhook{
   630  		Name:          "example.mutatingwebhookconfiguration.com",
   631  		FailurePolicy: &webhook1FailurePolicy,
   632  		ClientConfig: admissionregistrationv1.WebhookClientConfig{
   633  			Service: &admissionregistrationv1.ServiceReference{
   634  				Name:      "apple-service",
   635  				Namespace: "apples",
   636  				Path:      pointer.StringPtr("/apple"),
   637  			},
   638  			CABundle: CABundle,
   639  		},
   640  		NamespaceSelector: &metav1.LabelSelector{
   641  			MatchExpressions: []metav1.LabelSelectorRequirement{
   642  				{Key: "production", Operator: metav1.LabelSelectorOpDoesNotExist},
   643  			},
   644  		},
   645  		Rules: []admissionregistrationv1.RuleWithOperations{webhookRuleWithOperations1},
   646  	}
   647  
   648  	cfgs := []k8sspecs.K8sMutatingWebhook{
   649  		{
   650  			Meta: k8sspecs.Meta{Name: "example-mutatingwebhookconfiguration"},
   651  			Webhooks: []k8sspecs.K8sMutatingWebhookSpec{
   652  				{
   653  					Version: k8sspecs.K8sWebhookV1,
   654  					SpecV1:  webhook1,
   655  				},
   656  			},
   657  		},
   658  	}
   659  
   660  	cfg1 := &admissionregistrationv1.MutatingWebhookConfiguration{
   661  		ObjectMeta: metav1.ObjectMeta{
   662  			Name:        "test-example-mutatingwebhookconfiguration",
   663  			Namespace:   "test",
   664  			Labels:      map[string]string{"app.kubernetes.io/managed-by": "juju", "app.kubernetes.io/name": "app-name", "model.juju.is/name": "test"},
   665  			Annotations: map[string]string{"controller.juju.is/id": testing.ControllerTag.Id()},
   666  		},
   667  		Webhooks: []admissionregistrationv1.MutatingWebhook{webhook1},
   668  	}
   669  
   670  	s.assertMutatingWebhookConfigurations(
   671  		c, cfgs,
   672  		s.mockMutatingWebhookConfigurationV1.EXPECT().Create(gomock.Any(), cfg1, metav1.CreateOptions{}).Return(cfg1, s.k8sAlreadyExistsError()),
   673  		s.mockMutatingWebhookConfigurationV1.EXPECT().
   674  			List(gomock.Any(), metav1.ListOptions{LabelSelector: "app.kubernetes.io/managed-by=juju,app.kubernetes.io/name=app-name,model.juju.is/name=test"}).
   675  			Return(&admissionregistrationv1.MutatingWebhookConfigurationList{Items: []admissionregistrationv1.MutatingWebhookConfiguration{*cfg1}}, nil),
   676  		s.mockMutatingWebhookConfigurationV1.EXPECT().
   677  			Get(gomock.Any(), "test-example-mutatingwebhookconfiguration", metav1.GetOptions{}).
   678  			Return(cfg1, nil),
   679  		s.mockMutatingWebhookConfigurationV1.EXPECT().Update(gomock.Any(), cfg1, metav1.UpdateOptions{}).Return(cfg1, nil),
   680  	)
   681  }
   682  
   683  func (s *K8sBrokerSuite) assertValidatingWebhookConfigurations(c *gc.C, cfgs []k8sspecs.K8sValidatingWebhook, assertCalls ...any) {
   684  
   685  	basicPodSpec := getBasicPodspec()
   686  	basicPodSpec.ProviderPod = &k8sspecs.K8sPodSpec{
   687  		KubernetesResources: &k8sspecs.KubernetesResources{
   688  			ValidatingWebhookConfigurations: cfgs,
   689  		},
   690  	}
   691  	workloadSpec, err := provider.PrepareWorkloadSpec(
   692  		"app-name", "app-name", basicPodSpec, resources.DockerImageDetails{RegistryPath: "operator/image-path"},
   693  	)
   694  	c.Assert(err, jc.ErrorIsNil)
   695  	podSpec := provider.Pod(workloadSpec).PodSpec
   696  
   697  	numUnits := int32(2)
   698  	statefulSetArg := &appsv1.StatefulSet{
   699  		ObjectMeta: metav1.ObjectMeta{
   700  			Name:   "app-name",
   701  			Labels: map[string]string{"app.kubernetes.io/managed-by": "juju", "app.kubernetes.io/name": "app-name"},
   702  			Annotations: map[string]string{
   703  				"app.juju.is/uuid":               "appuuid",
   704  				"controller.juju.is/id":          testing.ControllerTag.Id(),
   705  				"charm.juju.is/modified-version": "0",
   706  			},
   707  		},
   708  		Spec: appsv1.StatefulSetSpec{
   709  			Replicas: &numUnits,
   710  			Selector: &metav1.LabelSelector{
   711  				MatchLabels: map[string]string{"app.kubernetes.io/name": "app-name"},
   712  			},
   713  			RevisionHistoryLimit: pointer.Int32Ptr(0),
   714  			Template: core.PodTemplateSpec{
   715  				ObjectMeta: metav1.ObjectMeta{
   716  					Labels: map[string]string{"app.kubernetes.io/name": "app-name"},
   717  					Annotations: map[string]string{
   718  						"apparmor.security.beta.kubernetes.io/pod": "runtime/default",
   719  						"seccomp.security.beta.kubernetes.io/pod":  "docker/default",
   720  						"controller.juju.is/id":                    testing.ControllerTag.Id(),
   721  						"charm.juju.is/modified-version":           "0",
   722  					},
   723  				},
   724  				Spec: podSpec,
   725  			},
   726  			PodManagementPolicy: apps.ParallelPodManagement,
   727  			ServiceName:         "app-name-endpoints",
   728  		},
   729  	}
   730  
   731  	serviceArg := *basicServiceArg
   732  	serviceArg.Spec.Type = core.ServiceTypeClusterIP
   733  
   734  	assertCalls = append(
   735  		[]any{
   736  			s.mockStatefulSets.EXPECT().Get(gomock.Any(), "juju-operator-app-name", metav1.GetOptions{}).
   737  				Return(nil, s.k8sNotFoundError()),
   738  		},
   739  		assertCalls...,
   740  	)
   741  
   742  	ociImageSecret := s.getOCIImageSecret(c, nil)
   743  	assertCalls = append(assertCalls, []any{
   744  		s.mockSecrets.EXPECT().Create(gomock.Any(), ociImageSecret, metav1.CreateOptions{}).
   745  			Return(ociImageSecret, nil),
   746  		s.mockServices.EXPECT().Get(gomock.Any(), "app-name", metav1.GetOptions{}).
   747  			Return(nil, s.k8sNotFoundError()),
   748  		s.mockServices.EXPECT().Update(gomock.Any(), &serviceArg, metav1.UpdateOptions{}).
   749  			Return(nil, s.k8sNotFoundError()),
   750  		s.mockServices.EXPECT().Create(gomock.Any(), &serviceArg, metav1.CreateOptions{}).
   751  			Return(nil, nil),
   752  		s.mockServices.EXPECT().Get(gomock.Any(), "app-name-endpoints", metav1.GetOptions{}).
   753  			Return(nil, s.k8sNotFoundError()),
   754  		s.mockServices.EXPECT().Update(gomock.Any(), basicHeadlessServiceArg, metav1.UpdateOptions{}).
   755  			Return(nil, s.k8sNotFoundError()),
   756  		s.mockServices.EXPECT().Create(gomock.Any(), basicHeadlessServiceArg, metav1.CreateOptions{}).
   757  			Return(nil, nil),
   758  		s.mockStatefulSets.EXPECT().Get(gomock.Any(), "app-name", metav1.GetOptions{}).
   759  			Return(statefulSetArg, nil),
   760  		s.mockStatefulSets.EXPECT().Create(gomock.Any(), statefulSetArg, metav1.CreateOptions{}).
   761  			Return(nil, nil),
   762  	}...)
   763  	gomock.InOrder(assertCalls...)
   764  
   765  	params := &caas.ServiceParams{
   766  		PodSpec: basicPodSpec,
   767  		Deployment: caas.DeploymentParams{
   768  			DeploymentType: caas.DeploymentStateful,
   769  		},
   770  		ImageDetails: resources.DockerImageDetails{RegistryPath: "operator/image-path"},
   771  		ResourceTags: map[string]string{"juju-controller-uuid": testing.ControllerTag.Id()},
   772  	}
   773  	err = s.broker.EnsureService("app-name", func(_ string, _ status.Status, e string, _ map[string]interface{}) error {
   774  		c.Logf("EnsureService error -> %q", e)
   775  		return nil
   776  	}, params, 2, config.ConfigAttributes{
   777  		"kubernetes-service-loadbalancer-ip": "10.0.0.1",
   778  		"kubernetes-service-externalname":    "ext-name",
   779  	})
   780  	c.Assert(err, jc.ErrorIsNil)
   781  }
   782  
   783  func (s *K8sBrokerSuite) TestEnsureValidatingWebhookConfigurationsCreateV1Beta1(c *gc.C) {
   784  	ctrl := s.setupController(c)
   785  	defer ctrl.Finish()
   786  
   787  	s.mockDiscovery.EXPECT().ServerVersion().AnyTimes().Return(&k8sversion.Info{
   788  		Major: "1", Minor: "15",
   789  	}, nil)
   790  
   791  	webhook1Rule1 := admissionregistrationv1beta1.Rule{
   792  		APIGroups:   []string{""},
   793  		APIVersions: []string{"v1"},
   794  		Resources:   []string{"pods"},
   795  	}
   796  	webhookRuleWithOperations1 := admissionregistrationv1beta1.RuleWithOperations{
   797  		Operations: []admissionregistrationv1beta1.OperationType{
   798  			admissionregistrationv1beta1.Create,
   799  			admissionregistrationv1beta1.Update,
   800  		},
   801  	}
   802  	webhookRuleWithOperations1.Rule = webhook1Rule1
   803  	CABundle, err := base64.StdEncoding.DecodeString("YXBwbGVz")
   804  	c.Assert(err, jc.ErrorIsNil)
   805  	webhook1FailurePolicy := admissionregistrationv1beta1.Ignore
   806  	webhook1 := admissionregistrationv1beta1.ValidatingWebhook{
   807  		Name:          "example.validatingwebhookconfiguration.com",
   808  		FailurePolicy: &webhook1FailurePolicy,
   809  		ClientConfig: admissionregistrationv1beta1.WebhookClientConfig{
   810  			Service: &admissionregistrationv1beta1.ServiceReference{
   811  				Name:      "apple-service",
   812  				Namespace: "apples",
   813  				Path:      pointer.StringPtr("/apple"),
   814  			},
   815  			CABundle: CABundle,
   816  		},
   817  		NamespaceSelector: &metav1.LabelSelector{
   818  			MatchExpressions: []metav1.LabelSelectorRequirement{
   819  				{Key: "production", Operator: metav1.LabelSelectorOpDoesNotExist},
   820  			},
   821  		},
   822  		Rules: []admissionregistrationv1beta1.RuleWithOperations{webhookRuleWithOperations1},
   823  	}
   824  
   825  	cfgs := []k8sspecs.K8sValidatingWebhook{
   826  		{
   827  			Meta: k8sspecs.Meta{Name: "example-validatingwebhookconfiguration"},
   828  			Webhooks: []k8sspecs.K8sValidatingWebhookSpec{
   829  				{
   830  					Version:     k8sspecs.K8sWebhookV1Beta1,
   831  					SpecV1Beta1: webhook1,
   832  				},
   833  			},
   834  		},
   835  	}
   836  
   837  	cfg1 := &admissionregistrationv1beta1.ValidatingWebhookConfiguration{
   838  		ObjectMeta: metav1.ObjectMeta{
   839  			Name:        "test-example-validatingwebhookconfiguration",
   840  			Namespace:   "test",
   841  			Labels:      map[string]string{"app.kubernetes.io/managed-by": "juju", "app.kubernetes.io/name": "app-name", "model.juju.is/name": "test"},
   842  			Annotations: map[string]string{"controller.juju.is/id": testing.ControllerTag.Id()},
   843  		},
   844  		Webhooks: []admissionregistrationv1beta1.ValidatingWebhook{webhook1},
   845  	}
   846  
   847  	s.assertValidatingWebhookConfigurations(
   848  		c, cfgs,
   849  		s.mockValidatingWebhookConfigurationV1Beta1.EXPECT().Create(gomock.Any(), cfg1, metav1.CreateOptions{}).Return(cfg1, nil),
   850  	)
   851  }
   852  
   853  func (s *K8sBrokerSuite) TestEnsureValidatingWebhookConfigurationsCreateV1Beta1Upgrade(c *gc.C) {
   854  	ctrl := s.setupController(c)
   855  	defer ctrl.Finish()
   856  
   857  	s.mockDiscovery.EXPECT().ServerVersion().AnyTimes().Return(&k8sversion.Info{
   858  		Major: "1", Minor: "16",
   859  	}, nil)
   860  
   861  	webhook1Rule1 := admissionregistrationv1beta1.Rule{
   862  		APIGroups:   []string{""},
   863  		APIVersions: []string{"v1"},
   864  		Resources:   []string{"pods"},
   865  	}
   866  	webhookRuleWithOperations1 := admissionregistrationv1beta1.RuleWithOperations{
   867  		Operations: []admissionregistrationv1beta1.OperationType{
   868  			admissionregistrationv1beta1.Create,
   869  			admissionregistrationv1beta1.Update,
   870  		},
   871  	}
   872  	webhookRuleWithOperations1.Rule = webhook1Rule1
   873  	CABundle, err := base64.StdEncoding.DecodeString("YXBwbGVz")
   874  	c.Assert(err, jc.ErrorIsNil)
   875  	webhook1FailurePolicy := admissionregistrationv1beta1.Ignore
   876  	webhook1 := admissionregistrationv1beta1.ValidatingWebhook{
   877  		Name:          "example.validatingwebhookconfiguration.com",
   878  		FailurePolicy: &webhook1FailurePolicy,
   879  		ClientConfig: admissionregistrationv1beta1.WebhookClientConfig{
   880  			Service: &admissionregistrationv1beta1.ServiceReference{
   881  				Name:      "apple-service",
   882  				Namespace: "apples",
   883  				Path:      pointer.StringPtr("/apple"),
   884  			},
   885  			CABundle: CABundle,
   886  		},
   887  		NamespaceSelector: &metav1.LabelSelector{
   888  			MatchExpressions: []metav1.LabelSelectorRequirement{
   889  				{Key: "production", Operator: metav1.LabelSelectorOpDoesNotExist},
   890  			},
   891  		},
   892  		Rules: []admissionregistrationv1beta1.RuleWithOperations{webhookRuleWithOperations1},
   893  	}
   894  
   895  	cfgs := []k8sspecs.K8sValidatingWebhook{
   896  		{
   897  			Meta: k8sspecs.Meta{Name: "example-validatingwebhookconfiguration"},
   898  			Webhooks: []k8sspecs.K8sValidatingWebhookSpec{
   899  				{
   900  					Version:     k8sspecs.K8sWebhookV1Beta1,
   901  					SpecV1Beta1: webhook1,
   902  				},
   903  			},
   904  		},
   905  	}
   906  
   907  	webhook2 := func() admissionregistrationv1.ValidatingWebhook {
   908  		webhook2Rule1 := admissionregistrationv1.Rule{
   909  			APIGroups:   []string{""},
   910  			APIVersions: []string{"v1"},
   911  			Resources:   []string{"pods"},
   912  		}
   913  		webhookRuleWithOperations2 := admissionregistrationv1.RuleWithOperations{
   914  			Operations: []admissionregistrationv1.OperationType{
   915  				admissionregistrationv1.Create,
   916  				admissionregistrationv1.Update,
   917  			},
   918  		}
   919  		webhookRuleWithOperations2.Rule = webhook2Rule1
   920  		CABundle, err := base64.StdEncoding.DecodeString("YXBwbGVz")
   921  		c.Assert(err, jc.ErrorIsNil)
   922  		webhook2FailurePolicy := admissionregistrationv1.Ignore
   923  		return admissionregistrationv1.ValidatingWebhook{
   924  			Name:          "example.validatingwebhookconfiguration.com",
   925  			FailurePolicy: &webhook2FailurePolicy,
   926  			ClientConfig: admissionregistrationv1.WebhookClientConfig{
   927  				Service: &admissionregistrationv1.ServiceReference{
   928  					Name:      "apple-service",
   929  					Namespace: "apples",
   930  					Path:      pointer.StringPtr("/apple"),
   931  				},
   932  				CABundle: CABundle,
   933  			},
   934  			NamespaceSelector: &metav1.LabelSelector{
   935  				MatchExpressions: []metav1.LabelSelectorRequirement{
   936  					{Key: "production", Operator: metav1.LabelSelectorOpDoesNotExist},
   937  				},
   938  			},
   939  			Rules:                   []admissionregistrationv1.RuleWithOperations{webhookRuleWithOperations2},
   940  			AdmissionReviewVersions: []string{"v1beta1"},
   941  		}
   942  	}()
   943  
   944  	cfg2 := &admissionregistrationv1.ValidatingWebhookConfiguration{
   945  		ObjectMeta: metav1.ObjectMeta{
   946  			Name:        "test-example-validatingwebhookconfiguration",
   947  			Namespace:   "test",
   948  			Labels:      map[string]string{"app.kubernetes.io/managed-by": "juju", "app.kubernetes.io/name": "app-name", "model.juju.is/name": "test"},
   949  			Annotations: map[string]string{"controller.juju.is/id": testing.ControllerTag.Id()},
   950  		},
   951  		Webhooks: []admissionregistrationv1.ValidatingWebhook{webhook2},
   952  	}
   953  
   954  	s.assertValidatingWebhookConfigurations(
   955  		c, cfgs,
   956  		s.mockValidatingWebhookConfigurationV1.EXPECT().Create(gomock.Any(), cfg2, gomock.Any()).Return(cfg2, nil),
   957  	)
   958  }
   959  
   960  func (s *K8sBrokerSuite) TestEnsureValidatingWebhookConfigurationsCreateKeepNameV1Beta1(c *gc.C) {
   961  	ctrl := s.setupController(c)
   962  	defer ctrl.Finish()
   963  
   964  	s.mockDiscovery.EXPECT().ServerVersion().AnyTimes().Return(&k8sversion.Info{
   965  		Major: "1", Minor: "15",
   966  	}, nil)
   967  
   968  	webhook1Rule1 := admissionregistrationv1beta1.Rule{
   969  		APIGroups:   []string{""},
   970  		APIVersions: []string{"v1"},
   971  		Resources:   []string{"pods"},
   972  	}
   973  	webhookRuleWithOperations1 := admissionregistrationv1beta1.RuleWithOperations{
   974  		Operations: []admissionregistrationv1beta1.OperationType{
   975  			admissionregistrationv1beta1.Create,
   976  			admissionregistrationv1beta1.Update,
   977  		},
   978  	}
   979  	webhookRuleWithOperations1.Rule = webhook1Rule1
   980  	CABundle, err := base64.StdEncoding.DecodeString("YXBwbGVz")
   981  	c.Assert(err, jc.ErrorIsNil)
   982  	webhook1FailurePolicy := admissionregistrationv1beta1.Ignore
   983  	webhook1 := admissionregistrationv1beta1.ValidatingWebhook{
   984  		Name:          "example.validatingwebhookconfiguration.com",
   985  		FailurePolicy: &webhook1FailurePolicy,
   986  		ClientConfig: admissionregistrationv1beta1.WebhookClientConfig{
   987  			Service: &admissionregistrationv1beta1.ServiceReference{
   988  				Name:      "apple-service",
   989  				Namespace: "apples",
   990  				Path:      pointer.StringPtr("/apple"),
   991  			},
   992  			CABundle: CABundle,
   993  		},
   994  		NamespaceSelector: &metav1.LabelSelector{
   995  			MatchExpressions: []metav1.LabelSelectorRequirement{
   996  				{Key: "production", Operator: metav1.LabelSelectorOpDoesNotExist},
   997  			},
   998  		},
   999  		Rules: []admissionregistrationv1beta1.RuleWithOperations{webhookRuleWithOperations1},
  1000  	}
  1001  
  1002  	cfgs := []k8sspecs.K8sValidatingWebhook{
  1003  		{
  1004  			Meta: k8sspecs.Meta{
  1005  				Name:        "example-validatingwebhookconfiguration",
  1006  				Annotations: map[string]string{"model.juju.is/disable-prefix": "true"},
  1007  			},
  1008  			Webhooks: []k8sspecs.K8sValidatingWebhookSpec{
  1009  				{
  1010  					Version:     k8sspecs.K8sWebhookV1Beta1,
  1011  					SpecV1Beta1: webhook1,
  1012  				},
  1013  			},
  1014  		},
  1015  	}
  1016  
  1017  	cfg1 := &admissionregistrationv1beta1.ValidatingWebhookConfiguration{
  1018  		ObjectMeta: metav1.ObjectMeta{
  1019  			Name:        "example-validatingwebhookconfiguration", // This name kept no change.
  1020  			Namespace:   "test",
  1021  			Labels:      map[string]string{"app.kubernetes.io/managed-by": "juju", "app.kubernetes.io/name": "app-name", "model.juju.is/name": "test"},
  1022  			Annotations: map[string]string{"controller.juju.is/id": testing.ControllerTag.Id(), "model.juju.is/disable-prefix": "true"},
  1023  		},
  1024  		Webhooks: []admissionregistrationv1beta1.ValidatingWebhook{webhook1},
  1025  	}
  1026  
  1027  	s.assertValidatingWebhookConfigurations(
  1028  		c, cfgs,
  1029  		s.mockValidatingWebhookConfigurationV1Beta1.EXPECT().Create(gomock.Any(), cfg1, gomock.Any()).Return(cfg1, nil),
  1030  	)
  1031  }
  1032  
  1033  func (s *K8sBrokerSuite) TestEnsureValidatingWebhookConfigurationsUpdateV1Beta1(c *gc.C) {
  1034  	ctrl := s.setupController(c)
  1035  	defer ctrl.Finish()
  1036  
  1037  	s.mockDiscovery.EXPECT().ServerVersion().AnyTimes().Return(&k8sversion.Info{
  1038  		Major: "1", Minor: "15",
  1039  	}, nil)
  1040  
  1041  	webhook1Rule1 := admissionregistrationv1beta1.Rule{
  1042  		APIGroups:   []string{""},
  1043  		APIVersions: []string{"v1"},
  1044  		Resources:   []string{"pods"},
  1045  	}
  1046  	webhookRuleWithOperations1 := admissionregistrationv1beta1.RuleWithOperations{
  1047  		Operations: []admissionregistrationv1beta1.OperationType{
  1048  			admissionregistrationv1beta1.Create,
  1049  			admissionregistrationv1beta1.Update,
  1050  		},
  1051  	}
  1052  	webhookRuleWithOperations1.Rule = webhook1Rule1
  1053  	CABundle, err := base64.StdEncoding.DecodeString("YXBwbGVz")
  1054  	c.Assert(err, jc.ErrorIsNil)
  1055  	webhook1FailurePolicy := admissionregistrationv1beta1.Ignore
  1056  	webhook1 := admissionregistrationv1beta1.ValidatingWebhook{
  1057  		Name:          "example.validatingwebhookconfiguration.com",
  1058  		FailurePolicy: &webhook1FailurePolicy,
  1059  		ClientConfig: admissionregistrationv1beta1.WebhookClientConfig{
  1060  			Service: &admissionregistrationv1beta1.ServiceReference{
  1061  				Name:      "apple-service",
  1062  				Namespace: "apples",
  1063  				Path:      pointer.StringPtr("/apple"),
  1064  			},
  1065  			CABundle: CABundle,
  1066  		},
  1067  		NamespaceSelector: &metav1.LabelSelector{
  1068  			MatchExpressions: []metav1.LabelSelectorRequirement{
  1069  				{Key: "production", Operator: metav1.LabelSelectorOpDoesNotExist},
  1070  			},
  1071  		},
  1072  		Rules: []admissionregistrationv1beta1.RuleWithOperations{webhookRuleWithOperations1},
  1073  	}
  1074  
  1075  	cfgs := []k8sspecs.K8sValidatingWebhook{
  1076  		{
  1077  			Meta: k8sspecs.Meta{Name: "example-validatingwebhookconfiguration"},
  1078  			Webhooks: []k8sspecs.K8sValidatingWebhookSpec{
  1079  				{
  1080  					Version:     k8sspecs.K8sWebhookV1Beta1,
  1081  					SpecV1Beta1: webhook1,
  1082  				},
  1083  			},
  1084  		},
  1085  	}
  1086  
  1087  	cfg1 := &admissionregistrationv1beta1.ValidatingWebhookConfiguration{
  1088  		ObjectMeta: metav1.ObjectMeta{
  1089  			Name:        "test-example-validatingwebhookconfiguration",
  1090  			Namespace:   "test",
  1091  			Labels:      map[string]string{"app.kubernetes.io/managed-by": "juju", "app.kubernetes.io/name": "app-name", "model.juju.is/name": "test"},
  1092  			Annotations: map[string]string{"controller.juju.is/id": testing.ControllerTag.Id()},
  1093  		},
  1094  		Webhooks: []admissionregistrationv1beta1.ValidatingWebhook{webhook1},
  1095  	}
  1096  
  1097  	s.assertValidatingWebhookConfigurations(
  1098  		c, cfgs,
  1099  		s.mockValidatingWebhookConfigurationV1Beta1.EXPECT().Create(gomock.Any(), cfg1, gomock.Any()).Return(cfg1, s.k8sAlreadyExistsError()),
  1100  		s.mockValidatingWebhookConfigurationV1Beta1.EXPECT().
  1101  			List(gomock.Any(), metav1.ListOptions{LabelSelector: "app.kubernetes.io/managed-by=juju,app.kubernetes.io/name=app-name,model.juju.is/name=test"}).
  1102  			Return(&admissionregistrationv1beta1.ValidatingWebhookConfigurationList{Items: []admissionregistrationv1beta1.ValidatingWebhookConfiguration{*cfg1}}, nil),
  1103  		s.mockValidatingWebhookConfigurationV1Beta1.EXPECT().
  1104  			Get(gomock.Any(), "test-example-validatingwebhookconfiguration", metav1.GetOptions{}).
  1105  			Return(cfg1, nil),
  1106  		s.mockValidatingWebhookConfigurationV1Beta1.EXPECT().Update(gomock.Any(), cfg1, gomock.Any()).Return(cfg1, nil),
  1107  	)
  1108  }
  1109  
  1110  func (s *K8sBrokerSuite) TestEnsureValidatingWebhookConfigurationsCreateV1(c *gc.C) {
  1111  	ctrl := s.setupController(c)
  1112  	defer ctrl.Finish()
  1113  
  1114  	s.mockDiscovery.EXPECT().ServerVersion().AnyTimes().Return(&k8sversion.Info{
  1115  		Major: "1", Minor: "16",
  1116  	}, nil)
  1117  
  1118  	webhook1Rule1 := admissionregistrationv1.Rule{
  1119  		APIGroups:   []string{""},
  1120  		APIVersions: []string{"v1"},
  1121  		Resources:   []string{"pods"},
  1122  	}
  1123  	webhookRuleWithOperations1 := admissionregistrationv1.RuleWithOperations{
  1124  		Operations: []admissionregistrationv1.OperationType{
  1125  			admissionregistrationv1.Create,
  1126  			admissionregistrationv1.Update,
  1127  		},
  1128  	}
  1129  	webhookRuleWithOperations1.Rule = webhook1Rule1
  1130  	CABundle, err := base64.StdEncoding.DecodeString("YXBwbGVz")
  1131  	c.Assert(err, jc.ErrorIsNil)
  1132  	webhook1FailurePolicy := admissionregistrationv1.Ignore
  1133  	webhook1 := admissionregistrationv1.ValidatingWebhook{
  1134  		Name:          "example.validatingwebhookconfiguration.com",
  1135  		FailurePolicy: &webhook1FailurePolicy,
  1136  		ClientConfig: admissionregistrationv1.WebhookClientConfig{
  1137  			Service: &admissionregistrationv1.ServiceReference{
  1138  				Name:      "apple-service",
  1139  				Namespace: "apples",
  1140  				Path:      pointer.StringPtr("/apple"),
  1141  			},
  1142  			CABundle: CABundle,
  1143  		},
  1144  		NamespaceSelector: &metav1.LabelSelector{
  1145  			MatchExpressions: []metav1.LabelSelectorRequirement{
  1146  				{Key: "production", Operator: metav1.LabelSelectorOpDoesNotExist},
  1147  			},
  1148  		},
  1149  		Rules: []admissionregistrationv1.RuleWithOperations{webhookRuleWithOperations1},
  1150  	}
  1151  
  1152  	cfgs := []k8sspecs.K8sValidatingWebhook{
  1153  		{
  1154  			Meta: k8sspecs.Meta{Name: "example-validatingwebhookconfiguration"},
  1155  			Webhooks: []k8sspecs.K8sValidatingWebhookSpec{
  1156  				{
  1157  					Version: k8sspecs.K8sWebhookV1,
  1158  					SpecV1:  webhook1,
  1159  				},
  1160  			},
  1161  		},
  1162  	}
  1163  
  1164  	cfg1 := &admissionregistrationv1.ValidatingWebhookConfiguration{
  1165  		ObjectMeta: metav1.ObjectMeta{
  1166  			Name:        "test-example-validatingwebhookconfiguration",
  1167  			Namespace:   "test",
  1168  			Labels:      map[string]string{"app.kubernetes.io/managed-by": "juju", "app.kubernetes.io/name": "app-name", "model.juju.is/name": "test"},
  1169  			Annotations: map[string]string{"controller.juju.is/id": testing.ControllerTag.Id()},
  1170  		},
  1171  		Webhooks: []admissionregistrationv1.ValidatingWebhook{webhook1},
  1172  	}
  1173  
  1174  	s.assertValidatingWebhookConfigurations(
  1175  		c, cfgs,
  1176  		s.mockValidatingWebhookConfigurationV1.EXPECT().Create(gomock.Any(), cfg1, gomock.Any()).Return(cfg1, nil),
  1177  	)
  1178  }
  1179  
  1180  func (s *K8sBrokerSuite) TestEnsureValidatingWebhookConfigurationsCreateKeepNameV1(c *gc.C) {
  1181  	ctrl := s.setupController(c)
  1182  	defer ctrl.Finish()
  1183  
  1184  	s.mockDiscovery.EXPECT().ServerVersion().AnyTimes().Return(&k8sversion.Info{
  1185  		Major: "1", Minor: "16",
  1186  	}, nil)
  1187  
  1188  	webhook1Rule1 := admissionregistrationv1.Rule{
  1189  		APIGroups:   []string{""},
  1190  		APIVersions: []string{"v1"},
  1191  		Resources:   []string{"pods"},
  1192  	}
  1193  	webhookRuleWithOperations1 := admissionregistrationv1.RuleWithOperations{
  1194  		Operations: []admissionregistrationv1.OperationType{
  1195  			admissionregistrationv1.Create,
  1196  			admissionregistrationv1.Update,
  1197  		},
  1198  	}
  1199  	webhookRuleWithOperations1.Rule = webhook1Rule1
  1200  	CABundle, err := base64.StdEncoding.DecodeString("YXBwbGVz")
  1201  	c.Assert(err, jc.ErrorIsNil)
  1202  	webhook1FailurePolicy := admissionregistrationv1.Ignore
  1203  	webhook1 := admissionregistrationv1.ValidatingWebhook{
  1204  		Name:          "example.validatingwebhookconfiguration.com",
  1205  		FailurePolicy: &webhook1FailurePolicy,
  1206  		ClientConfig: admissionregistrationv1.WebhookClientConfig{
  1207  			Service: &admissionregistrationv1.ServiceReference{
  1208  				Name:      "apple-service",
  1209  				Namespace: "apples",
  1210  				Path:      pointer.StringPtr("/apple"),
  1211  			},
  1212  			CABundle: CABundle,
  1213  		},
  1214  		NamespaceSelector: &metav1.LabelSelector{
  1215  			MatchExpressions: []metav1.LabelSelectorRequirement{
  1216  				{Key: "production", Operator: metav1.LabelSelectorOpDoesNotExist},
  1217  			},
  1218  		},
  1219  		Rules: []admissionregistrationv1.RuleWithOperations{webhookRuleWithOperations1},
  1220  	}
  1221  
  1222  	cfgs := []k8sspecs.K8sValidatingWebhook{
  1223  		{
  1224  			Meta: k8sspecs.Meta{
  1225  				Name:        "example-validatingwebhookconfiguration",
  1226  				Annotations: map[string]string{"model.juju.is/disable-prefix": "true"},
  1227  			},
  1228  			Webhooks: []k8sspecs.K8sValidatingWebhookSpec{
  1229  				{
  1230  					Version: k8sspecs.K8sWebhookV1,
  1231  					SpecV1:  webhook1,
  1232  				},
  1233  			},
  1234  		},
  1235  	}
  1236  
  1237  	cfg1 := &admissionregistrationv1.ValidatingWebhookConfiguration{
  1238  		ObjectMeta: metav1.ObjectMeta{
  1239  			Name:        "example-validatingwebhookconfiguration", // This name kept no change.
  1240  			Namespace:   "test",
  1241  			Labels:      map[string]string{"app.kubernetes.io/managed-by": "juju", "app.kubernetes.io/name": "app-name", "model.juju.is/name": "test"},
  1242  			Annotations: map[string]string{"controller.juju.is/id": testing.ControllerTag.Id(), "model.juju.is/disable-prefix": "true"},
  1243  		},
  1244  		Webhooks: []admissionregistrationv1.ValidatingWebhook{webhook1},
  1245  	}
  1246  
  1247  	s.assertValidatingWebhookConfigurations(
  1248  		c, cfgs,
  1249  		s.mockValidatingWebhookConfigurationV1.EXPECT().Create(gomock.Any(), cfg1, gomock.Any()).Return(cfg1, nil),
  1250  	)
  1251  }
  1252  
  1253  func (s *K8sBrokerSuite) TestEnsureValidatingWebhookConfigurationsUpdateV1(c *gc.C) {
  1254  	ctrl := s.setupController(c)
  1255  	defer ctrl.Finish()
  1256  
  1257  	s.mockDiscovery.EXPECT().ServerVersion().AnyTimes().Return(&k8sversion.Info{
  1258  		Major: "1", Minor: "16",
  1259  	}, nil)
  1260  
  1261  	webhook1Rule1 := admissionregistrationv1.Rule{
  1262  		APIGroups:   []string{""},
  1263  		APIVersions: []string{"v1"},
  1264  		Resources:   []string{"pods"},
  1265  	}
  1266  	webhookRuleWithOperations1 := admissionregistrationv1.RuleWithOperations{
  1267  		Operations: []admissionregistrationv1.OperationType{
  1268  			admissionregistrationv1.Create,
  1269  			admissionregistrationv1.Update,
  1270  		},
  1271  	}
  1272  	webhookRuleWithOperations1.Rule = webhook1Rule1
  1273  	CABundle, err := base64.StdEncoding.DecodeString("YXBwbGVz")
  1274  	c.Assert(err, jc.ErrorIsNil)
  1275  	webhook1FailurePolicy := admissionregistrationv1.Ignore
  1276  	webhook1 := admissionregistrationv1.ValidatingWebhook{
  1277  		Name:          "example.validatingwebhookconfiguration.com",
  1278  		FailurePolicy: &webhook1FailurePolicy,
  1279  		ClientConfig: admissionregistrationv1.WebhookClientConfig{
  1280  			Service: &admissionregistrationv1.ServiceReference{
  1281  				Name:      "apple-service",
  1282  				Namespace: "apples",
  1283  				Path:      pointer.StringPtr("/apple"),
  1284  			},
  1285  			CABundle: CABundle,
  1286  		},
  1287  		NamespaceSelector: &metav1.LabelSelector{
  1288  			MatchExpressions: []metav1.LabelSelectorRequirement{
  1289  				{Key: "production", Operator: metav1.LabelSelectorOpDoesNotExist},
  1290  			},
  1291  		},
  1292  		Rules: []admissionregistrationv1.RuleWithOperations{webhookRuleWithOperations1},
  1293  	}
  1294  
  1295  	cfgs := []k8sspecs.K8sValidatingWebhook{
  1296  		{
  1297  			Meta: k8sspecs.Meta{Name: "example-validatingwebhookconfiguration"},
  1298  			Webhooks: []k8sspecs.K8sValidatingWebhookSpec{
  1299  				{
  1300  					Version: k8sspecs.K8sWebhookV1,
  1301  					SpecV1:  webhook1,
  1302  				},
  1303  			},
  1304  		},
  1305  	}
  1306  
  1307  	cfg1 := &admissionregistrationv1.ValidatingWebhookConfiguration{
  1308  		ObjectMeta: metav1.ObjectMeta{
  1309  			Name:        "test-example-validatingwebhookconfiguration",
  1310  			Namespace:   "test",
  1311  			Labels:      map[string]string{"app.kubernetes.io/managed-by": "juju", "app.kubernetes.io/name": "app-name", "model.juju.is/name": "test"},
  1312  			Annotations: map[string]string{"controller.juju.is/id": testing.ControllerTag.Id()},
  1313  		},
  1314  		Webhooks: []admissionregistrationv1.ValidatingWebhook{webhook1},
  1315  	}
  1316  
  1317  	s.assertValidatingWebhookConfigurations(
  1318  		c, cfgs,
  1319  		s.mockValidatingWebhookConfigurationV1.EXPECT().Create(gomock.Any(), cfg1, metav1.CreateOptions{}).Return(cfg1, s.k8sAlreadyExistsError()),
  1320  		s.mockValidatingWebhookConfigurationV1.EXPECT().
  1321  			List(gomock.Any(), metav1.ListOptions{LabelSelector: "app.kubernetes.io/managed-by=juju,app.kubernetes.io/name=app-name,model.juju.is/name=test"}).
  1322  			Return(&admissionregistrationv1.ValidatingWebhookConfigurationList{Items: []admissionregistrationv1.ValidatingWebhookConfiguration{*cfg1}}, nil),
  1323  		s.mockValidatingWebhookConfigurationV1.EXPECT().
  1324  			Get(gomock.Any(), "test-example-validatingwebhookconfiguration", metav1.GetOptions{}).
  1325  			Return(cfg1, nil),
  1326  		s.mockValidatingWebhookConfigurationV1.EXPECT().Update(gomock.Any(), cfg1, metav1.UpdateOptions{}).Return(cfg1, nil),
  1327  	)
  1328  }