github.com/juju/juju@v0.0.0-20240430160146-1752b71fcf00/caas/kubernetes/provider/operator_upgrade_test.go (about)

     1  // Copyright 2020 Canonical Ltd.
     2  // Licensed under the AGPLv3, see LICENCE file for details.
     3  
     4  package provider
     5  
     6  import (
     7  	"context"
     8  	"fmt"
     9  	"time"
    10  
    11  	"github.com/juju/clock"
    12  	"github.com/juju/clock/testclock"
    13  	jc "github.com/juju/testing/checkers"
    14  	"github.com/juju/version/v2"
    15  	gc "gopkg.in/check.v1"
    16  	apps "k8s.io/api/apps/v1"
    17  	core "k8s.io/api/core/v1"
    18  	v1 "k8s.io/api/core/v1"
    19  	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
    20  	"k8s.io/client-go/kubernetes"
    21  	"k8s.io/client-go/kubernetes/fake"
    22  
    23  	"github.com/juju/juju/caas"
    24  	"github.com/juju/juju/caas/kubernetes/provider/constants"
    25  	"github.com/juju/juju/cloudconfig/podcfg"
    26  )
    27  
    28  type DummyUpgradeCAASOperator struct {
    29  	client *fake.Clientset
    30  	clock  clock.Clock
    31  }
    32  
    33  type OperatorUpgraderSuite struct {
    34  	broker *DummyUpgradeCAASOperator
    35  }
    36  
    37  var _ = gc.Suite(&OperatorUpgraderSuite{})
    38  
    39  func (d *DummyUpgradeCAASOperator) Client() kubernetes.Interface {
    40  	return d.client
    41  }
    42  
    43  func (d *DummyUpgradeCAASOperator) Clock() clock.Clock {
    44  	return d.clock
    45  }
    46  
    47  func (d *DummyUpgradeCAASOperator) DeploymentName(n string, _ bool) string {
    48  	return n
    49  }
    50  
    51  func (d *DummyUpgradeCAASOperator) IsLegacyLabels() bool {
    52  	return false
    53  }
    54  
    55  func (d *DummyUpgradeCAASOperator) Namespace() string {
    56  	return "test"
    57  }
    58  
    59  func (d *DummyUpgradeCAASOperator) Operator(appName string) (*caas.Operator, error) {
    60  	return operator(d.client, d.Namespace(), d.OperatorName(appName), appName, false, d.Clock().Now())
    61  }
    62  
    63  func (d *DummyUpgradeCAASOperator) OperatorName(n string) string {
    64  	return n + "-operator"
    65  }
    66  
    67  func (o *OperatorUpgraderSuite) SetUpTest(c *gc.C) {
    68  	o.broker = &DummyUpgradeCAASOperator{
    69  		client: fake.NewSimpleClientset(),
    70  	}
    71  }
    72  
    73  func (o *OperatorUpgraderSuite) TestOperatorUpgradeToBaseCharm(c *gc.C) {
    74  	var (
    75  		appName        = "testinitss"
    76  		oldImagePath   = fmt.Sprintf("%s/%s:2.9.33", podcfg.JujudOCINamespace, podcfg.JujudOCIName)
    77  		newImagePath   = fmt.Sprintf("%s/%s:3.0.0", podcfg.JujudOCINamespace, podcfg.JujudOCIName)
    78  		focalCharmBase = fmt.Sprintf("%s/%s:ubuntu-20.04", podcfg.JujudOCINamespace, podcfg.CharmBaseName)
    79  		newVersion     = version.Number{Major: 3}
    80  	)
    81  
    82  	_, err := o.broker.Client().AppsV1().StatefulSets(o.broker.Namespace()).Create(context.TODO(),
    83  		&apps.StatefulSet{
    84  			ObjectMeta: meta.ObjectMeta{
    85  				Name: o.broker.OperatorName(appName),
    86  			},
    87  			Spec: apps.StatefulSetSpec{
    88  				Selector: &meta.LabelSelector{
    89  					MatchLabels: map[string]string{
    90  						"operator-label": "true",
    91  					},
    92  				},
    93  				Template: core.PodTemplateSpec{
    94  					Spec: core.PodSpec{
    95  						Containers: []core.Container{
    96  							{
    97  								Name:  operatorContainerName,
    98  								Image: oldImagePath,
    99  							},
   100  						},
   101  					},
   102  				},
   103  			},
   104  		}, meta.CreateOptions{})
   105  	c.Assert(err, jc.ErrorIsNil)
   106  
   107  	_, err = o.broker.Client().CoreV1().Pods(o.broker.Namespace()).Create(context.TODO(), &core.Pod{
   108  		ObjectMeta: meta.ObjectMeta{
   109  			Name: o.broker.OperatorName(appName) + "-0",
   110  			Labels: map[string]string{
   111  				constants.LabelJujuOperatorName:   "testinitss",
   112  				constants.LabelJujuOperatorTarget: "application",
   113  			},
   114  		},
   115  		Spec: core.PodSpec{
   116  			Containers: []core.Container{
   117  				{
   118  					Name:  operatorContainerName,
   119  					Image: oldImagePath,
   120  				},
   121  			},
   122  		},
   123  		Status: core.PodStatus{
   124  			Phase: core.PodRunning,
   125  		},
   126  	}, meta.CreateOptions{})
   127  	c.Assert(err, jc.ErrorIsNil)
   128  
   129  	_, err = o.broker.Client().AppsV1().StatefulSets(o.broker.Namespace()).Create(context.TODO(),
   130  		&apps.StatefulSet{
   131  			ObjectMeta: meta.ObjectMeta{
   132  				Name: o.broker.DeploymentName(appName, true),
   133  			},
   134  			Spec: apps.StatefulSetSpec{
   135  				Selector: &meta.LabelSelector{
   136  					MatchLabels: map[string]string{
   137  						"match-label": "true",
   138  					},
   139  				},
   140  				Template: core.PodTemplateSpec{
   141  					Spec: core.PodSpec{
   142  						InitContainers: []core.Container{
   143  							{
   144  								Name:  caas.InitContainerName,
   145  								Image: oldImagePath,
   146  							},
   147  						},
   148  					},
   149  				},
   150  			},
   151  		}, meta.CreateOptions{})
   152  	c.Assert(err, jc.ErrorIsNil)
   153  
   154  	_, err = o.broker.Client().CoreV1().Pods(o.broker.Namespace()).Create(context.TODO(), &core.Pod{
   155  		ObjectMeta: meta.ObjectMeta{
   156  			Name: "pod1",
   157  			Labels: map[string]string{
   158  				"match-label": "true",
   159  			},
   160  		},
   161  		Spec: core.PodSpec{
   162  			InitContainers: []core.Container{
   163  				{
   164  					Name:  caas.InitContainerName,
   165  					Image: newImagePath,
   166  				},
   167  			},
   168  		},
   169  		Status: core.PodStatus{
   170  			Phase: core.PodRunning,
   171  		},
   172  	}, meta.CreateOptions{})
   173  	c.Assert(err, jc.ErrorIsNil)
   174  
   175  	o.broker.clock = testclock.NewDilatedWallClock(10 * time.Millisecond)
   176  	err = operatorUpgrade(appName, newVersion, o.broker)
   177  	c.Assert(err, jc.ErrorIsNil)
   178  
   179  	appSS, err := o.broker.Client().AppsV1().StatefulSets(o.broker.Namespace()).
   180  		Get(context.TODO(), o.broker.DeploymentName(appName, true), meta.GetOptions{})
   181  	c.Assert(err, jc.ErrorIsNil)
   182  	c.Assert(appSS.Spec.Template.Spec.InitContainers[0].Image, gc.Equals, newImagePath)
   183  
   184  	operatorSS, err := o.broker.Client().AppsV1().StatefulSets(o.broker.Namespace()).
   185  		Get(context.TODO(), o.broker.OperatorName(appName), meta.GetOptions{})
   186  	c.Assert(err, jc.ErrorIsNil)
   187  	c.Assert(operatorSS.Spec.Template.Spec.InitContainers[0].Image, gc.Equals, newImagePath)
   188  	c.Assert(operatorSS.Spec.Template.Spec.InitContainers[0].VolumeMounts, gc.DeepEquals, []v1.VolumeMount{{
   189  		Name:      "juju-bins",
   190  		MountPath: "/opt/juju",
   191  	}})
   192  	c.Assert(operatorSS.Spec.Template.Spec.Containers[0].Image, gc.Equals, focalCharmBase)
   193  	c.Assert(operatorSS.Spec.Template.Spec.Containers[0].Args, gc.DeepEquals, []string{"-c", "export JUJU_DATA_DIR=/var/lib/juju\nexport JUJU_TOOLS_DIR=$JUJU_DATA_DIR/tools\n\nmkdir -p $JUJU_TOOLS_DIR\ncp /opt/juju/jujud $JUJU_TOOLS_DIR/jujud\n\nexec $JUJU_TOOLS_DIR/jujud caasoperator --application-name=testinitss --debug\n"})
   194  	c.Assert(operatorSS.Spec.Template.Spec.Containers[0].VolumeMounts, gc.DeepEquals, []v1.VolumeMount{{
   195  		Name:      "juju-bins",
   196  		MountPath: "/opt/juju",
   197  	}})
   198  	c.Assert(operatorSS.Spec.Template.Spec.Volumes, gc.DeepEquals, []v1.Volume{{
   199  		Name: "juju-bins",
   200  		VolumeSource: v1.VolumeSource{
   201  			EmptyDir: &v1.EmptyDirVolumeSource{},
   202  		}},
   203  	})
   204  }
   205  
   206  func (o *OperatorUpgraderSuite) TestStatefulSetInitUpgrade(c *gc.C) {
   207  	var (
   208  		appName      = "testinitss"
   209  		oldImagePath = fmt.Sprintf("%s/%s:9.9.8", podcfg.JujudOCINamespace, podcfg.JujudOCIName)
   210  		newImagePath = fmt.Sprintf("%s/%s:9.9.9", podcfg.JujudOCINamespace, podcfg.JujudOCIName)
   211  	)
   212  
   213  	_, err := o.broker.Client().AppsV1().StatefulSets(o.broker.Namespace()).Create(context.TODO(),
   214  		&apps.StatefulSet{
   215  			ObjectMeta: meta.ObjectMeta{
   216  				Name: o.broker.DeploymentName(appName, true),
   217  			},
   218  			Spec: apps.StatefulSetSpec{
   219  				Selector: &meta.LabelSelector{
   220  					MatchLabels: map[string]string{
   221  						"match-label": "true",
   222  					},
   223  				},
   224  				Template: core.PodTemplateSpec{
   225  					Spec: core.PodSpec{
   226  						InitContainers: []core.Container{
   227  							{
   228  								Name:  caas.InitContainerName,
   229  								Image: oldImagePath,
   230  							},
   231  						},
   232  					},
   233  				},
   234  			},
   235  		}, meta.CreateOptions{})
   236  	c.Assert(err, jc.ErrorIsNil)
   237  
   238  	podChecker, err := workloadInitUpgrade(appName, newImagePath, o.broker)
   239  	c.Assert(err, jc.ErrorIsNil)
   240  
   241  	ss, err := o.broker.Client().AppsV1().StatefulSets(o.broker.Namespace()).
   242  		Get(context.TODO(), o.broker.DeploymentName(appName, true), meta.GetOptions{})
   243  	c.Assert(err, jc.ErrorIsNil)
   244  	c.Assert(ss.Spec.Template.Spec.InitContainers[0].Image, gc.Equals, newImagePath)
   245  
   246  	_, err = o.broker.Client().CoreV1().Pods(o.broker.Namespace()).Create(context.TODO(), &core.Pod{
   247  		ObjectMeta: meta.ObjectMeta{
   248  			Name: "pod1",
   249  			Labels: map[string]string{
   250  				"match-label": "true",
   251  			},
   252  		},
   253  		Spec: core.PodSpec{
   254  			InitContainers: []core.Container{
   255  				{
   256  					Name:  caas.InitContainerName,
   257  					Image: newImagePath,
   258  				},
   259  			},
   260  		},
   261  		Status: core.PodStatus{
   262  			Phase: core.PodPending,
   263  		},
   264  	}, meta.CreateOptions{})
   265  	c.Assert(err, jc.ErrorIsNil)
   266  
   267  	ready, err := podChecker()
   268  	c.Assert(err, jc.ErrorIsNil)
   269  	c.Assert(ready, jc.IsFalse)
   270  
   271  	_, err = o.broker.Client().CoreV1().Pods(o.broker.Namespace()).Update(context.TODO(), &core.Pod{
   272  		ObjectMeta: meta.ObjectMeta{
   273  			Name: "pod1",
   274  			Labels: map[string]string{
   275  				"match-label": "true",
   276  			},
   277  		},
   278  		Spec: core.PodSpec{
   279  			InitContainers: []core.Container{
   280  				{
   281  					Name:  caas.InitContainerName,
   282  					Image: newImagePath,
   283  				},
   284  			},
   285  		},
   286  		Status: core.PodStatus{
   287  			Phase: core.PodRunning,
   288  		},
   289  	}, meta.UpdateOptions{})
   290  	c.Assert(err, jc.ErrorIsNil)
   291  
   292  	ready, err = podChecker()
   293  	c.Assert(err, jc.ErrorIsNil)
   294  	c.Assert(ready, jc.IsTrue)
   295  }
   296  
   297  func (o *OperatorUpgraderSuite) TestStatefulSetInitUpgradePodNotReadyYet(c *gc.C) {
   298  	var (
   299  		appName      = "testinitss"
   300  		oldImagePath = fmt.Sprintf("%s/%s:9.9.8", podcfg.JujudOCINamespace, podcfg.JujudOCIName)
   301  		newImagePath = fmt.Sprintf("%s/%s:9.9.9", podcfg.JujudOCINamespace, podcfg.JujudOCIName)
   302  	)
   303  
   304  	_, err := o.broker.Client().AppsV1().StatefulSets(o.broker.Namespace()).Create(
   305  		context.TODO(),
   306  		&apps.StatefulSet{
   307  			ObjectMeta: meta.ObjectMeta{
   308  				Name: o.broker.DeploymentName(appName, true),
   309  			},
   310  			Spec: apps.StatefulSetSpec{
   311  				Selector: &meta.LabelSelector{
   312  					MatchLabels: map[string]string{
   313  						"match-label": "true",
   314  					},
   315  				},
   316  				Template: core.PodTemplateSpec{
   317  					Spec: core.PodSpec{
   318  						InitContainers: []core.Container{
   319  							{
   320  								Name:  caas.InitContainerName,
   321  								Image: oldImagePath,
   322  							},
   323  						},
   324  					},
   325  				},
   326  			},
   327  		}, meta.CreateOptions{})
   328  	c.Assert(err, jc.ErrorIsNil)
   329  
   330  	podChecker, err := workloadInitUpgrade(appName, newImagePath, o.broker)
   331  	c.Assert(err, jc.ErrorIsNil)
   332  
   333  	ss, err := o.broker.Client().AppsV1().StatefulSets(o.broker.Namespace()).
   334  		Get(context.TODO(), o.broker.DeploymentName(appName, true), meta.GetOptions{})
   335  	c.Assert(err, jc.ErrorIsNil)
   336  	c.Assert(ss.Spec.Template.Spec.InitContainers[0].Image, gc.Equals, newImagePath)
   337  
   338  	ready, err := podChecker()
   339  	c.Assert(err, jc.ErrorIsNil)
   340  	c.Assert(ready, jc.IsFalse)
   341  
   342  	_, err = o.broker.Client().CoreV1().Pods(o.broker.Namespace()).Create(
   343  		context.TODO(),
   344  		&core.Pod{
   345  			ObjectMeta: meta.ObjectMeta{
   346  				Name: "pod1",
   347  				Labels: map[string]string{
   348  					"match-label": "true",
   349  				},
   350  			},
   351  			Spec: core.PodSpec{
   352  				InitContainers: []core.Container{
   353  					{
   354  						Name:  caas.InitContainerName,
   355  						Image: newImagePath,
   356  					},
   357  				},
   358  			},
   359  			Status: core.PodStatus{
   360  				Phase: core.PodRunning,
   361  			},
   362  		}, meta.CreateOptions{})
   363  	c.Assert(err, jc.ErrorIsNil)
   364  
   365  	ready, err = podChecker()
   366  	c.Assert(err, jc.ErrorIsNil)
   367  	c.Assert(ready, jc.IsTrue)
   368  }
   369  
   370  func (o *OperatorUpgraderSuite) TestDaemonSetInitUpgrade(c *gc.C) {
   371  	var (
   372  		appName      = "testinitds"
   373  		oldImagePath = fmt.Sprintf("%s/%s:9.9.8", podcfg.JujudOCINamespace, podcfg.JujudOCIName)
   374  		newImagePath = fmt.Sprintf("%s/%s:9.9.9", podcfg.JujudOCINamespace, podcfg.JujudOCIName)
   375  	)
   376  
   377  	_, err := o.broker.Client().AppsV1().DaemonSets(o.broker.Namespace()).Create(context.TODO(),
   378  		&apps.DaemonSet{
   379  			ObjectMeta: meta.ObjectMeta{
   380  				Name: o.broker.DeploymentName(appName, true),
   381  			},
   382  			Spec: apps.DaemonSetSpec{
   383  				Selector: &meta.LabelSelector{
   384  					MatchLabels: map[string]string{
   385  						"match-label": "true",
   386  					},
   387  				},
   388  				Template: core.PodTemplateSpec{
   389  					Spec: core.PodSpec{
   390  						InitContainers: []core.Container{
   391  							{
   392  								Name:  caas.InitContainerName,
   393  								Image: oldImagePath,
   394  							},
   395  						},
   396  					},
   397  				},
   398  			},
   399  		}, meta.CreateOptions{})
   400  	c.Assert(err, jc.ErrorIsNil)
   401  
   402  	podChecker, err := workloadInitUpgrade(appName, newImagePath, o.broker)
   403  	c.Assert(err, jc.ErrorIsNil)
   404  
   405  	ds, err := o.broker.Client().AppsV1().DaemonSets(o.broker.Namespace()).
   406  		Get(context.TODO(), o.broker.DeploymentName(appName, true), meta.GetOptions{})
   407  	c.Assert(err, jc.ErrorIsNil)
   408  	c.Assert(ds.Spec.Template.Spec.InitContainers[0].Image, gc.Equals, newImagePath)
   409  
   410  	_, err = o.broker.Client().CoreV1().Pods(o.broker.Namespace()).Create(context.TODO(), &core.Pod{
   411  		ObjectMeta: meta.ObjectMeta{
   412  			Name: "pod1",
   413  			Labels: map[string]string{
   414  				"match-label": "true",
   415  			},
   416  		},
   417  		Spec: core.PodSpec{
   418  			InitContainers: []core.Container{
   419  				{
   420  					Name:  caas.InitContainerName,
   421  					Image: newImagePath,
   422  				},
   423  			},
   424  		},
   425  		Status: core.PodStatus{
   426  			Phase: core.PodPending,
   427  		},
   428  	}, meta.CreateOptions{})
   429  	c.Assert(err, jc.ErrorIsNil)
   430  
   431  	ready, err := podChecker()
   432  	c.Assert(err, jc.ErrorIsNil)
   433  	c.Assert(ready, jc.IsFalse)
   434  
   435  	_, err = o.broker.Client().CoreV1().Pods(o.broker.Namespace()).Update(context.TODO(), &core.Pod{
   436  		ObjectMeta: meta.ObjectMeta{
   437  			Name: "pod1",
   438  			Labels: map[string]string{
   439  				"match-label": "true",
   440  			},
   441  		},
   442  		Spec: core.PodSpec{
   443  			InitContainers: []core.Container{
   444  				{
   445  					Name:  caas.InitContainerName,
   446  					Image: newImagePath,
   447  				},
   448  			},
   449  		},
   450  		Status: core.PodStatus{
   451  			Phase: core.PodRunning,
   452  		},
   453  	}, meta.UpdateOptions{})
   454  	c.Assert(err, jc.ErrorIsNil)
   455  
   456  	ready, err = podChecker()
   457  	c.Assert(err, jc.ErrorIsNil)
   458  	c.Assert(ready, jc.IsTrue)
   459  }
   460  
   461  func (o *OperatorUpgraderSuite) TestDeploymentInitUpgrade(c *gc.C) {
   462  	var (
   463  		appName      = "testinitds"
   464  		oldImagePath = fmt.Sprintf("%s/%s:9.9.8", podcfg.JujudOCINamespace, podcfg.JujudOCIName)
   465  		newImagePath = fmt.Sprintf("%s/%s:9.9.9", podcfg.JujudOCINamespace, podcfg.JujudOCIName)
   466  	)
   467  
   468  	_, err := o.broker.Client().AppsV1().Deployments(o.broker.Namespace()).Create(context.TODO(),
   469  		&apps.Deployment{
   470  			ObjectMeta: meta.ObjectMeta{
   471  				Name: o.broker.DeploymentName(appName, true),
   472  			},
   473  			Spec: apps.DeploymentSpec{
   474  				Selector: &meta.LabelSelector{
   475  					MatchLabels: map[string]string{
   476  						"match-label": "true",
   477  					},
   478  				},
   479  				Template: core.PodTemplateSpec{
   480  					Spec: core.PodSpec{
   481  						InitContainers: []core.Container{
   482  							{
   483  								Name:  caas.InitContainerName,
   484  								Image: oldImagePath,
   485  							},
   486  						},
   487  					},
   488  				},
   489  			},
   490  		}, meta.CreateOptions{})
   491  	c.Assert(err, jc.ErrorIsNil)
   492  
   493  	podChecker, err := workloadInitUpgrade(appName, newImagePath, o.broker)
   494  	c.Assert(err, jc.ErrorIsNil)
   495  
   496  	de, err := o.broker.Client().AppsV1().Deployments(o.broker.Namespace()).
   497  		Get(context.TODO(), o.broker.DeploymentName(appName, true), meta.GetOptions{})
   498  	c.Assert(err, jc.ErrorIsNil)
   499  	c.Assert(de.Spec.Template.Spec.InitContainers[0].Image, gc.Equals, newImagePath)
   500  
   501  	_, err = o.broker.Client().CoreV1().Pods(o.broker.Namespace()).Create(context.TODO(), &core.Pod{
   502  		ObjectMeta: meta.ObjectMeta{
   503  			Name: "pod1",
   504  			Labels: map[string]string{
   505  				"match-label": "true",
   506  			},
   507  		},
   508  		Spec: core.PodSpec{
   509  			InitContainers: []core.Container{
   510  				{
   511  					Name:  caas.InitContainerName,
   512  					Image: newImagePath,
   513  				},
   514  			},
   515  		},
   516  		Status: core.PodStatus{
   517  			Phase: core.PodPending,
   518  		},
   519  	}, meta.CreateOptions{})
   520  	c.Assert(err, jc.ErrorIsNil)
   521  
   522  	ready, err := podChecker()
   523  	c.Assert(err, jc.ErrorIsNil)
   524  	c.Assert(ready, jc.IsFalse)
   525  
   526  	_, err = o.broker.Client().CoreV1().Pods(o.broker.Namespace()).Update(context.TODO(), &core.Pod{
   527  		ObjectMeta: meta.ObjectMeta{
   528  			Name: "pod1",
   529  			Labels: map[string]string{
   530  				"match-label": "true",
   531  			},
   532  		},
   533  		Spec: core.PodSpec{
   534  			InitContainers: []core.Container{
   535  				{
   536  					Name:  caas.InitContainerName,
   537  					Image: newImagePath,
   538  				},
   539  			},
   540  		},
   541  		Status: core.PodStatus{
   542  			Phase: core.PodRunning,
   543  		},
   544  	}, meta.UpdateOptions{})
   545  	c.Assert(err, jc.ErrorIsNil)
   546  
   547  	ready, err = podChecker()
   548  	c.Assert(err, jc.ErrorIsNil)
   549  	c.Assert(ready, jc.IsTrue)
   550  }