github.com/GoogleContainerTools/skaffold@v1.39.18/pkg/skaffold/kubernetes/status/status_check_test.go (about)

     1  /*
     2  Copyright 2019 The Skaffold Authors
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package status
    18  
    19  import (
    20  	"bytes"
    21  	"context"
    22  	"errors"
    23  	"testing"
    24  	"time"
    25  
    26  	"github.com/google/go-cmp/cmp"
    27  	"github.com/google/go-cmp/cmp/cmpopts"
    28  	"google.golang.org/protobuf/testing/protocmp"
    29  	appsv1 "k8s.io/api/apps/v1"
    30  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    31  	"k8s.io/apimachinery/pkg/runtime"
    32  	fakekubeclientset "k8s.io/client-go/kubernetes/fake"
    33  	utilpointer "k8s.io/utils/pointer"
    34  
    35  	"github.com/GoogleContainerTools/skaffold/pkg/diag"
    36  	"github.com/GoogleContainerTools/skaffold/pkg/diag/validator"
    37  	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/deploy/label"
    38  	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes/status/resource"
    39  	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/runner/runcontext"
    40  	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
    41  	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
    42  	"github.com/GoogleContainerTools/skaffold/proto/v1"
    43  	"github.com/GoogleContainerTools/skaffold/testutil"
    44  	testEvent "github.com/GoogleContainerTools/skaffold/testutil/event"
    45  )
    46  
    47  var TestKubeContext = "kubecontext"
    48  
    49  func TestGetDeployments(t *testing.T) {
    50  	labeller := label.NewLabeller(true, nil, "run-id")
    51  	tests := []struct {
    52  		description string
    53  		deps        []*appsv1.Deployment
    54  		expected    []*resource.Resource
    55  		shouldErr   bool
    56  	}{
    57  		{
    58  			description: "multiple deployments in same namespace",
    59  			deps: []*appsv1.Deployment{
    60  				{
    61  					ObjectMeta: metav1.ObjectMeta{
    62  						Name:      "dep1",
    63  						Namespace: "test",
    64  						Labels: map[string]string{
    65  							label.RunIDLabel: labeller.GetRunID(),
    66  							"random":         "foo",
    67  						},
    68  					},
    69  					Spec: appsv1.DeploymentSpec{ProgressDeadlineSeconds: utilpointer.Int32Ptr(10)},
    70  				},
    71  				{
    72  					ObjectMeta: metav1.ObjectMeta{
    73  						Name:      "dep2",
    74  						Namespace: "test",
    75  						Labels: map[string]string{
    76  							label.RunIDLabel: labeller.GetRunID(),
    77  						},
    78  					},
    79  					Spec: appsv1.DeploymentSpec{ProgressDeadlineSeconds: utilpointer.Int32Ptr(20)},
    80  				},
    81  			},
    82  			expected: []*resource.Resource{
    83  				resource.NewResource("dep1", resource.ResourceTypes.Deployment, "test", 10*time.Second),
    84  				resource.NewResource("dep2", resource.ResourceTypes.Deployment, "test", 20*time.Second),
    85  			},
    86  		},
    87  		{
    88  			description: "command flag deadline is less than deployment spec.",
    89  			deps: []*appsv1.Deployment{
    90  				{
    91  					ObjectMeta: metav1.ObjectMeta{
    92  						Name:      "dep1",
    93  						Namespace: "test",
    94  						Labels: map[string]string{
    95  							label.RunIDLabel: labeller.GetRunID(),
    96  							"random":         "foo",
    97  						},
    98  					},
    99  					Spec: appsv1.DeploymentSpec{ProgressDeadlineSeconds: utilpointer.Int32Ptr(300)},
   100  				},
   101  			},
   102  			expected: []*resource.Resource{
   103  				resource.NewResource("dep1", resource.ResourceTypes.Deployment, "test", 300*time.Second),
   104  			},
   105  		},
   106  		{
   107  			description: "multiple deployments with no progress deadline set",
   108  			deps: []*appsv1.Deployment{
   109  				{
   110  					ObjectMeta: metav1.ObjectMeta{
   111  						Name:      "dep1",
   112  						Namespace: "test",
   113  						Labels: map[string]string{
   114  							label.RunIDLabel: labeller.GetRunID(),
   115  						},
   116  					},
   117  					Spec: appsv1.DeploymentSpec{ProgressDeadlineSeconds: utilpointer.Int32Ptr(100)},
   118  				},
   119  				{
   120  					ObjectMeta: metav1.ObjectMeta{
   121  						Name:      "dep2",
   122  						Namespace: "test",
   123  						Labels: map[string]string{
   124  							label.RunIDLabel: labeller.GetRunID(),
   125  						},
   126  					},
   127  				},
   128  			},
   129  			expected: []*resource.Resource{
   130  				resource.NewResource("dep1", resource.ResourceTypes.Deployment, "test", 100*time.Second),
   131  				resource.NewResource("dep2", resource.ResourceTypes.Deployment, "test", 200*time.Second),
   132  			},
   133  		},
   134  		{
   135  			description: "multiple deployments with progress deadline set to max",
   136  			deps: []*appsv1.Deployment{
   137  				{
   138  					ObjectMeta: metav1.ObjectMeta{
   139  						Name:      "dep1",
   140  						Namespace: "test",
   141  						Labels: map[string]string{
   142  							label.RunIDLabel: labeller.GetRunID(),
   143  						},
   144  					},
   145  					Spec: appsv1.DeploymentSpec{ProgressDeadlineSeconds: utilpointer.Int32Ptr(600)},
   146  				},
   147  			},
   148  			expected: []*resource.Resource{
   149  				resource.NewResource("dep1", resource.ResourceTypes.Deployment, "test", 200*time.Second),
   150  			},
   151  		},
   152  		{
   153  			description: "no deployments",
   154  			expected:    []*resource.Resource{},
   155  		},
   156  		{
   157  			description: "multiple deployments in different namespaces",
   158  			deps: []*appsv1.Deployment{
   159  				{
   160  					ObjectMeta: metav1.ObjectMeta{
   161  						Name:      "dep1",
   162  						Namespace: "test",
   163  						Labels: map[string]string{
   164  							label.RunIDLabel: labeller.GetRunID(),
   165  						},
   166  					},
   167  					Spec: appsv1.DeploymentSpec{ProgressDeadlineSeconds: utilpointer.Int32Ptr(100)},
   168  				},
   169  				{
   170  					ObjectMeta: metav1.ObjectMeta{
   171  						Name:      "dep2",
   172  						Namespace: "test1",
   173  						Labels: map[string]string{
   174  							label.RunIDLabel: labeller.GetRunID(),
   175  						},
   176  					},
   177  					Spec: appsv1.DeploymentSpec{ProgressDeadlineSeconds: utilpointer.Int32Ptr(100)},
   178  				},
   179  			},
   180  			expected: []*resource.Resource{
   181  				resource.NewResource("dep1", resource.ResourceTypes.Deployment, "test", 100*time.Second),
   182  			},
   183  		},
   184  		{
   185  			description: "deployment in correct namespace but not deployed by skaffold",
   186  			deps: []*appsv1.Deployment{
   187  				{
   188  					ObjectMeta: metav1.ObjectMeta{
   189  						Name:      "dep1",
   190  						Namespace: "test",
   191  						Labels: map[string]string{
   192  							"some-other-tool": "helm",
   193  						},
   194  					},
   195  					Spec: appsv1.DeploymentSpec{ProgressDeadlineSeconds: utilpointer.Int32Ptr(100)},
   196  				},
   197  			},
   198  			expected: []*resource.Resource{},
   199  		},
   200  		{
   201  			description: "deployment in correct namespace deployed by skaffold but different run",
   202  			deps: []*appsv1.Deployment{
   203  				{
   204  					ObjectMeta: metav1.ObjectMeta{
   205  						Name:      "dep1",
   206  						Namespace: "test",
   207  						Labels: map[string]string{
   208  							label.RunIDLabel: "9876-6789",
   209  						},
   210  					},
   211  					Spec: appsv1.DeploymentSpec{ProgressDeadlineSeconds: utilpointer.Int32Ptr(100)},
   212  				},
   213  			},
   214  			expected: []*resource.Resource{},
   215  		},
   216  	}
   217  
   218  	for _, test := range tests {
   219  		testutil.Run(t, test.description, func(t *testutil.T) {
   220  			objs := make([]runtime.Object, len(test.deps))
   221  			for i, dep := range test.deps {
   222  				objs[i] = dep
   223  			}
   224  			client := fakekubeclientset.NewSimpleClientset(objs...)
   225  			actual, err := getDeployments(context.Background(), client, "test", labeller, 200*time.Second)
   226  			t.CheckErrorAndDeepEqual(test.shouldErr, err, &test.expected, &actual,
   227  				cmp.AllowUnexported(resource.Resource{}, resource.Status{}),
   228  				cmpopts.IgnoreInterfaces(struct{ diag.Diagnose }{}))
   229  		})
   230  	}
   231  }
   232  
   233  func TestGetDeployStatus(t *testing.T) {
   234  	tests := []struct {
   235  		description  string
   236  		counter      *counter
   237  		sc           proto.StatusCode
   238  		expected     string
   239  		expectedCode proto.StatusCode
   240  		shouldErr    bool
   241  	}{
   242  		{
   243  			description:  "one error",
   244  			counter:      &counter{total: 2, failed: 1},
   245  			expected:     "1/2 deployment(s) failed",
   246  			sc:           proto.StatusCode_STATUSCHECK_POD_INITIALIZING,
   247  			expectedCode: proto.StatusCode_STATUSCHECK_POD_INITIALIZING,
   248  			shouldErr:    true,
   249  		},
   250  		{
   251  			description:  "no error",
   252  			sc:           proto.StatusCode_STATUSCHECK_SUCCESS,
   253  			expectedCode: proto.StatusCode_STATUSCHECK_SUCCESS,
   254  			counter:      &counter{total: 2},
   255  		},
   256  		{
   257  			description:  "multiple errors",
   258  			counter:      &counter{total: 3, failed: 2},
   259  			expected:     "2/3 deployment(s) failed",
   260  			sc:           proto.StatusCode_STATUSCHECK_CONFIG_CONNECTOR_FAILED,
   261  			expectedCode: proto.StatusCode_STATUSCHECK_CONFIG_CONNECTOR_FAILED,
   262  			shouldErr:    true,
   263  		},
   264  		{
   265  			description:  "0 deployments",
   266  			counter:      &counter{total: 0},
   267  			expectedCode: proto.StatusCode_STATUSCHECK_SUCCESS,
   268  		},
   269  		{
   270  			description:  "unable to retrieve pods for deployment",
   271  			counter:      &counter{total: 1, failed: 1},
   272  			sc:           proto.StatusCode_STATUSCHECK_DEPLOYMENT_FETCH_ERR,
   273  			expectedCode: proto.StatusCode_STATUSCHECK_DEPLOYMENT_FETCH_ERR,
   274  			shouldErr:    true,
   275  		},
   276  		{
   277  			description:  "one deployment failed and others cancelled and or succeeded",
   278  			counter:      &counter{total: 3, failed: 2},
   279  			sc:           proto.StatusCode_STATUSCHECK_NODE_DISK_PRESSURE,
   280  			expectedCode: proto.StatusCode_STATUSCHECK_NODE_DISK_PRESSURE,
   281  			expected:     "2/3 deployment(s) failed",
   282  			shouldErr:    true,
   283  		},
   284  		{
   285  			description:  "deployments did not stabilize within deadline returns the pod error",
   286  			counter:      &counter{total: 1, failed: 1},
   287  			sc:           proto.StatusCode_STATUSCHECK_UNHEALTHY,
   288  			expected:     "1/1 deployment(s) failed",
   289  			expectedCode: proto.StatusCode_STATUSCHECK_UNHEALTHY,
   290  			shouldErr:    true,
   291  		},
   292  		{
   293  			description:  "user cancelled session",
   294  			counter:      &counter{total: 2, failed: 0, cancelled: 2},
   295  			sc:           proto.StatusCode_STATUSCHECK_USER_CANCELLED,
   296  			expected:     "2/2 deployment(s) status check cancelled",
   297  			expectedCode: proto.StatusCode_STATUSCHECK_USER_CANCELLED,
   298  			shouldErr:    true,
   299  		},
   300  	}
   301  
   302  	for _, test := range tests {
   303  		testutil.Run(t, test.description, func(t *testutil.T) {
   304  			actual, err := getSkaffoldDeployStatus(context.Background(), test.counter, test.sc)
   305  			t.CheckError(test.shouldErr, err)
   306  			t.CheckDeepEqual(test.expectedCode, actual)
   307  			if test.shouldErr {
   308  				t.CheckErrorContains(test.expected, err)
   309  			}
   310  		})
   311  	}
   312  }
   313  
   314  func TestPrintSummaryStatus(t *testing.T) {
   315  	labeller := label.NewLabeller(true, nil, "run-id")
   316  	tests := []struct {
   317  		description string
   318  		namespace   string
   319  		deployment  string
   320  		pending     int32
   321  		ae          *proto.ActionableErr
   322  		expected    string
   323  	}{
   324  		{
   325  			description: "no deployment left and current is in success",
   326  			namespace:   "test",
   327  			deployment:  "dep",
   328  			pending:     0,
   329  			ae:          &proto.ActionableErr{ErrCode: proto.StatusCode_STATUSCHECK_SUCCESS},
   330  			expected:    " - test:deployment/dep is ready.\n",
   331  		},
   332  		{
   333  			description: "default namespace",
   334  			namespace:   "default",
   335  			deployment:  "dep",
   336  			pending:     0,
   337  			ae:          &proto.ActionableErr{ErrCode: proto.StatusCode_STATUSCHECK_SUCCESS},
   338  			expected:    " - deployment/dep is ready.\n",
   339  		},
   340  		{
   341  			description: "no deployment left and current is in error",
   342  			namespace:   "test",
   343  			deployment:  "dep",
   344  			pending:     0,
   345  			ae:          &proto.ActionableErr{ErrCode: proto.StatusCode_STATUSCHECK_DEADLINE_EXCEEDED, Message: "context deadline expired"},
   346  			expected:    " - test:deployment/dep failed. Error: context deadline expired.\n",
   347  		},
   348  		{
   349  			description: "more than 1 deployment left and current is in success",
   350  			namespace:   "test",
   351  			deployment:  "dep",
   352  			pending:     4,
   353  			ae:          &proto.ActionableErr{ErrCode: proto.StatusCode_STATUSCHECK_SUCCESS},
   354  			expected:    " - test:deployment/dep is ready. [4/10 deployment(s) still pending]\n",
   355  		},
   356  		{
   357  			description: "more than 1 deployment left and current is in error",
   358  			namespace:   "test",
   359  			deployment:  "dep",
   360  			pending:     8,
   361  			ae:          &proto.ActionableErr{ErrCode: proto.StatusCode_STATUSCHECK_DEADLINE_EXCEEDED, Message: "context deadline expired"},
   362  			expected:    " - test:deployment/dep failed. Error: context deadline expired.\n",
   363  		},
   364  		{
   365  			description: "skip printing if status check is cancelled",
   366  			namespace:   "test",
   367  			deployment:  "dep",
   368  			pending:     4,
   369  			ae:          &proto.ActionableErr{ErrCode: proto.StatusCode_STATUSCHECK_USER_CANCELLED},
   370  			expected:    "",
   371  		},
   372  	}
   373  
   374  	for _, test := range tests {
   375  		testutil.Run(t, test.description, func(t *testutil.T) {
   376  			monitor := monitor{labeller: labeller}
   377  			out := new(bytes.Buffer)
   378  			rc := newCounter(10)
   379  			rc.pending = test.pending
   380  			testEvent.InitializeState([]latest.Pipeline{{}})
   381  			r := withStatus(resource.NewResource(test.deployment, resource.ResourceTypes.Deployment, test.namespace, 0), test.ae)
   382  			// report status once and set it changed to false.
   383  			r.ReportSinceLastUpdated(false)
   384  			r.UpdateStatus(test.ae)
   385  			monitor.printStatusCheckSummary(out, r, *rc)
   386  			t.CheckDeepEqual(test.expected, out.String())
   387  		})
   388  	}
   389  }
   390  
   391  func TestPrintStatus(t *testing.T) {
   392  	labeller := label.NewLabeller(true, nil, "run-id")
   393  	tests := []struct {
   394  		description string
   395  		rs          []*resource.Resource
   396  		expectedOut string
   397  		expected    bool
   398  	}{
   399  		{
   400  			description: "single resource successful marked complete - skip print",
   401  			rs: []*resource.Resource{
   402  				withStatus(
   403  					resource.NewResource("r1", resource.ResourceTypes.Deployment, "test", 1),
   404  					&proto.ActionableErr{ErrCode: proto.StatusCode_STATUSCHECK_SUCCESS},
   405  				),
   406  			},
   407  			expected: true,
   408  		},
   409  		{
   410  			description: "single resource in error marked complete -skip print",
   411  			rs: []*resource.Resource{
   412  				withStatus(
   413  					resource.NewResource("r1", resource.ResourceTypes.Deployment, "test", 1),
   414  					&proto.ActionableErr{ErrCode: proto.StatusCode_STATUSCHECK_UNKNOWN, Message: "error"},
   415  				),
   416  			},
   417  			expected: true,
   418  		},
   419  		{
   420  			description: "multiple resources 1 not complete",
   421  			rs: []*resource.Resource{
   422  				withStatus(
   423  					resource.NewResource("r1", resource.ResourceTypes.Deployment, "test", 1),
   424  					&proto.ActionableErr{ErrCode: proto.StatusCode_STATUSCHECK_SUCCESS},
   425  				),
   426  				withStatus(
   427  					resource.NewResource("r2", resource.ResourceTypes.Deployment, "test", 1).
   428  						WithPodStatuses([]proto.StatusCode{proto.StatusCode_STATUSCHECK_IMAGE_PULL_ERR}),
   429  					&proto.ActionableErr{ErrCode: proto.StatusCode_STATUSCHECK_DEPLOYMENT_ROLLOUT_PENDING,
   430  						Message: "pending\n"},
   431  				),
   432  			},
   433  			expectedOut: ` - test:deployment/r2: pod failed
   434      - test:pod/foo: pod failed
   435  `,
   436  		},
   437  		{
   438  			description: "multiple resources 1 not complete and retry-able error",
   439  			rs: []*resource.Resource{
   440  				withStatus(
   441  					resource.NewResource("r1", resource.ResourceTypes.Deployment, "test", 1),
   442  					&proto.ActionableErr{ErrCode: proto.StatusCode_STATUSCHECK_SUCCESS},
   443  				),
   444  				withStatus(
   445  					resource.NewResource("r2", resource.ResourceTypes.Deployment, "test", 1),
   446  					&proto.ActionableErr{
   447  						ErrCode: proto.StatusCode_STATUSCHECK_KUBECTL_CONNECTION_ERR,
   448  						Message: resource.MsgKubectlConnection},
   449  				),
   450  			},
   451  			expectedOut: ` - test:deployment/r2: kubectl connection error
   452  `,
   453  		},
   454  		{
   455  			description: "skip printing if status check is cancelled",
   456  			rs: []*resource.Resource{
   457  				withStatus(
   458  					resource.NewResource("r1", resource.ResourceTypes.Deployment, "test", 1),
   459  					&proto.ActionableErr{ErrCode: proto.StatusCode_STATUSCHECK_USER_CANCELLED},
   460  				),
   461  			},
   462  			expected:    true,
   463  			expectedOut: "",
   464  		},
   465  	}
   466  
   467  	for _, test := range tests {
   468  		testutil.Run(t, test.description, func(t *testutil.T) {
   469  			out := new(bytes.Buffer)
   470  			testEvent.InitializeState([]latest.Pipeline{{}})
   471  			monitor := monitor{labeller: labeller}
   472  			actual := monitor.printStatus(test.rs, out)
   473  			t.CheckDeepEqual(test.expectedOut, out.String())
   474  			t.CheckDeepEqual(test.expected, actual)
   475  		})
   476  	}
   477  }
   478  
   479  func withStatus(d *resource.Resource, ae *proto.ActionableErr) *resource.Resource {
   480  	d.UpdateStatus(ae)
   481  	return d
   482  }
   483  
   484  func TestCounterCopy(t *testing.T) {
   485  	tests := []struct {
   486  		description string
   487  		c           *counter
   488  		expected    counter
   489  	}{
   490  		{
   491  			description: "initial counter is copied correctly ",
   492  			c:           newCounter(10),
   493  			expected:    *newCounter(10),
   494  		},
   495  		{
   496  			description: "counter with updated pending is copied correctly",
   497  			c:           &counter{total: 10, pending: 2},
   498  			expected:    counter{total: 10, pending: 2},
   499  		},
   500  		{
   501  			description: "counter with updated failed and pending is copied correctly",
   502  			c:           &counter{total: 10, pending: 5, failed: 3},
   503  			expected:    counter{total: 10, pending: 5, failed: 3},
   504  		},
   505  	}
   506  	for _, test := range tests {
   507  		testutil.Run(t, test.description, func(t *testutil.T) {
   508  			t.CheckDeepEqual(test.expected, test.c.copy(), cmp.AllowUnexported(counter{}))
   509  		})
   510  	}
   511  }
   512  
   513  func TestResourceMarkProcessed(t *testing.T) {
   514  	tests := []struct {
   515  		description string
   516  		c           *counter
   517  		expected    counter
   518  		ctxErr      error
   519  		sc          proto.StatusCode
   520  		expectedB   bool
   521  	}{
   522  		{
   523  			description: "when deployment failed, counter is updated",
   524  			c:           newCounter(10),
   525  			sc:          proto.StatusCode_STATUSCHECK_DEADLINE_EXCEEDED,
   526  			expected:    counter{total: 10, failed: 1, pending: 9},
   527  			expectedB:   true,
   528  		},
   529  		{
   530  			description: "when deployment is cancelled, failed is not updated",
   531  			c:           newCounter(10),
   532  			ctxErr:      context.Canceled,
   533  			expected:    counter{total: 10, failed: 0, pending: 9, cancelled: 1},
   534  		},
   535  		{
   536  			description: "when deployment is successful, counter is updated",
   537  			c:           newCounter(10),
   538  			sc:          proto.StatusCode_STATUSCHECK_SUCCESS,
   539  			expected:    counter{total: 10, failed: 0, pending: 9},
   540  		},
   541  		{
   542  			description: "counter when 1 deployment is updated correctly",
   543  			c:           newCounter(1),
   544  			sc:          proto.StatusCode_STATUSCHECK_SUCCESS,
   545  			expected:    counter{total: 1, failed: 0, pending: 0},
   546  		},
   547  	}
   548  	for _, test := range tests {
   549  		testutil.Run(t, test.description, func(t *testutil.T) {
   550  			ctx := testCtx{err: test.ctxErr, Context: context.Background()}
   551  			actual, actualB := test.c.markProcessed(ctx, test.sc)
   552  			t.CheckDeepEqual(test.expected, actual, cmp.AllowUnexported(counter{}))
   553  			t.CheckDeepEqual(test.expectedB, actualB, cmp.AllowUnexported(counter{}))
   554  		})
   555  	}
   556  }
   557  
   558  func TestPollDeployment(t *testing.T) {
   559  	rolloutCmd := "kubectl --context kubecontext rollout status deployment dep --namespace test --watch=false"
   560  	tests := []struct {
   561  		description string
   562  		dep         *resource.Resource
   563  		runs        [][]validator.Resource
   564  		command     util.Command
   565  		expected    proto.StatusCode
   566  	}{
   567  		{
   568  			description: "pollDeploymentStatus errors out immediately when container error can't recover",
   569  			dep:         resource.NewResource("dep", resource.ResourceTypes.Deployment, "test", time.Second),
   570  			command:     testutil.CmdRunOut(rolloutCmd, "Waiting for replicas to be available"),
   571  			runs: [][]validator.Resource{
   572  				{validator.NewResource(
   573  					"test",
   574  					"pod",
   575  					"dep-pod",
   576  					"Pending",
   577  					&proto.ActionableErr{ErrCode: proto.StatusCode_STATUSCHECK_CONTAINER_TERMINATED},
   578  					[]string{"err"})},
   579  			},
   580  			expected: proto.StatusCode_STATUSCHECK_DEPLOYMENT_ROLLOUT_PENDING,
   581  		},
   582  		{
   583  			description: "pollDeploymentStatus waits when a container can recover and eventually succeeds",
   584  			dep:         resource.NewResource("dep", resource.ResourceTypes.Deployment, "test", time.Second),
   585  			command: testutil.CmdRunOutErr(
   586  				// pending due to recoverable error
   587  				rolloutCmd, "", errors.New("Unable to connect to the server")).
   588  				// successfully rolled out run
   589  				AndRunOut(rolloutCmd, "successfully rolled out"),
   590  			runs: [][]validator.Resource{
   591  				// pod pending due to some k8 infra related recoverable error.
   592  				{validator.NewResource(
   593  					"test",
   594  					"pod",
   595  					"dep-pod",
   596  					"Pending",
   597  					&proto.ActionableErr{ErrCode: proto.StatusCode_STATUSCHECK_NODE_DISK_PRESSURE},
   598  					[]string{"err"})},
   599  			},
   600  			expected: proto.StatusCode_STATUSCHECK_SUCCESS,
   601  		},
   602  	}
   603  	for _, test := range tests {
   604  		testutil.Run(t, test.description, func(t *testutil.T) {
   605  			t.Override(&util.DefaultExecCommand, test.command)
   606  			t.Override(&defaultPollPeriodInMilliseconds, 100)
   607  			testEvent.InitializeState([]latest.Pipeline{{}})
   608  			mockVal := mockValidator{runs: test.runs}
   609  			dep := test.dep.WithValidator(mockVal)
   610  
   611  			pollResourceStatus(context.Background(), &statusConfig{}, dep)
   612  
   613  			t.CheckDeepEqual(test.expected, test.dep.Status().ActionableError().ErrCode, protocmp.Transform())
   614  		})
   615  	}
   616  }
   617  
   618  type mockValidator struct {
   619  	runs      [][]validator.Resource
   620  	iteration int
   621  }
   622  
   623  func (m mockValidator) Run(context.Context) ([]validator.Resource, error) {
   624  	if m.iteration < len(m.runs) {
   625  		m.iteration++
   626  	}
   627  	// keep replaying the last result.
   628  	return m.runs[m.iteration-1], nil
   629  }
   630  
   631  func (m mockValidator) WithLabel(string, string) diag.Diagnose {
   632  	return m
   633  }
   634  
   635  func (m mockValidator) WithValidators([]validator.Validator) diag.Diagnose {
   636  	return m
   637  }
   638  
   639  type statusConfig struct {
   640  	runcontext.RunContext // Embedded to provide the default values.
   641  }
   642  
   643  func (c *statusConfig) GetKubeContext() string { return TestKubeContext }
   644  
   645  type testCtx struct {
   646  	context.Context
   647  	err error
   648  }
   649  
   650  func (t testCtx) Err() error {
   651  	return t.err
   652  }