github.com/kyma-project/kyma-environment-broker@v0.0.1/internal/orchestration/manager/upgrade_cluster_test.go (about)

     1  package manager_test
     2  
     3  import (
     4  	"fmt"
     5  	"testing"
     6  	"time"
     7  
     8  	"github.com/kyma-project/kyma-environment-broker/common/orchestration"
     9  	"github.com/kyma-project/kyma-environment-broker/common/orchestration/automock"
    10  	"github.com/kyma-project/kyma-environment-broker/internal"
    11  	"github.com/kyma-project/kyma-environment-broker/internal/notification"
    12  	notificationAutomock "github.com/kyma-project/kyma-environment-broker/internal/notification/mocks"
    13  	internalOrchestration "github.com/kyma-project/kyma-environment-broker/internal/orchestration"
    14  	"github.com/kyma-project/kyma-environment-broker/internal/orchestration/manager"
    15  	"github.com/kyma-project/kyma-environment-broker/internal/storage"
    16  	"github.com/kyma-project/kyma-environment-broker/internal/storage/dbmodel"
    17  	"github.com/sirupsen/logrus"
    18  	"github.com/stretchr/testify/assert"
    19  	"github.com/stretchr/testify/require"
    20  
    21  	"sigs.k8s.io/controller-runtime/pkg/client/fake"
    22  )
    23  
    24  func TestUpgradeClusterManager_Execute(t *testing.T) {
    25  	k8sClient := fake.NewFakeClient()
    26  	orchestrationConfig := internalOrchestration.Config{
    27  		KymaVersion:       "1.24.5",
    28  		KubernetesVersion: "1.22",
    29  		Namespace:         "default",
    30  		Name:              "policyConfig",
    31  	}
    32  
    33  	t.Run("Empty", func(t *testing.T) {
    34  		// given
    35  		store := storage.NewMemoryStorage()
    36  
    37  		resolver := &automock.RuntimeResolver{}
    38  		defer resolver.AssertExpectations(t)
    39  
    40  		resolver.On("Resolve", orchestration.TargetSpec{
    41  			Include: nil,
    42  			Exclude: nil,
    43  		}).Return([]orchestration.Runtime{}, nil)
    44  
    45  		id := "id"
    46  		err := store.Orchestrations().Insert(internal.Orchestration{
    47  			OrchestrationID: id,
    48  			State:           orchestration.Pending,
    49  			Type:            orchestration.UpgradeClusterOrchestration,
    50  			Parameters: orchestration.Parameters{
    51  				Kyma:       &orchestration.KymaParameters{Version: ""},
    52  				Kubernetes: &orchestration.KubernetesParameters{KubernetesVersion: ""},
    53  				Strategy: orchestration.StrategySpec{
    54  					ScheduleTime: time.Time{},
    55  				},
    56  				Notification: true,
    57  			},
    58  		})
    59  		require.NoError(t, err)
    60  
    61  		notificationTenants := []notification.NotificationTenant{}
    62  		notificationParas := notification.NotificationParams{
    63  			OrchestrationID: id,
    64  			EventType:       notification.KubernetesMaintenanceNumber,
    65  			Tenants:         notificationTenants,
    66  		}
    67  		notificationBuilder := &notificationAutomock.BundleBuilder{}
    68  		bundle := &notificationAutomock.Bundle{}
    69  		notificationBuilder.On("NewBundle", id, notificationParas).Return(bundle, nil).Once()
    70  		bundle.On("CreateNotificationEvent").Return(nil).Once()
    71  
    72  		svc := manager.NewUpgradeClusterManager(store.Orchestrations(), store.Operations(), store.Instances(), nil,
    73  			resolver, 20*time.Millisecond, logrus.New(), k8sClient, orchestrationConfig, notificationBuilder, 1000)
    74  
    75  		// when
    76  		_, err = svc.Execute(id)
    77  		require.NoError(t, err)
    78  
    79  		o, err := store.Orchestrations().GetByID(id)
    80  		require.NoError(t, err)
    81  
    82  		assert.Equal(t, orchestration.Succeeded, o.State)
    83  	})
    84  	t.Run("InProgress", func(t *testing.T) {
    85  		// given
    86  		store := storage.NewMemoryStorage()
    87  
    88  		resolver := &automock.RuntimeResolver{}
    89  		defer resolver.AssertExpectations(t)
    90  
    91  		id := "id"
    92  		err := store.Orchestrations().Insert(internal.Orchestration{
    93  			OrchestrationID: id,
    94  			State:           orchestration.InProgress,
    95  			Type:            orchestration.UpgradeClusterOrchestration,
    96  			Parameters: orchestration.Parameters{
    97  				Strategy: orchestration.StrategySpec{
    98  					Type:         orchestration.ParallelStrategy,
    99  					Schedule:     time.Now().Format(time.RFC3339),
   100  					ScheduleTime: time.Time{},
   101  				},
   102  				Notification: true,
   103  			},
   104  		})
   105  		require.NoError(t, err)
   106  
   107  		notificationTenants := []notification.NotificationTenant{}
   108  		notificationParas := notification.NotificationParams{
   109  			OrchestrationID: id,
   110  			EventType:       notification.KubernetesMaintenanceNumber,
   111  			Tenants:         notificationTenants,
   112  		}
   113  		notificationBuilder := &notificationAutomock.BundleBuilder{}
   114  		bundle := &notificationAutomock.Bundle{}
   115  		notificationBuilder.On("NewBundle", id, notificationParas).Return(bundle, nil).Once()
   116  		bundle.On("CreateNotificationEvent").Return(nil).Once()
   117  
   118  		svc := manager.NewUpgradeClusterManager(store.Orchestrations(), store.Operations(), store.Instances(), &testExecutor{},
   119  			resolver, poolingInterval, logrus.New(), k8sClient, orchestrationConfig, notificationBuilder, 1000)
   120  
   121  		// when
   122  		_, err = svc.Execute(id)
   123  		require.NoError(t, err)
   124  
   125  		o, err := store.Orchestrations().GetByID(id)
   126  		require.NoError(t, err)
   127  
   128  		assert.Equal(t, orchestration.Succeeded, o.State)
   129  
   130  	})
   131  
   132  	t.Run("DryRun", func(t *testing.T) {
   133  		// given
   134  		store := storage.NewMemoryStorage()
   135  
   136  		resolver := &automock.RuntimeResolver{}
   137  		defer resolver.AssertExpectations(t)
   138  		resolver.On("Resolve", orchestration.TargetSpec{}).Return([]orchestration.Runtime{}, nil).Once()
   139  
   140  		id := "id"
   141  		err := store.Orchestrations().Insert(internal.Orchestration{
   142  			OrchestrationID: id,
   143  			State:           orchestration.Pending,
   144  			Type:            orchestration.UpgradeClusterOrchestration,
   145  			Parameters: orchestration.Parameters{
   146  				DryRun:     true,
   147  				Kubernetes: &orchestration.KubernetesParameters{KubernetesVersion: ""},
   148  				Kyma:       &orchestration.KymaParameters{Version: ""},
   149  				Strategy: orchestration.StrategySpec{
   150  					ScheduleTime: time.Time{},
   151  				},
   152  				Notification: true,
   153  			}})
   154  		require.NoError(t, err)
   155  
   156  		notificationTenants := []notification.NotificationTenant{}
   157  		notificationParas := notification.NotificationParams{
   158  			OrchestrationID: id,
   159  			EventType:       notification.KubernetesMaintenanceNumber,
   160  			Tenants:         notificationTenants,
   161  		}
   162  		notificationBuilder := &notificationAutomock.BundleBuilder{}
   163  		bundle := &notificationAutomock.Bundle{}
   164  		notificationBuilder.On("NewBundle", id, notificationParas).Return(bundle, nil).Once()
   165  		bundle.On("CreateNotificationEvent").Return(nil).Once()
   166  
   167  		svc := manager.NewUpgradeClusterManager(store.Orchestrations(), store.Operations(), store.Instances(), nil,
   168  			resolver, poolingInterval, logrus.New(), k8sClient, orchestrationConfig, notificationBuilder, 1000)
   169  
   170  		// when
   171  		_, err = svc.Execute(id)
   172  		require.NoError(t, err)
   173  
   174  		o, err := store.Orchestrations().GetByID(id)
   175  		require.NoError(t, err)
   176  
   177  		assert.Equal(t, orchestration.Succeeded, o.State)
   178  
   179  	})
   180  
   181  	t.Run("InProgressWithRuntimeOperations", func(t *testing.T) {
   182  		// given
   183  		store := storage.NewMemoryStorage()
   184  
   185  		resolver := &automock.RuntimeResolver{}
   186  		defer resolver.AssertExpectations(t)
   187  
   188  		id := "id"
   189  
   190  		upgradeOperation := internal.UpgradeClusterOperation{
   191  			Operation: internal.Operation{
   192  				ID:                     id,
   193  				Version:                0,
   194  				CreatedAt:              time.Now(),
   195  				UpdatedAt:              time.Now(),
   196  				InstanceID:             "",
   197  				ProvisionerOperationID: "",
   198  				OrchestrationID:        id,
   199  				State:                  orchestration.Succeeded,
   200  				Description:            "operation created",
   201  				ProvisioningParameters: internal.ProvisioningParameters{},
   202  				RuntimeOperation: orchestration.RuntimeOperation{
   203  					Runtime: orchestration.Runtime{
   204  						RuntimeID:    id,
   205  						SubAccountID: "sub",
   206  					},
   207  					DryRun: false,
   208  				},
   209  				InputCreator: nil,
   210  			},
   211  		}
   212  		err := store.Operations().InsertUpgradeClusterOperation(upgradeOperation)
   213  		require.NoError(t, err)
   214  
   215  		givenO := internal.Orchestration{
   216  			OrchestrationID: id,
   217  			State:           orchestration.InProgress,
   218  			Type:            orchestration.UpgradeClusterOrchestration,
   219  			Parameters: orchestration.Parameters{
   220  				Strategy: orchestration.StrategySpec{
   221  					Type:         orchestration.ParallelStrategy,
   222  					Schedule:     time.Now().Format(time.RFC3339),
   223  					ScheduleTime: time.Time{},
   224  				},
   225  				Notification: true,
   226  			},
   227  		}
   228  		err = store.Orchestrations().Insert(givenO)
   229  		require.NoError(t, err)
   230  
   231  		notificationTenants := []notification.NotificationTenant{}
   232  		notificationParas := notification.NotificationParams{
   233  			OrchestrationID: id,
   234  			EventType:       notification.KubernetesMaintenanceNumber,
   235  			Tenants:         notificationTenants,
   236  		}
   237  		notificationBuilder := &notificationAutomock.BundleBuilder{}
   238  		bundle := &notificationAutomock.Bundle{}
   239  		notificationBuilder.On("NewBundle", id, notificationParas).Return(bundle, nil).Once()
   240  		bundle.On("CreateNotificationEvent").Return(nil).Once()
   241  
   242  		svc := manager.NewUpgradeClusterManager(store.Orchestrations(), store.Operations(), store.Instances(), &testExecutor{},
   243  			resolver, poolingInterval, logrus.New(), k8sClient, orchestrationConfig, notificationBuilder, 1000)
   244  
   245  		// when
   246  		_, err = svc.Execute(id)
   247  		require.NoError(t, err)
   248  
   249  		o, err := store.Orchestrations().GetByID(id)
   250  		require.NoError(t, err)
   251  
   252  		assert.Equal(t, orchestration.Succeeded, o.State)
   253  	})
   254  
   255  	t.Run("Canceled", func(t *testing.T) {
   256  		// given
   257  		fmt.Println("case started.")
   258  		store := storage.NewMemoryStorage()
   259  
   260  		resolver := &automock.RuntimeResolver{}
   261  		defer resolver.AssertExpectations(t)
   262  
   263  		id := "id"
   264  		err := store.Orchestrations().Insert(internal.Orchestration{
   265  			OrchestrationID: id,
   266  			State:           orchestration.Canceling,
   267  			Parameters: orchestration.Parameters{Strategy: orchestration.StrategySpec{
   268  				Type:         orchestration.ParallelStrategy,
   269  				Schedule:     time.Now().Format(time.RFC3339),
   270  				ScheduleTime: time.Time{},
   271  			},
   272  				Notification: true,
   273  			},
   274  		})
   275  
   276  		require.NoError(t, err)
   277  		err = store.Operations().InsertUpgradeClusterOperation(internal.UpgradeClusterOperation{
   278  			Operation: internal.Operation{
   279  				ID:              id,
   280  				OrchestrationID: id,
   281  				State:           orchestration.Pending,
   282  			},
   283  		})
   284  		require.NoError(t, err)
   285  
   286  		notificationParas := notification.NotificationParams{
   287  			OrchestrationID: id,
   288  			Tenants:         []notification.NotificationTenant{},
   289  		}
   290  		notificationBuilder := &notificationAutomock.BundleBuilder{}
   291  		bundle := &notificationAutomock.Bundle{}
   292  		notificationBuilder.On("NewBundle", id, notificationParas).Return(bundle, nil).Once()
   293  		bundle.On("CancelNotificationEvent").Return(nil).Once()
   294  
   295  		svc := manager.NewUpgradeClusterManager(store.Orchestrations(), store.Operations(), store.Instances(), &testExecutor{}, resolver,
   296  			poolingInterval, logrus.New(), k8sClient, orchestrationConfig, notificationBuilder, 1000)
   297  
   298  		// when
   299  		_, err = svc.Execute(id)
   300  		require.NoError(t, err)
   301  
   302  		o, err := store.Orchestrations().GetByID(id)
   303  		require.NoError(t, err)
   304  
   305  		assert.Equal(t, orchestration.Canceled, o.State)
   306  
   307  		op, err := store.Operations().GetUpgradeClusterOperationByID(id)
   308  		require.NoError(t, err)
   309  
   310  		assert.Equal(t, orchestration.Canceled, string(op.State))
   311  	})
   312  
   313  	t.Run("Retrying failed orchestration", func(t *testing.T) {
   314  		// given
   315  		store := storage.NewMemoryStorage()
   316  
   317  		resolver := &automock.RuntimeResolver{}
   318  		defer resolver.AssertExpectations(t)
   319  
   320  		id := "id"
   321  		opId := "op-" + id
   322  		err := store.Orchestrations().Insert(internal.Orchestration{
   323  			OrchestrationID: id,
   324  			State:           orchestration.Retrying,
   325  			Type:            orchestration.UpgradeClusterOrchestration,
   326  			Parameters: orchestration.Parameters{Strategy: orchestration.StrategySpec{
   327  				Type:         orchestration.ParallelStrategy,
   328  				Schedule:     time.Now().Format(time.RFC3339),
   329  				Parallel:     orchestration.ParallelStrategySpec{Workers: 2},
   330  				ScheduleTime: time.Time{},
   331  			},
   332  				Notification: true,
   333  			},
   334  		})
   335  		require.NoError(t, err)
   336  
   337  		err = store.Operations().InsertUpgradeClusterOperation(internal.UpgradeClusterOperation{
   338  			Operation: internal.Operation{
   339  				ID:              opId,
   340  				OrchestrationID: id,
   341  				State:           orchestration.Retrying,
   342  				RuntimeOperation: orchestration.RuntimeOperation{
   343  					ID:      opId,
   344  					Runtime: orchestration.Runtime{},
   345  					DryRun:  false,
   346  				},
   347  				InputCreator: nil,
   348  			},
   349  		})
   350  		require.NoError(t, err)
   351  
   352  		notificationTenants := []notification.NotificationTenant{
   353  			{
   354  				StartDate: time.Now().Format("2006-01-02 15:04:05"),
   355  			},
   356  		}
   357  		notificationParas := notification.NotificationParams{
   358  			OrchestrationID: id,
   359  			EventType:       notification.KubernetesMaintenanceNumber,
   360  			Tenants:         notificationTenants,
   361  		}
   362  		notificationBuilder := &notificationAutomock.BundleBuilder{}
   363  		bundle := &notificationAutomock.Bundle{}
   364  		notificationBuilder.On("NewBundle", id, notificationParas).Return(bundle, nil).Once()
   365  		bundle.On("CreateNotificationEvent").Return(nil).Once()
   366  
   367  		executor := retryTestExecutor{
   368  			store:       store,
   369  			upgradeType: orchestration.UpgradeClusterOrchestration,
   370  		}
   371  		svc := manager.NewUpgradeClusterManager(store.Orchestrations(), store.Operations(), store.Instances(), &executor, resolver,
   372  			poolingInterval, logrus.New(), k8sClient, orchestrationConfig, notificationBuilder, 1000)
   373  
   374  		_, err = store.Orchestrations().GetByID(id)
   375  		require.NoError(t, err)
   376  
   377  		_, err = store.Operations().GetUpgradeClusterOperationByID("op-id")
   378  		require.NoError(t, err)
   379  
   380  		// when
   381  		_, err = svc.Execute(id)
   382  		require.NoError(t, err)
   383  
   384  		o, err := store.Orchestrations().GetByID(id)
   385  		require.NoError(t, err)
   386  
   387  		assert.Equal(t, orchestration.Failed, o.State)
   388  
   389  		op, err := store.Operations().GetUpgradeClusterOperationByID(opId)
   390  		require.NoError(t, err)
   391  
   392  		assert.Equal(t, orchestration.Retrying, string(op.State))
   393  	})
   394  
   395  	t.Run("Retrying failed orchestration and create a new operation on same instanceID", func(t *testing.T) {
   396  		// given
   397  		store := storage.NewMemoryStorage()
   398  
   399  		resolver := &automock.RuntimeResolver{}
   400  		defer resolver.AssertExpectations(t)
   401  
   402  		id := "id"
   403  		opId := "op-" + id
   404  		instanceID := opId + "-1234"
   405  		runtimeID := opId + "-5678"
   406  
   407  		resolver.On("Resolve", orchestration.TargetSpec{
   408  			Include: []orchestration.RuntimeTarget{
   409  				{RuntimeID: opId},
   410  			},
   411  			Exclude: nil,
   412  		}).Return([]orchestration.Runtime{{
   413  			InstanceID: instanceID,
   414  			RuntimeID:  runtimeID,
   415  		}}, nil)
   416  
   417  		err := store.Instances().Insert(internal.Instance{
   418  			InstanceID: instanceID,
   419  			RuntimeID:  runtimeID,
   420  		})
   421  		require.NoError(t, err)
   422  
   423  		err = store.Orchestrations().Insert(
   424  			internal.Orchestration{
   425  				OrchestrationID: id,
   426  				State:           orchestration.Retrying,
   427  				Type:            orchestration.UpgradeClusterOrchestration,
   428  				Parameters: orchestration.Parameters{
   429  					Strategy: orchestration.StrategySpec{
   430  						Type:         orchestration.ParallelStrategy,
   431  						Schedule:     time.Now().Format(time.RFC3339),
   432  						Parallel:     orchestration.ParallelStrategySpec{Workers: 2},
   433  						ScheduleTime: time.Time{},
   434  					},
   435  					Targets: orchestration.TargetSpec{
   436  						Include: []orchestration.RuntimeTarget{
   437  							{RuntimeID: opId},
   438  						},
   439  						Exclude: nil,
   440  					},
   441  					RetryOperation: orchestration.RetryOperationParameters{
   442  						RetryOperations: []string{"op-id"},
   443  					},
   444  					Notification: false,
   445  				},
   446  			})
   447  
   448  		require.NoError(t, err)
   449  
   450  		err = store.Operations().InsertUpgradeClusterOperation(internal.UpgradeClusterOperation{
   451  			Operation: internal.Operation{
   452  				ID:              opId,
   453  				OrchestrationID: id,
   454  				State:           orchestration.Failed,
   455  				InstanceID:      instanceID,
   456  				Type:            internal.OperationTypeUpgradeCluster,
   457  				RuntimeOperation: orchestration.RuntimeOperation{
   458  					ID: opId,
   459  					Runtime: orchestration.Runtime{
   460  						InstanceID: instanceID,
   461  						RuntimeID:  runtimeID,
   462  					},
   463  					DryRun: false,
   464  				},
   465  			},
   466  		})
   467  		require.NoError(t, err)
   468  
   469  		notificationBuilder := &notificationAutomock.BundleBuilder{}
   470  
   471  		executor := retryTestExecutor{
   472  			store:       store,
   473  			upgradeType: orchestration.UpgradeClusterOrchestration,
   474  		}
   475  		svc := manager.NewUpgradeClusterManager(store.Orchestrations(), store.Operations(), store.Instances(), &executor, resolver,
   476  			poolingInterval, logrus.New(), k8sClient, orchestrationConfig, notificationBuilder, 1000)
   477  
   478  		_, err = store.Operations().GetUpgradeClusterOperationByID(opId)
   479  		require.NoError(t, err)
   480  
   481  		// when
   482  		_, err = svc.Execute(id)
   483  		require.NoError(t, err)
   484  
   485  		o, err := store.Orchestrations().GetByID(id)
   486  		require.NoError(t, err)
   487  
   488  		assert.Equal(t, orchestration.Succeeded, o.State)
   489  
   490  		op, err := store.Operations().GetUpgradeClusterOperationByID(opId)
   491  		require.NoError(t, err)
   492  
   493  		assert.Equal(t, orchestration.Failed, string(op.State))
   494  
   495  		//verify a new operation with same instanceID is created
   496  		ops, _, _, err := store.Operations().ListUpgradeClusterOperationsByOrchestrationID(id, dbmodel.OperationFilter{})
   497  		require.NoError(t, err)
   498  		assert.Equal(t, 2, len(ops))
   499  
   500  		for _, op := range ops {
   501  			if op.Operation.ID != opId {
   502  				assert.Equal(t, orchestration.Succeeded, string(op.State))
   503  				assert.Equal(t, instanceID, string(op.Operation.InstanceID))
   504  				assert.Equal(t, internal.OperationTypeUpgradeCluster, op.Operation.Type)
   505  			}
   506  		}
   507  	})
   508  
   509  	t.Run("Retrying resumed in progress orchestration", func(t *testing.T) {
   510  		// given
   511  		store := storage.NewMemoryStorage()
   512  
   513  		resolver := &automock.RuntimeResolver{}
   514  		defer resolver.AssertExpectations(t)
   515  
   516  		id := "id"
   517  		opId := "op-" + id
   518  		err := store.Orchestrations().Insert(internal.Orchestration{
   519  			OrchestrationID: id,
   520  			State:           orchestration.InProgress,
   521  			Type:            orchestration.UpgradeClusterOrchestration,
   522  			Parameters: orchestration.Parameters{Strategy: orchestration.StrategySpec{
   523  				Type:         orchestration.ParallelStrategy,
   524  				Schedule:     time.Now().Format(time.RFC3339),
   525  				Parallel:     orchestration.ParallelStrategySpec{Workers: 2},
   526  				ScheduleTime: time.Time{},
   527  			},
   528  				Notification: true,
   529  			},
   530  		})
   531  		require.NoError(t, err)
   532  
   533  		err = store.Operations().InsertUpgradeClusterOperation(internal.UpgradeClusterOperation{
   534  			Operation: internal.Operation{
   535  				ID:              opId,
   536  				OrchestrationID: id,
   537  				State:           orchestration.Retrying,
   538  				RuntimeOperation: orchestration.RuntimeOperation{
   539  					ID:      opId,
   540  					Runtime: orchestration.Runtime{},
   541  					DryRun:  false,
   542  				},
   543  				InputCreator: nil,
   544  			},
   545  		})
   546  		require.NoError(t, err)
   547  
   548  		notificationTenants := []notification.NotificationTenant{
   549  			{
   550  				StartDate: time.Now().Format("2006-01-02 15:04:05"),
   551  			},
   552  		}
   553  		notificationParas := notification.NotificationParams{
   554  			OrchestrationID: id,
   555  			EventType:       notification.KubernetesMaintenanceNumber,
   556  			Tenants:         notificationTenants,
   557  		}
   558  		notificationBuilder := &notificationAutomock.BundleBuilder{}
   559  		bundle := &notificationAutomock.Bundle{}
   560  		notificationBuilder.On("NewBundle", id, notificationParas).Return(bundle, nil).Once()
   561  		bundle.On("CreateNotificationEvent").Return(nil).Once()
   562  
   563  		executor := retryTestExecutor{
   564  			store:       store,
   565  			upgradeType: orchestration.UpgradeClusterOrchestration,
   566  		}
   567  		svc := manager.NewUpgradeClusterManager(store.Orchestrations(), store.Operations(), store.Instances(), &executor, resolver,
   568  			poolingInterval, logrus.New(), k8sClient, orchestrationConfig, notificationBuilder, 1000)
   569  
   570  		// when
   571  		_, err = svc.Execute(id)
   572  		require.NoError(t, err)
   573  
   574  		o, err := store.Orchestrations().GetByID(id)
   575  		require.NoError(t, err)
   576  
   577  		assert.Equal(t, orchestration.Succeeded, o.State)
   578  
   579  		op, err := store.Operations().GetUpgradeClusterOperationByID(opId)
   580  		require.NoError(t, err)
   581  
   582  		assert.Equal(t, orchestration.Succeeded, string(op.State))
   583  	})
   584  }