github.com/kyma-project/kyma-environment-broker@v0.0.1/internal/orchestration/manager/upgrade_kyma_test.go (about)

     1  package manager_test
     2  
     3  import (
     4  	"fmt"
     5  	"testing"
     6  	"time"
     7  
     8  	"github.com/kyma-project/kyma-environment-broker/internal/notification"
     9  	internalOrchestration "github.com/kyma-project/kyma-environment-broker/internal/orchestration"
    10  	"github.com/kyma-project/kyma-environment-broker/internal/storage/dbmodel"
    11  
    12  	"github.com/stretchr/testify/assert"
    13  	"github.com/stretchr/testify/mock"
    14  
    15  	"sigs.k8s.io/controller-runtime/pkg/client/fake"
    16  
    17  	"github.com/kyma-project/kyma-environment-broker/common/orchestration"
    18  	"github.com/kyma-project/kyma-environment-broker/common/orchestration/automock"
    19  	"github.com/kyma-project/kyma-environment-broker/internal"
    20  	notificationAutomock "github.com/kyma-project/kyma-environment-broker/internal/notification/mocks"
    21  	"github.com/kyma-project/kyma-environment-broker/internal/orchestration/manager"
    22  	"github.com/kyma-project/kyma-environment-broker/internal/storage"
    23  
    24  	"github.com/sirupsen/logrus"
    25  	"github.com/stretchr/testify/require"
    26  )
    27  
    28  const (
    29  	poolingInterval    = 20 * time.Millisecond
    30  	defaultKymaVersion = "1.24.5"
    31  )
    32  
    33  func TestUpgradeKymaManager_Execute(t *testing.T) {
    34  	k8sClient := fake.NewFakeClient()
    35  	orchestrationConfig := internalOrchestration.Config{
    36  		KymaVersion:       defaultKymaVersion,
    37  		KubernetesVersion: "1.22",
    38  		Namespace:         "default",
    39  		Name:              "policyConfig",
    40  	}
    41  
    42  	t.Run("Empty", func(t *testing.T) {
    43  		// given
    44  		store := storage.NewMemoryStorage()
    45  
    46  		resolver := &automock.RuntimeResolver{}
    47  		defer resolver.AssertExpectations(t)
    48  
    49  		resolver.On("Resolve", orchestration.TargetSpec{
    50  			Include: nil,
    51  			Exclude: nil,
    52  		}).Return([]orchestration.Runtime{}, nil)
    53  
    54  		id := "id"
    55  		err := store.Orchestrations().Insert(internal.Orchestration{
    56  			OrchestrationID: id,
    57  			State:           orchestration.Pending,
    58  			Type:            orchestration.UpgradeKymaOrchestration,
    59  			Parameters: orchestration.Parameters{
    60  				Kyma:       &orchestration.KymaParameters{Version: ""},
    61  				Kubernetes: &orchestration.KubernetesParameters{KubernetesVersion: ""},
    62  				Strategy: orchestration.StrategySpec{
    63  					ScheduleTime: time.Time{},
    64  				},
    65  				Notification: true,
    66  			},
    67  		})
    68  		require.NoError(t, err)
    69  
    70  		notificationTenants := []notification.NotificationTenant{}
    71  		notificationParas := notification.NotificationParams{
    72  			OrchestrationID: id,
    73  			EventType:       notification.KymaMaintenanceNumber,
    74  			Tenants:         notificationTenants,
    75  		}
    76  		notificationBuilder := &notificationAutomock.BundleBuilder{}
    77  		bundle := &notificationAutomock.Bundle{}
    78  		notificationBuilder.On("NewBundle", mock.Anything, notificationParas).Return(bundle, nil).Once()
    79  		bundle.On("CreateNotificationEvent").Return(nil).Once()
    80  
    81  		svc := manager.NewUpgradeKymaManager(store.Orchestrations(), store.Operations(), store.Instances(), nil,
    82  			resolver, 20*time.Millisecond, logrus.New(), k8sClient, &orchestrationConfig, notificationBuilder, 1000)
    83  
    84  		// when
    85  		_, err = svc.Execute(id)
    86  		require.NoError(t, err)
    87  
    88  		o, err := store.Orchestrations().GetByID(id)
    89  		require.NoError(t, err)
    90  
    91  		assert.Equal(t, orchestration.Succeeded, o.State)
    92  	})
    93  	t.Run("InProgress", func(t *testing.T) {
    94  		// given
    95  		store := storage.NewMemoryStorage()
    96  
    97  		resolver := &automock.RuntimeResolver{}
    98  		defer resolver.AssertExpectations(t)
    99  
   100  		id := "id"
   101  		err := store.Orchestrations().Insert(internal.Orchestration{
   102  			OrchestrationID: id,
   103  			State:           orchestration.InProgress,
   104  			Type:            orchestration.UpgradeKymaOrchestration,
   105  			Parameters: orchestration.Parameters{
   106  				Strategy: orchestration.StrategySpec{
   107  					Type:         orchestration.ParallelStrategy,
   108  					Schedule:     time.Now().Format(time.RFC3339),
   109  					ScheduleTime: time.Time{},
   110  				},
   111  				Notification: true,
   112  			},
   113  		})
   114  		require.NoError(t, err)
   115  
   116  		notificationTenants := []notification.NotificationTenant{}
   117  		notificationParas := notification.NotificationParams{
   118  			OrchestrationID: id,
   119  			EventType:       notification.KymaMaintenanceNumber,
   120  			Tenants:         notificationTenants,
   121  		}
   122  		notificationBuilder := &notificationAutomock.BundleBuilder{}
   123  		bundle := &notificationAutomock.Bundle{}
   124  		notificationBuilder.On("NewBundle", id, notificationParas).Return(bundle, nil).Once()
   125  		bundle.On("CreateNotificationEvent").Return(nil).Once()
   126  
   127  		svc := manager.NewUpgradeKymaManager(store.Orchestrations(), store.Operations(), store.Instances(), &testExecutor{},
   128  			resolver, poolingInterval, logrus.New(), k8sClient, &orchestrationConfig, notificationBuilder, 1000)
   129  
   130  		// when
   131  		_, err = svc.Execute(id)
   132  		require.NoError(t, err)
   133  
   134  		o, err := store.Orchestrations().GetByID(id)
   135  		require.NoError(t, err)
   136  
   137  		assert.Equal(t, orchestration.Succeeded, o.State)
   138  
   139  	})
   140  
   141  	t.Run("DryRun", func(t *testing.T) {
   142  		// given
   143  		store := storage.NewMemoryStorage()
   144  
   145  		resolver := &automock.RuntimeResolver{}
   146  		defer resolver.AssertExpectations(t)
   147  		resolver.On("Resolve", orchestration.TargetSpec{}).Return([]orchestration.Runtime{}, nil).Once()
   148  
   149  		id := "id"
   150  		err := store.Orchestrations().Insert(internal.Orchestration{
   151  			OrchestrationID: id,
   152  			State:           orchestration.Pending,
   153  			Type:            orchestration.UpgradeKymaOrchestration,
   154  			Parameters: orchestration.Parameters{
   155  				DryRun:     true,
   156  				Kyma:       &orchestration.KymaParameters{Version: ""},
   157  				Kubernetes: &orchestration.KubernetesParameters{KubernetesVersion: ""},
   158  				Strategy: orchestration.StrategySpec{
   159  					ScheduleTime: time.Time{},
   160  				},
   161  				Notification: true,
   162  			}})
   163  		require.NoError(t, err)
   164  
   165  		notificationTenants := []notification.NotificationTenant{}
   166  		notificationParas := notification.NotificationParams{
   167  			OrchestrationID: id,
   168  			EventType:       notification.KymaMaintenanceNumber,
   169  			Tenants:         notificationTenants,
   170  		}
   171  		notificationBuilder := &notificationAutomock.BundleBuilder{}
   172  		bundle := &notificationAutomock.Bundle{}
   173  		notificationBuilder.On("NewBundle", id, notificationParas).Return(bundle, nil).Once()
   174  		bundle.On("CreateNotificationEvent").Return(nil).Once()
   175  
   176  		svc := manager.NewUpgradeKymaManager(store.Orchestrations(), store.Operations(), store.Instances(), nil,
   177  			resolver, poolingInterval, logrus.New(), k8sClient, &orchestrationConfig, notificationBuilder, 1000)
   178  
   179  		// when
   180  		_, err = svc.Execute(id)
   181  		require.NoError(t, err)
   182  
   183  		o, err := store.Orchestrations().GetByID(id)
   184  		require.NoError(t, err)
   185  
   186  		assert.Equal(t, orchestration.Succeeded, o.State)
   187  
   188  	})
   189  
   190  	t.Run("InProgressWithRuntimeOperations", func(t *testing.T) {
   191  		// given
   192  		store := storage.NewMemoryStorage()
   193  
   194  		resolver := &automock.RuntimeResolver{}
   195  		defer resolver.AssertExpectations(t)
   196  
   197  		id := "id"
   198  
   199  		upgradeOperation := internal.UpgradeKymaOperation{
   200  			Operation: internal.Operation{
   201  				ID:                     id,
   202  				Version:                0,
   203  				CreatedAt:              time.Now(),
   204  				UpdatedAt:              time.Now(),
   205  				InstanceID:             "",
   206  				ProvisionerOperationID: "",
   207  				OrchestrationID:        id,
   208  				State:                  orchestration.Succeeded,
   209  				Description:            "operation created",
   210  				ProvisioningParameters: internal.ProvisioningParameters{},
   211  				RuntimeOperation: orchestration.RuntimeOperation{
   212  					Runtime: orchestration.Runtime{
   213  						RuntimeID:    id,
   214  						SubAccountID: "sub",
   215  					},
   216  					DryRun: false,
   217  				},
   218  				InputCreator: nil,
   219  			},
   220  		}
   221  		err := store.Operations().InsertUpgradeKymaOperation(upgradeOperation)
   222  		require.NoError(t, err)
   223  
   224  		givenO := internal.Orchestration{
   225  			OrchestrationID: id,
   226  			State:           orchestration.InProgress,
   227  			Type:            orchestration.UpgradeKymaOrchestration,
   228  			Parameters: orchestration.Parameters{
   229  				Strategy: orchestration.StrategySpec{
   230  					Type:         orchestration.ParallelStrategy,
   231  					Schedule:     time.Now().Format(time.RFC3339),
   232  					ScheduleTime: time.Time{},
   233  				},
   234  				Notification: true,
   235  			},
   236  		}
   237  		err = store.Orchestrations().Insert(givenO)
   238  		require.NoError(t, err)
   239  
   240  		notificationTenants := []notification.NotificationTenant{}
   241  		notificationParas := notification.NotificationParams{
   242  			OrchestrationID: id,
   243  			EventType:       notification.KymaMaintenanceNumber,
   244  			Tenants:         notificationTenants,
   245  		}
   246  		notificationBuilder := &notificationAutomock.BundleBuilder{}
   247  		bundle := &notificationAutomock.Bundle{}
   248  		notificationBuilder.On("NewBundle", id, notificationParas).Return(bundle, nil).Once()
   249  		bundle.On("CreateNotificationEvent").Return(nil).Once()
   250  
   251  		svc := manager.NewUpgradeKymaManager(store.Orchestrations(), store.Operations(), store.Instances(), &testExecutor{},
   252  			resolver, poolingInterval, logrus.New(), k8sClient, &orchestrationConfig, notificationBuilder, 1000)
   253  
   254  		// when
   255  		_, err = svc.Execute(id)
   256  		require.NoError(t, err)
   257  
   258  		o, err := store.Orchestrations().GetByID(id)
   259  		require.NoError(t, err)
   260  
   261  		assert.Equal(t, orchestration.Succeeded, o.State)
   262  	})
   263  
   264  	t.Run("Canceled", func(t *testing.T) {
   265  		// given
   266  		store := storage.NewMemoryStorage()
   267  
   268  		resolver := &automock.RuntimeResolver{}
   269  		defer resolver.AssertExpectations(t)
   270  
   271  		id := "id"
   272  		err := store.Orchestrations().Insert(internal.Orchestration{
   273  			OrchestrationID: id,
   274  			State:           orchestration.Canceling,
   275  			Parameters: orchestration.Parameters{Strategy: orchestration.StrategySpec{
   276  				Type:         orchestration.ParallelStrategy,
   277  				Schedule:     time.Now().Format(time.RFC3339),
   278  				ScheduleTime: time.Time{},
   279  			},
   280  				Notification: true,
   281  			},
   282  		})
   283  
   284  		require.NoError(t, err)
   285  		err = store.Operations().InsertUpgradeKymaOperation(internal.UpgradeKymaOperation{
   286  			Operation: internal.Operation{
   287  				ID:              id,
   288  				OrchestrationID: id,
   289  				State:           orchestration.Pending,
   290  				RuntimeOperation: orchestration.RuntimeOperation{
   291  					Runtime: orchestration.Runtime{
   292  						RuntimeID:    id,
   293  						SubAccountID: "sub",
   294  					},
   295  					DryRun: false,
   296  				},
   297  			},
   298  		})
   299  
   300  		notificationParas := notification.NotificationParams{
   301  			OrchestrationID: id,
   302  			Tenants:         []notification.NotificationTenant{},
   303  		}
   304  		notificationBuilder := &notificationAutomock.BundleBuilder{}
   305  		bundle := &notificationAutomock.Bundle{}
   306  		notificationBuilder.On("NewBundle", id, notificationParas).Return(bundle, nil).Once()
   307  		bundle.On("CancelNotificationEvent").Return(nil).Once()
   308  
   309  		svc := manager.NewUpgradeKymaManager(store.Orchestrations(), store.Operations(), store.Instances(), &testExecutor{},
   310  			resolver, poolingInterval, logrus.New(), k8sClient, &orchestrationConfig, notificationBuilder, 1000)
   311  
   312  		// when
   313  		_, err = svc.Execute(id)
   314  		require.NoError(t, err)
   315  
   316  		o, err := store.Orchestrations().GetByID(id)
   317  		require.NoError(t, err)
   318  
   319  		assert.Equal(t, orchestration.Canceled, o.State)
   320  
   321  		op, err := store.Operations().GetUpgradeKymaOperationByID(id)
   322  		require.NoError(t, err)
   323  
   324  		assert.Equal(t, orchestration.Canceled, string(op.State))
   325  	})
   326  
   327  	t.Run("Retrying failed orchestration", func(t *testing.T) {
   328  		// given
   329  		store := storage.NewMemoryStorage()
   330  
   331  		resolver := &automock.RuntimeResolver{}
   332  		defer resolver.AssertExpectations(t)
   333  
   334  		resolver.On("Resolve", orchestration.TargetSpec{
   335  			Include: nil,
   336  			Exclude: nil,
   337  		}).Return([]orchestration.Runtime{}, nil)
   338  
   339  		id := "id"
   340  		opId := "op-" + id
   341  		err := store.Orchestrations().Insert(internal.Orchestration{
   342  			OrchestrationID: id,
   343  			State:           orchestration.Retrying,
   344  			Type:            orchestration.UpgradeKymaOrchestration,
   345  			Parameters: orchestration.Parameters{
   346  				Strategy: orchestration.StrategySpec{
   347  					Type:         orchestration.ParallelStrategy,
   348  					Schedule:     time.Now().Format(time.RFC3339),
   349  					Parallel:     orchestration.ParallelStrategySpec{Workers: 2},
   350  					ScheduleTime: time.Time{},
   351  				},
   352  				RetryOperation: orchestration.RetryOperationParameters{
   353  					RetryOperations: []string{"op-id"},
   354  				},
   355  				Notification: true,
   356  			},
   357  		})
   358  		require.NoError(t, err)
   359  
   360  		err = store.Operations().InsertUpgradeKymaOperation(internal.UpgradeKymaOperation{
   361  			Operation: internal.Operation{
   362  				ID:              opId,
   363  				OrchestrationID: id,
   364  				State:           orchestration.Retrying,
   365  				RuntimeOperation: orchestration.RuntimeOperation{
   366  					ID:      opId,
   367  					Runtime: orchestration.Runtime{},
   368  					DryRun:  false,
   369  				},
   370  				InputCreator: nil,
   371  			},
   372  		})
   373  		require.NoError(t, err)
   374  
   375  		notificationTenants := []notification.NotificationTenant{
   376  			{
   377  				StartDate: time.Now().Format("2006-01-02 15:04:05"),
   378  			},
   379  		}
   380  		notificationParas := notification.NotificationParams{
   381  			OrchestrationID: id,
   382  			EventType:       notification.KymaMaintenanceNumber,
   383  			Tenants:         notificationTenants,
   384  		}
   385  		notificationBuilder := &notificationAutomock.BundleBuilder{}
   386  		bundle := &notificationAutomock.Bundle{}
   387  		notificationBuilder.On("NewBundle", id, notificationParas).Return(bundle, nil).Once()
   388  		bundle.On("CreateNotificationEvent").Return(nil).Once()
   389  
   390  		executor := retryTestExecutor{
   391  			store:       store,
   392  			upgradeType: orchestration.UpgradeKymaOrchestration,
   393  		}
   394  		svc := manager.NewUpgradeKymaManager(store.Orchestrations(), store.Operations(), store.Instances(), &executor,
   395  			resolver, poolingInterval, logrus.New(), k8sClient, &orchestrationConfig, notificationBuilder, 1000)
   396  
   397  		// when
   398  		_, err = svc.Execute(id)
   399  		require.NoError(t, err)
   400  
   401  		o, err := store.Orchestrations().GetByID(id)
   402  		require.NoError(t, err)
   403  
   404  		assert.Equal(t, orchestration.Succeeded, o.State)
   405  
   406  		op, err := store.Operations().GetUpgradeKymaOperationByID(opId)
   407  		require.NoError(t, err)
   408  
   409  		assert.Equal(t, orchestration.Retrying, string(op.State))
   410  	})
   411  
   412  	t.Run("Retrying resumed in progress orchestration", func(t *testing.T) {
   413  		// given
   414  		store := storage.NewMemoryStorage()
   415  
   416  		resolver := &automock.RuntimeResolver{}
   417  		defer resolver.AssertExpectations(t)
   418  
   419  		id := "id"
   420  		opId := "op-" + id
   421  		err := store.Orchestrations().Insert(internal.Orchestration{
   422  			OrchestrationID: id,
   423  			State:           orchestration.InProgress,
   424  			Type:            orchestration.UpgradeKymaOrchestration,
   425  			Parameters: orchestration.Parameters{Strategy: orchestration.StrategySpec{
   426  				Type:         orchestration.ParallelStrategy,
   427  				Schedule:     time.Now().Format(time.RFC3339),
   428  				Parallel:     orchestration.ParallelStrategySpec{Workers: 2},
   429  				ScheduleTime: time.Time{},
   430  			},
   431  				Notification: true,
   432  			},
   433  		})
   434  		require.NoError(t, err)
   435  
   436  		err = store.Operations().InsertUpgradeKymaOperation(internal.UpgradeKymaOperation{
   437  			Operation: internal.Operation{
   438  				ID:              opId,
   439  				OrchestrationID: id,
   440  				State:           orchestration.Retrying,
   441  				RuntimeOperation: orchestration.RuntimeOperation{
   442  					ID:      opId,
   443  					Runtime: orchestration.Runtime{},
   444  					DryRun:  false,
   445  				},
   446  				InputCreator: nil,
   447  				Type:         internal.OperationTypeUpgradeKyma,
   448  			},
   449  		})
   450  		require.NoError(t, err)
   451  
   452  		notificationTenants := []notification.NotificationTenant{
   453  			{
   454  				StartDate: time.Now().Format("2006-01-02 15:04:05"),
   455  			},
   456  		}
   457  		notificationParas := notification.NotificationParams{
   458  			OrchestrationID: id,
   459  			EventType:       notification.KymaMaintenanceNumber,
   460  			Tenants:         notificationTenants,
   461  		}
   462  		notificationBuilder := &notificationAutomock.BundleBuilder{}
   463  		bundle := &notificationAutomock.Bundle{}
   464  		notificationBuilder.On("NewBundle", id, notificationParas).Return(bundle, nil).Once()
   465  		bundle.On("CreateNotificationEvent").Return(nil).Once()
   466  
   467  		executor := retryTestExecutor{
   468  			store:       store,
   469  			upgradeType: orchestration.UpgradeKymaOrchestration,
   470  		}
   471  		svc := manager.NewUpgradeKymaManager(store.Orchestrations(), store.Operations(), store.Instances(), &executor,
   472  			resolver, poolingInterval, logrus.New(), k8sClient, &orchestrationConfig, notificationBuilder, 1000)
   473  
   474  		// when
   475  		_, err = svc.Execute(id)
   476  		require.NoError(t, err)
   477  
   478  		o, err := store.Orchestrations().GetByID(id)
   479  		require.NoError(t, err)
   480  
   481  		assert.Equal(t, orchestration.Succeeded, o.State)
   482  
   483  		op, err := store.Operations().GetUpgradeKymaOperationByID(opId)
   484  		require.NoError(t, err)
   485  
   486  		assert.Equal(t, orchestration.Succeeded, string(op.State))
   487  	})
   488  
   489  	t.Run("Retrying failed orchestration with --schedule immediate and create a new operation on same instanceID", func(t *testing.T) {
   490  		// given
   491  		store := storage.NewMemoryStorage()
   492  
   493  		resolver := &automock.RuntimeResolver{}
   494  		defer resolver.AssertExpectations(t)
   495  
   496  		id := "id"
   497  		opId := "op-" + id
   498  		instanceID := opId + "-1234"
   499  		runtimeID := opId + "-5678"
   500  
   501  		resolver.On("Resolve", orchestration.TargetSpec{
   502  			Include: []orchestration.RuntimeTarget{
   503  				{RuntimeID: opId},
   504  			},
   505  			Exclude: nil,
   506  		}).Return([]orchestration.Runtime{{
   507  			InstanceID: instanceID,
   508  			RuntimeID:  runtimeID,
   509  		}}, nil)
   510  
   511  		err := store.Instances().Insert(internal.Instance{
   512  			InstanceID: instanceID,
   513  			RuntimeID:  runtimeID,
   514  		})
   515  		require.NoError(t, err)
   516  		err = store.Orchestrations().Insert(
   517  			internal.Orchestration{
   518  				OrchestrationID: id,
   519  				State:           orchestration.Retrying,
   520  				Type:            orchestration.UpgradeClusterOrchestration,
   521  				Parameters: orchestration.Parameters{
   522  					Strategy: orchestration.StrategySpec{
   523  						Type:         orchestration.ParallelStrategy,
   524  						Schedule:     time.Now().Format(time.RFC3339),
   525  						Parallel:     orchestration.ParallelStrategySpec{Workers: 2},
   526  						ScheduleTime: time.Time{},
   527  					},
   528  					Kyma: &orchestration.KymaParameters{Version: ""},
   529  					Targets: orchestration.TargetSpec{
   530  						Include: []orchestration.RuntimeTarget{
   531  							{RuntimeID: opId},
   532  						},
   533  						Exclude: nil,
   534  					},
   535  					RetryOperation: orchestration.RetryOperationParameters{
   536  						RetryOperations: []string{"op-id"},
   537  						Immediate:       true,
   538  					},
   539  					Notification: false,
   540  				},
   541  			})
   542  		require.NoError(t, err)
   543  
   544  		err = store.Operations().InsertUpgradeKymaOperation(internal.UpgradeKymaOperation{
   545  			Operation: internal.Operation{
   546  				ID:              opId,
   547  				OrchestrationID: id,
   548  				State:           orchestration.Failed,
   549  				InstanceID:      instanceID,
   550  				Type:            internal.OperationTypeUpgradeKyma,
   551  				RuntimeOperation: orchestration.RuntimeOperation{
   552  					ID: opId,
   553  					Runtime: orchestration.Runtime{
   554  						InstanceID: instanceID,
   555  						RuntimeID:  runtimeID},
   556  					DryRun: false,
   557  				},
   558  			},
   559  		})
   560  		require.NoError(t, err)
   561  
   562  		notificationBuilder := &notificationAutomock.BundleBuilder{}
   563  
   564  		executor := retryTestExecutor{
   565  			store:       store,
   566  			upgradeType: orchestration.UpgradeKymaOrchestration,
   567  		}
   568  		svc := manager.NewUpgradeKymaManager(store.Orchestrations(), store.Operations(), store.Instances(), &executor,
   569  			resolver, poolingInterval, logrus.New(), k8sClient, &orchestrationConfig, notificationBuilder, 1000)
   570  
   571  		// when
   572  		_, err = svc.Execute(id)
   573  		require.NoError(t, err)
   574  
   575  		o, err := store.Orchestrations().GetByID(id)
   576  		require.NoError(t, err)
   577  
   578  		assert.Equal(t, orchestration.Succeeded, o.State)
   579  
   580  		op, err := store.Operations().GetUpgradeKymaOperationByID(opId)
   581  		require.NoError(t, err)
   582  
   583  		assert.Equal(t, orchestration.Failed, string(op.State))
   584  
   585  		//verify a new operation with same instanceID is created
   586  		ops, _, _, err := store.Operations().ListUpgradeKymaOperationsByOrchestrationID(id, dbmodel.OperationFilter{})
   587  		require.NoError(t, err)
   588  		assert.Equal(t, 2, len(ops))
   589  
   590  		for _, op := range ops {
   591  			if op.Operation.ID != opId {
   592  				assert.Equal(t, op.Operation.RuntimeOperation.MaintenanceWindowBegin, time.Time{})
   593  				assert.Equal(t, op.Operation.RuntimeOperation.MaintenanceWindowEnd, time.Time{})
   594  
   595  				assert.Equal(t, orchestration.Succeeded, string(op.State))
   596  				assert.Equal(t, instanceID, string(op.Operation.InstanceID))
   597  				assert.Equal(t, internal.OperationTypeUpgradeKyma, op.Operation.Type)
   598  			}
   599  		}
   600  	})
   601  
   602  	t.Run("Retrying --now failed orchestration with `--schedule maintancewindow`  and create a new operation on same instanceID", func(t *testing.T) {
   603  		// given
   604  		store := storage.NewMemoryStorage()
   605  
   606  		resolver := &automock.RuntimeResolver{}
   607  		defer resolver.AssertExpectations(t)
   608  
   609  		id := "id"
   610  		opId := "op-" + id
   611  		instanceID := opId + "-1234"
   612  		runtimeID := opId + "-5678"
   613  
   614  		resolver.On("Resolve", orchestration.TargetSpec{
   615  			Include: []orchestration.RuntimeTarget{
   616  				{RuntimeID: opId},
   617  			},
   618  			Exclude: nil,
   619  		}).Return([]orchestration.Runtime{{
   620  			InstanceID: instanceID,
   621  			RuntimeID:  runtimeID,
   622  		}}, nil)
   623  
   624  		err := store.Instances().Insert(internal.Instance{
   625  			InstanceID: instanceID,
   626  			RuntimeID:  runtimeID,
   627  		})
   628  		require.NoError(t, err)
   629  		err = store.Orchestrations().Insert(
   630  			internal.Orchestration{
   631  				OrchestrationID: id,
   632  				State:           orchestration.Retrying,
   633  				Type:            orchestration.UpgradeClusterOrchestration,
   634  				Parameters: orchestration.Parameters{
   635  					Strategy: orchestration.StrategySpec{
   636  						Type:         orchestration.ParallelStrategy,
   637  						Schedule:     time.Now().Format(time.RFC3339),
   638  						Parallel:     orchestration.ParallelStrategySpec{Workers: 2},
   639  						ScheduleTime: time.Time{},
   640  					},
   641  					Kyma: &orchestration.KymaParameters{Version: ""},
   642  					Targets: orchestration.TargetSpec{
   643  						Include: []orchestration.RuntimeTarget{
   644  							{RuntimeID: opId},
   645  						},
   646  						Exclude: nil,
   647  					},
   648  					RetryOperation: orchestration.RetryOperationParameters{
   649  						RetryOperations: []string{"op-id"},
   650  						Immediate:       true,
   651  					},
   652  					Notification: false,
   653  				},
   654  			})
   655  		require.NoError(t, err)
   656  
   657  		err = store.Operations().InsertUpgradeKymaOperation(internal.UpgradeKymaOperation{
   658  			Operation: internal.Operation{
   659  				ID:              opId,
   660  				OrchestrationID: id,
   661  				State:           orchestration.Failed,
   662  				InstanceID:      instanceID,
   663  				Type:            internal.OperationTypeUpgradeKyma,
   664  				RuntimeOperation: orchestration.RuntimeOperation{
   665  					ID: opId,
   666  					Runtime: orchestration.Runtime{
   667  						InstanceID: instanceID,
   668  						RuntimeID:  runtimeID},
   669  					DryRun: false,
   670  				},
   671  			},
   672  		})
   673  		require.NoError(t, err)
   674  
   675  		notificationBuilder := &notificationAutomock.BundleBuilder{}
   676  
   677  		executor := retryTestExecutor{
   678  			store:       store,
   679  			upgradeType: orchestration.UpgradeKymaOrchestration,
   680  		}
   681  		svc := manager.NewUpgradeKymaManager(store.Orchestrations(), store.Operations(), store.Instances(), &executor,
   682  			resolver, poolingInterval, logrus.New(), k8sClient, &orchestrationConfig, notificationBuilder, 1000)
   683  
   684  		// when
   685  		_, err = svc.Execute(id)
   686  		require.NoError(t, err)
   687  
   688  		o, err := store.Orchestrations().GetByID(id)
   689  		require.NoError(t, err)
   690  
   691  		assert.Equal(t, orchestration.Succeeded, o.State)
   692  
   693  		op, err := store.Operations().GetUpgradeKymaOperationByID(opId)
   694  		require.NoError(t, err)
   695  
   696  		assert.Equal(t, orchestration.Failed, string(op.State))
   697  
   698  		//verify a new operation with same instanceID is created
   699  		ops, _, _, err := store.Operations().ListUpgradeKymaOperationsByOrchestrationID(id, dbmodel.OperationFilter{})
   700  		require.NoError(t, err)
   701  		assert.Equal(t, 2, len(ops))
   702  
   703  		for _, op := range ops {
   704  			if op.Operation.ID != opId {
   705  				assert.Equal(t, op.Operation.RuntimeOperation.MaintenanceWindowBegin, time.Time{})
   706  				assert.Equal(t, op.Operation.RuntimeOperation.MaintenanceWindowEnd, time.Time{})
   707  				assert.Equal(t, orchestration.Succeeded, string(op.State))
   708  				assert.Equal(t, instanceID, string(op.Operation.InstanceID))
   709  				assert.Equal(t, internal.OperationTypeUpgradeKyma, op.Operation.Type)
   710  			}
   711  		}
   712  	})
   713  
   714  	t.Run("Retrying failed orchestration with `--schedule maintancewindow` and without `--now`  and create a new operation on same instanceID", func(t *testing.T) {
   715  		// given
   716  		store := storage.NewMemoryStorage()
   717  
   718  		resolver := &automock.RuntimeResolver{}
   719  		defer resolver.AssertExpectations(t)
   720  
   721  		id := "id"
   722  		opId := "op-" + id
   723  		instanceID := opId + "-1234"
   724  		runtimeID := opId + "-5678"
   725  
   726  		resolver.On("Resolve", orchestration.TargetSpec{
   727  			Include: []orchestration.RuntimeTarget{
   728  				{RuntimeID: opId},
   729  			},
   730  			Exclude: nil,
   731  		}).Return([]orchestration.Runtime{{
   732  			InstanceID: instanceID,
   733  			RuntimeID:  runtimeID,
   734  		}}, nil)
   735  
   736  		err := store.Instances().Insert(internal.Instance{
   737  			InstanceID: instanceID,
   738  			RuntimeID:  runtimeID,
   739  		})
   740  		require.NoError(t, err)
   741  		err = store.Orchestrations().Insert(
   742  			internal.Orchestration{
   743  				OrchestrationID: id,
   744  				State:           orchestration.Retrying,
   745  				Type:            orchestration.UpgradeClusterOrchestration,
   746  				Parameters: orchestration.Parameters{
   747  					Strategy: orchestration.StrategySpec{
   748  						Type:         orchestration.ParallelStrategy,
   749  						Schedule:     time.Now().Format(time.RFC3339),
   750  						Parallel:     orchestration.ParallelStrategySpec{Workers: 2},
   751  						ScheduleTime: time.Time{},
   752  					},
   753  					Kyma: &orchestration.KymaParameters{Version: ""},
   754  					Targets: orchestration.TargetSpec{
   755  						Include: []orchestration.RuntimeTarget{
   756  							{RuntimeID: opId},
   757  						},
   758  						Exclude: nil,
   759  					},
   760  					RetryOperation: orchestration.RetryOperationParameters{
   761  						RetryOperations: []string{"op-id"},
   762  						Immediate:       false,
   763  					},
   764  					Notification: false,
   765  				},
   766  			})
   767  		require.NoError(t, err)
   768  
   769  		err = store.Operations().InsertUpgradeKymaOperation(internal.UpgradeKymaOperation{
   770  			Operation: internal.Operation{
   771  				ID:              opId,
   772  				OrchestrationID: id,
   773  				State:           orchestration.Failed,
   774  				InstanceID:      instanceID,
   775  				Type:            internal.OperationTypeUpgradeKyma,
   776  				RuntimeOperation: orchestration.RuntimeOperation{
   777  					ID: opId,
   778  					Runtime: orchestration.Runtime{
   779  						InstanceID: instanceID,
   780  						RuntimeID:  runtimeID},
   781  					DryRun: false,
   782  				},
   783  			},
   784  		})
   785  		require.NoError(t, err)
   786  
   787  		notificationBuilder := &notificationAutomock.BundleBuilder{}
   788  
   789  		executor := retryTestExecutor{
   790  			store:       store,
   791  			upgradeType: orchestration.UpgradeKymaOrchestration,
   792  		}
   793  		svc := manager.NewUpgradeKymaManager(store.Orchestrations(), store.Operations(), store.Instances(), &executor,
   794  			resolver, poolingInterval, logrus.New(), k8sClient, &orchestrationConfig, notificationBuilder, 1000)
   795  
   796  		// when
   797  		_, err = svc.Execute(id)
   798  		require.NoError(t, err)
   799  
   800  		o, err := store.Orchestrations().GetByID(id)
   801  		require.NoError(t, err)
   802  
   803  		assert.Equal(t, orchestration.Succeeded, o.State)
   804  
   805  		op, err := store.Operations().GetUpgradeKymaOperationByID(opId)
   806  		require.NoError(t, err)
   807  
   808  		assert.Equal(t, orchestration.Failed, string(op.State))
   809  
   810  		//verify a new operation with same instanceID is created
   811  		ops, _, _, err := store.Operations().ListUpgradeKymaOperationsByOrchestrationID(id, dbmodel.OperationFilter{})
   812  		require.NoError(t, err)
   813  		assert.Equal(t, 2, len(ops))
   814  
   815  		for _, op := range ops {
   816  			if op.Operation.ID != opId {
   817  				assert.Equal(t, op.Operation.RuntimeOperation.MaintenanceWindowBegin, time.Time{})
   818  				assert.Equal(t, op.Operation.RuntimeOperation.MaintenanceWindowEnd, time.Time{})
   819  				assert.Equal(t, orchestration.Succeeded, string(op.State))
   820  				assert.Equal(t, instanceID, string(op.Operation.InstanceID))
   821  				assert.Equal(t, internal.OperationTypeUpgradeKyma, op.Operation.Type)
   822  			}
   823  		}
   824  	})
   825  }
   826  
   827  type testExecutor struct{}
   828  
   829  func (t *testExecutor) Execute(opID string) (time.Duration, error) {
   830  	return 0, nil
   831  }
   832  
   833  func (t *testExecutor) Reschedule(operationID string, maintenanceWindowBegin, maintenanceWindowEnd time.Time) error {
   834  	return nil
   835  }
   836  
   837  type retryTestExecutor struct {
   838  	store       storage.BrokerStorage
   839  	upgradeType orchestration.Type
   840  }
   841  
   842  func (t *retryTestExecutor) Execute(opID string) (time.Duration, error) {
   843  	switch t.upgradeType {
   844  	case orchestration.UpgradeKymaOrchestration:
   845  		op, err := t.store.Operations().GetUpgradeKymaOperationByID(opID)
   846  		if err != nil {
   847  			return 0, err
   848  		}
   849  		op.State = orchestration.Succeeded
   850  		_, err = t.store.Operations().UpdateUpgradeKymaOperation(*op)
   851  
   852  		return 0, err
   853  	case orchestration.UpgradeClusterOrchestration:
   854  		op, err := t.store.Operations().GetUpgradeClusterOperationByID(opID)
   855  		if err != nil {
   856  			return 0, err
   857  		}
   858  		op.State = orchestration.Succeeded
   859  		_, err = t.store.Operations().UpdateUpgradeClusterOperation(*op)
   860  
   861  		return 0, err
   862  	}
   863  
   864  	return 0, fmt.Errorf("unknown upgrade type")
   865  }
   866  
   867  func (t *retryTestExecutor) Reschedule(operationID string, maintenanceWindowBegin, maintenanceWindowEnd time.Time) error {
   868  	return nil
   869  }