github.com/kyma-project/kyma-environment-broker@v0.0.1/internal/process/upgrade_cluster/initialisation_test.go (about)

     1  package upgrade_cluster
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"testing"
     7  	"time"
     8  
     9  	"github.com/kyma-project/control-plane/components/provisioner/pkg/gqlschema"
    10  	"github.com/kyma-project/kyma-environment-broker/internal/fixture"
    11  	"github.com/kyma-project/kyma-environment-broker/internal/notification"
    12  	notificationAutomock "github.com/kyma-project/kyma-environment-broker/internal/notification/mocks"
    13  	"github.com/kyma-project/kyma-environment-broker/internal/process/input"
    14  	"github.com/kyma-project/kyma-environment-broker/internal/process/upgrade_kyma/automock"
    15  	cloudProvider "github.com/kyma-project/kyma-environment-broker/internal/provider"
    16  	provisionerAutomock "github.com/kyma-project/kyma-environment-broker/internal/provisioner/automock"
    17  	"github.com/kyma-project/kyma-environment-broker/internal/ptr"
    18  	"github.com/pivotal-cf/brokerapi/v8/domain"
    19  	"github.com/stretchr/testify/assert"
    20  	"github.com/stretchr/testify/mock"
    21  
    22  	"github.com/kyma-project/kyma-environment-broker/internal/avs"
    23  
    24  	"github.com/stretchr/testify/require"
    25  
    26  	"github.com/kyma-project/kyma-environment-broker/common/orchestration"
    27  	"github.com/kyma-project/kyma-environment-broker/internal"
    28  	"github.com/kyma-project/kyma-environment-broker/internal/broker"
    29  	"github.com/kyma-project/kyma-environment-broker/internal/storage"
    30  	"github.com/sirupsen/logrus"
    31  )
    32  
    33  const (
    34  	fixProvisioningOperationID                      = "17f3ddba-1132-466d-a3c5-920f544d7ea6"
    35  	fixOrchestrationID                              = "fd5cee4d-0eeb-40d0-a7a7-0708eseba470"
    36  	fixUpgradeOperationID                           = "fd5cee4d-0eeb-40d0-a7a7-0708e5eba470"
    37  	fixInstanceID                                   = "9d75a545-2e1e-4786-abd8-a37b14e185b9"
    38  	fixRuntimeID                                    = "ef4e3210-652c-453e-8015-bba1c1cd1e1c"
    39  	fixGlobalAccountID                              = "abf73c71-a653-4951-b9c2-a26d6c2cccbd"
    40  	fixSubAccountID                                 = "6424cc6d-5fce-49fc-b720-cf1fc1f36c7d"
    41  	fixProvisionerOperationID                       = "e04de524-53b3-4890-b05a-296be393e4ba"
    42  	fixMaintenanceModeAlwaysDisabledGlobalAccountID = "maintenance-mode-always-disabled-ga-1"
    43  )
    44  
    45  type fixHyperscalerInputProvider interface {
    46  	Defaults() *gqlschema.ClusterConfigInput
    47  }
    48  
    49  func createMonitors(t *testing.T, client *avs.Client, internalStatus string, externalStatus string) internal.AvsLifecycleData {
    50  	// monitors
    51  	var (
    52  		operationInternalId int64
    53  		operationExternalId int64
    54  	)
    55  
    56  	// internal
    57  	inMonitor, err := client.CreateEvaluation(&avs.BasicEvaluationCreateRequest{
    58  		Name: "internal monitor",
    59  	})
    60  	require.NoError(t, err)
    61  	operationInternalId = inMonitor.Id
    62  
    63  	if avs.ValidStatus(internalStatus) {
    64  		_, err = client.SetStatus(inMonitor.Id, internalStatus)
    65  		require.NoError(t, err)
    66  	}
    67  
    68  	// external
    69  	exMonitor, err := client.CreateEvaluation(&avs.BasicEvaluationCreateRequest{
    70  		Name: "external monitor",
    71  	})
    72  	require.NoError(t, err)
    73  	operationExternalId = exMonitor.Id
    74  
    75  	if avs.ValidStatus(externalStatus) {
    76  		_, err = client.SetStatus(exMonitor.Id, externalStatus)
    77  		require.NoError(t, err)
    78  	}
    79  
    80  	// return AvsLifecycleData
    81  	avsData := internal.AvsLifecycleData{
    82  		AvsEvaluationInternalId: operationInternalId,
    83  		AVSEvaluationExternalId: operationExternalId,
    84  		AvsInternalEvaluationStatus: internal.AvsEvaluationStatus{
    85  			Current:  internalStatus,
    86  			Original: "",
    87  		},
    88  		AvsExternalEvaluationStatus: internal.AvsEvaluationStatus{
    89  			Current:  externalStatus,
    90  			Original: "",
    91  		},
    92  		AVSInternalEvaluationDeleted: false,
    93  		AVSExternalEvaluationDeleted: false,
    94  	}
    95  
    96  	return avsData
    97  }
    98  
    99  func createEvalManagerWithValidity(t *testing.T, storage storage.BrokerStorage, log *logrus.Logger, valid bool) (*avs.EvaluationManager, *avs.Client) {
   100  	server := avs.NewMockAvsServer(t)
   101  	mockServer := avs.FixMockAvsServer(server)
   102  	client, err := avs.NewClient(context.TODO(), avs.Config{
   103  		OauthTokenEndpoint: fmt.Sprintf("%s/oauth/token", mockServer.URL),
   104  		ApiEndpoint:        fmt.Sprintf("%s/api/v2/evaluationmetadata", mockServer.URL),
   105  	}, logrus.New())
   106  	require.NoError(t, err)
   107  
   108  	if !valid {
   109  		client, err = avs.NewClient(context.TODO(), avs.Config{}, logrus.New())
   110  	}
   111  	require.NoError(t, err)
   112  
   113  	avsDel := avs.NewDelegator(client, avs.Config{}, storage.Operations())
   114  	upgradeEvalManager := avs.NewEvaluationManager(avsDel, avs.Config{})
   115  
   116  	return upgradeEvalManager, client
   117  }
   118  
   119  func createEvalManager(t *testing.T, storage storage.BrokerStorage, log *logrus.Logger) (*avs.EvaluationManager, *avs.Client) {
   120  	return createEvalManagerWithValidity(t, storage, log, true)
   121  }
   122  
   123  func createEvalManagerWithMaintenanceModeConfig(t *testing.T, storage storage.BrokerStorage, maintenanceModeDisabled bool,
   124  	maintenanceModeAlwaysDisabledGAIDs []string) (*avs.EvaluationManager, *avs.Client) {
   125  	server := avs.NewMockAvsServer(t)
   126  	mockServer := avs.FixMockAvsServer(server)
   127  	client, err := avs.NewClient(context.TODO(), avs.Config{
   128  		OauthTokenEndpoint: fmt.Sprintf("%s/oauth/token", mockServer.URL),
   129  		ApiEndpoint:        fmt.Sprintf("%s/api/v2/evaluationmetadata", mockServer.URL),
   130  	}, logrus.New())
   131  	require.NoError(t, err)
   132  
   133  	avsDel := avs.NewDelegator(client, avs.Config{}, storage.Operations())
   134  	upgradeEvalManager := avs.NewEvaluationManager(avsDel, avs.Config{
   135  		MaintenanceModeDuringUpgradeDisabled:            maintenanceModeDisabled,
   136  		MaintenanceModeDuringUpgradeAlwaysDisabledGAIDs: maintenanceModeAlwaysDisabledGAIDs,
   137  	})
   138  
   139  	return upgradeEvalManager, client
   140  }
   141  
   142  func TestInitialisationStep_Run(t *testing.T) {
   143  	t.Run("should mark operation as Succeeded when upgrade was successful", func(t *testing.T) {
   144  		// given
   145  		log := logrus.New()
   146  		memoryStorage := storage.NewMemoryStorage()
   147  		evalManager, _ := createEvalManager(t, memoryStorage, log)
   148  
   149  		orch := internal.Orchestration{
   150  			OrchestrationID: fixOrchestrationID,
   151  			State:           orchestration.InProgress,
   152  			Parameters: orchestration.Parameters{
   153  				Kyma: &orchestration.KymaParameters{
   154  					Version: fixKymaVersion,
   155  				},
   156  				Notification: true,
   157  			},
   158  		}
   159  		err := memoryStorage.Orchestrations().Insert(orch)
   160  		require.NoError(t, err)
   161  
   162  		provisioningOperation := fixProvisioningOperation()
   163  		err = memoryStorage.Operations().InsertOperation(provisioningOperation)
   164  		require.NoError(t, err)
   165  
   166  		upgradeOperation := fixUpgradeClusterOperation()
   167  		err = memoryStorage.Operations().InsertUpgradeClusterOperation(upgradeOperation)
   168  		require.NoError(t, err)
   169  
   170  		instance := fixInstanceRuntimeStatus()
   171  		err = memoryStorage.Instances().Insert(instance)
   172  		require.NoError(t, err)
   173  
   174  		provisionerClient := &provisionerAutomock.Client{}
   175  		provisionerClient.On("RuntimeOperationStatus", fixGlobalAccountID, fixProvisionerOperationID).Return(gqlschema.OperationStatus{
   176  			ID:        ptr.String(fixProvisionerOperationID),
   177  			Operation: "",
   178  			State:     gqlschema.OperationStateSucceeded,
   179  			Message:   nil,
   180  			RuntimeID: StringPtr(fixRuntimeID),
   181  		}, nil)
   182  
   183  		notificationTenants := []notification.NotificationTenant{
   184  			{
   185  				InstanceID: fixInstanceID,
   186  				State:      notification.FinishedMaintenanceState,
   187  				EndDate:    time.Now().Format("2006-01-02 15:04:05"),
   188  			},
   189  		}
   190  		notificationParas := notification.NotificationParams{
   191  			OrchestrationID: fixOrchestrationID,
   192  			Tenants:         notificationTenants,
   193  		}
   194  		notificationBuilder := &notificationAutomock.BundleBuilder{}
   195  		bundle := &notificationAutomock.Bundle{}
   196  		notificationBuilder.On("NewBundle", fixOrchestrationID, notificationParas).Return(bundle, nil).Once()
   197  		bundle.On("UpdateNotificationEvent").Return(nil).Once()
   198  
   199  		step := NewInitialisationStep(memoryStorage.Operations(), memoryStorage.Orchestrations(), provisionerClient,
   200  			nil, evalManager, nil, notificationBuilder)
   201  
   202  		// when
   203  		upgradeOperation, repeat, err := step.Run(upgradeOperation, log)
   204  
   205  		// then
   206  		assert.NoError(t, err)
   207  		assert.Equal(t, time.Duration(0), repeat)
   208  		assert.Equal(t, domain.Succeeded, upgradeOperation.State)
   209  
   210  		storedOp, err := memoryStorage.Operations().GetUpgradeClusterOperationByID(upgradeOperation.Operation.ID)
   211  		assert.Equal(t, upgradeOperation, *storedOp)
   212  		assert.NoError(t, err)
   213  
   214  	})
   215  
   216  	t.Run("should initialize UpgradeRuntimeInput request when run", func(t *testing.T) {
   217  		// given
   218  		log := logrus.New()
   219  		memoryStorage := storage.NewMemoryStorage()
   220  		evalManager, _ := createEvalManager(t, memoryStorage, log)
   221  
   222  		err := memoryStorage.Orchestrations().Insert(fixOrchestrationWithKymaVer())
   223  		require.NoError(t, err)
   224  
   225  		provisioningOperation := fixProvisioningOperation()
   226  		err = memoryStorage.Operations().InsertOperation(provisioningOperation)
   227  		require.NoError(t, err)
   228  
   229  		upgradeOperation := fixUpgradeClusterOperation()
   230  		upgradeOperation.ProvisionerOperationID = ""
   231  		err = memoryStorage.Operations().InsertUpgradeClusterOperation(upgradeOperation)
   232  		require.NoError(t, err)
   233  
   234  		instance := fixInstanceRuntimeStatus()
   235  		err = memoryStorage.Instances().Insert(instance)
   236  		require.NoError(t, err)
   237  
   238  		provisionerClient := &provisionerAutomock.Client{}
   239  		inputBuilder := &automock.CreatorForPlan{}
   240  		inputBuilder.On("CreateUpgradeShootInput",
   241  			fixProvisioningParameters(), mock.AnythingOfType("internal.RuntimeVersionData")).
   242  			Return(&input.RuntimeInput{},
   243  				nil)
   244  
   245  		expectedOperation := upgradeOperation
   246  		expectedOperation.Version++
   247  		expectedOperation.State = orchestration.InProgress
   248  
   249  		notificationTenants := []notification.NotificationTenant{
   250  			{
   251  				InstanceID: fixInstanceID,
   252  				State:      notification.FinishedMaintenanceState,
   253  				EndDate:    time.Now().Format("2006-01-02 15:04:05"),
   254  			},
   255  		}
   256  		notificationParas := notification.NotificationParams{
   257  			OrchestrationID: fixOrchestrationID,
   258  			Tenants:         notificationTenants,
   259  		}
   260  		notificationBuilder := &notificationAutomock.BundleBuilder{}
   261  		bundle := &notificationAutomock.Bundle{}
   262  		notificationBuilder.On("NewBundle", fixOrchestrationID, notificationParas).Return(bundle, nil).Once()
   263  		bundle.On("UpdateNotificationEvent").Return(nil).Once()
   264  
   265  		step := NewInitialisationStep(memoryStorage.Operations(), memoryStorage.Orchestrations(), provisionerClient, inputBuilder, evalManager, nil, notificationBuilder)
   266  
   267  		// when
   268  		op, repeat, err := step.Run(upgradeOperation, log)
   269  
   270  		// then
   271  		assert.NoError(t, err)
   272  		inputBuilder.AssertNumberOfCalls(t, "CreateUpgradeShootInput", 1)
   273  		assert.Equal(t, time.Duration(0), repeat)
   274  		assert.NotNil(t, op.InputCreator)
   275  
   276  		storedOp, err := memoryStorage.Operations().GetUpgradeClusterOperationByID(op.Operation.ID)
   277  		op.InputCreator = nil
   278  		assert.Equal(t, op, *storedOp)
   279  		assert.NoError(t, err)
   280  	})
   281  
   282  	t.Run("should mark finish if orchestration was canceled", func(t *testing.T) {
   283  		// given
   284  		log := logrus.New()
   285  		memoryStorage := storage.NewMemoryStorage()
   286  		evalManager, _ := createEvalManager(t, memoryStorage, log)
   287  
   288  		err := memoryStorage.Orchestrations().Insert(internal.Orchestration{
   289  			OrchestrationID: fixOrchestrationID,
   290  			State:           orchestration.Canceled,
   291  			Parameters: orchestration.Parameters{
   292  				Notification: true,
   293  			},
   294  		})
   295  		require.NoError(t, err)
   296  
   297  		upgradeOperation := fixUpgradeClusterOperation()
   298  		err = memoryStorage.Operations().InsertUpgradeClusterOperation(upgradeOperation)
   299  		require.NoError(t, err)
   300  
   301  		provisioningOperation := fixProvisioningOperation()
   302  		err = memoryStorage.Operations().InsertOperation(provisioningOperation)
   303  		require.NoError(t, err)
   304  
   305  		notificationTenants := []notification.NotificationTenant{
   306  			{
   307  				InstanceID: fixInstanceID,
   308  				State:      notification.FinishedMaintenanceState,
   309  				EndDate:    time.Now().Format("2006-01-02 15:04:05"),
   310  			},
   311  		}
   312  		notificationParas := notification.NotificationParams{
   313  			OrchestrationID: fixOrchestrationID,
   314  			Tenants:         notificationTenants,
   315  		}
   316  		notificationBuilder := &notificationAutomock.BundleBuilder{}
   317  		bundle := &notificationAutomock.Bundle{}
   318  		notificationBuilder.On("NewBundle", fixOrchestrationID, notificationParas).Return(bundle, nil).Once()
   319  		bundle.On("UpdateNotificationEvent").Return(nil).Once()
   320  
   321  		step := NewInitialisationStep(memoryStorage.Operations(), memoryStorage.Orchestrations(), nil, nil, evalManager, nil, notificationBuilder)
   322  
   323  		// when
   324  		upgradeOperation, repeat, err := step.Run(upgradeOperation, log)
   325  
   326  		// then
   327  		require.NoError(t, err)
   328  		assert.Equal(t, time.Duration(0), repeat)
   329  		assert.Equal(t, orchestration.Canceled, string(upgradeOperation.State))
   330  
   331  		storedOp, err := memoryStorage.Operations().GetUpgradeClusterOperationByID(upgradeOperation.Operation.ID)
   332  		require.NoError(t, err)
   333  		assert.Equal(t, upgradeOperation, *storedOp)
   334  	})
   335  
   336  	t.Run("should refresh avs on success (both monitors, empty init)", func(t *testing.T) {
   337  		// given
   338  		log := logrus.New()
   339  		memoryStorage := storage.NewMemoryStorage()
   340  		evalManager, client := createEvalManager(t, memoryStorage, log)
   341  		inputBuilder := &automock.CreatorForPlan{}
   342  
   343  		err := memoryStorage.Orchestrations().Insert(fixOrchestrationWithKymaVer())
   344  		require.NoError(t, err)
   345  
   346  		provisioningOperation := fixProvisioningOperation()
   347  		err = memoryStorage.Operations().InsertOperation(provisioningOperation)
   348  		require.NoError(t, err)
   349  
   350  		avsData := createMonitors(t, client, "", "")
   351  		upgradeOperation := fixUpgradeClusterOperationWithAvs(avsData)
   352  
   353  		err = memoryStorage.Operations().InsertUpgradeClusterOperation(upgradeOperation)
   354  		require.NoError(t, err)
   355  
   356  		instance := fixInstanceRuntimeStatus()
   357  		err = memoryStorage.Instances().Insert(instance)
   358  		require.NoError(t, err)
   359  
   360  		provisionerClient := &provisionerAutomock.Client{}
   361  		provisionerClient.On("RuntimeOperationStatus", fixGlobalAccountID, fixProvisionerOperationID).Return(gqlschema.OperationStatus{
   362  			ID:        ptr.String(fixProvisionerOperationID),
   363  			Operation: "",
   364  			State:     gqlschema.OperationStateSucceeded,
   365  			Message:   nil,
   366  			RuntimeID: StringPtr(fixRuntimeID),
   367  		}, nil)
   368  
   369  		notificationTenants := []notification.NotificationTenant{
   370  			{
   371  				InstanceID: fixInstanceID,
   372  				State:      notification.FinishedMaintenanceState,
   373  				EndDate:    time.Now().Format("2006-01-02 15:04:05"),
   374  			},
   375  		}
   376  		notificationParas := notification.NotificationParams{
   377  			OrchestrationID: fixOrchestrationID,
   378  			Tenants:         notificationTenants,
   379  		}
   380  		notificationBuilder := &notificationAutomock.BundleBuilder{}
   381  		bundle := &notificationAutomock.Bundle{}
   382  		notificationBuilder.On("NewBundle", fixOrchestrationID, notificationParas).Return(bundle, nil).Once()
   383  		bundle.On("UpdateNotificationEvent").Return(nil).Once()
   384  
   385  		step := NewInitialisationStep(memoryStorage.Operations(), memoryStorage.Orchestrations(), provisionerClient, inputBuilder, evalManager, nil, notificationBuilder)
   386  
   387  		// when
   388  		upgradeOperation, repeat, err := step.Run(upgradeOperation, log)
   389  
   390  		// then
   391  		assert.NoError(t, err)
   392  		assert.Equal(t, time.Duration(0), repeat)
   393  		assert.Equal(t, domain.Succeeded, upgradeOperation.State)
   394  		assert.Equal(t, upgradeOperation.Avs.AvsInternalEvaluationStatus, internal.AvsEvaluationStatus{Current: avs.StatusActive, Original: avs.StatusMaintenance})
   395  		assert.Equal(t, upgradeOperation.Avs.AvsExternalEvaluationStatus, internal.AvsEvaluationStatus{Current: avs.StatusActive, Original: avs.StatusMaintenance})
   396  
   397  		storedOp, err := memoryStorage.Operations().GetUpgradeClusterOperationByID(upgradeOperation.Operation.ID)
   398  		assert.Equal(t, upgradeOperation, *storedOp)
   399  		assert.NoError(t, err)
   400  	})
   401  
   402  	t.Run("should refresh avs on success (both monitors)", func(t *testing.T) {
   403  		// given
   404  		log := logrus.New()
   405  		memoryStorage := storage.NewMemoryStorage()
   406  		evalManager, client := createEvalManager(t, memoryStorage, log)
   407  		inputBuilder := &automock.CreatorForPlan{}
   408  
   409  		err := memoryStorage.Orchestrations().Insert(fixOrchestrationWithKymaVer())
   410  		require.NoError(t, err)
   411  
   412  		provisioningOperation := fixProvisioningOperation()
   413  		err = memoryStorage.Operations().InsertOperation(provisioningOperation)
   414  		require.NoError(t, err)
   415  
   416  		internalStatus, externalStatus := avs.StatusActive, avs.StatusInactive
   417  		avsData := createMonitors(t, client, internalStatus, externalStatus)
   418  		upgradeOperation := fixUpgradeClusterOperationWithAvs(avsData)
   419  
   420  		err = memoryStorage.Operations().InsertUpgradeClusterOperation(upgradeOperation)
   421  		require.NoError(t, err)
   422  
   423  		instance := fixInstanceRuntimeStatus()
   424  		err = memoryStorage.Instances().Insert(instance)
   425  		require.NoError(t, err)
   426  
   427  		provisionerClient := &provisionerAutomock.Client{}
   428  		provisionerClient.On("RuntimeOperationStatus", fixGlobalAccountID, fixProvisionerOperationID).Return(gqlschema.OperationStatus{
   429  			ID:        ptr.String(fixProvisionerOperationID),
   430  			Operation: "",
   431  			State:     gqlschema.OperationStateSucceeded,
   432  			Message:   nil,
   433  			RuntimeID: StringPtr(fixRuntimeID),
   434  		}, nil)
   435  
   436  		notificationTenants := []notification.NotificationTenant{
   437  			{
   438  				InstanceID: fixInstanceID,
   439  				State:      notification.FinishedMaintenanceState,
   440  				EndDate:    time.Now().Format("2006-01-02 15:04:05"),
   441  			},
   442  		}
   443  		notificationParas := notification.NotificationParams{
   444  			OrchestrationID: fixOrchestrationID,
   445  			Tenants:         notificationTenants,
   446  		}
   447  		notificationBuilder := &notificationAutomock.BundleBuilder{}
   448  		bundle := &notificationAutomock.Bundle{}
   449  		notificationBuilder.On("NewBundle", fixOrchestrationID, notificationParas).Return(bundle, nil).Once()
   450  		bundle.On("UpdateNotificationEvent").Return(nil).Once()
   451  
   452  		step := NewInitialisationStep(memoryStorage.Operations(), memoryStorage.Orchestrations(), provisionerClient, inputBuilder, evalManager, nil, notificationBuilder)
   453  
   454  		// when
   455  		upgradeOperation, repeat, err := step.Run(upgradeOperation, log)
   456  
   457  		// then
   458  		assert.NoError(t, err)
   459  		assert.Equal(t, time.Duration(0), repeat)
   460  		assert.Equal(t, domain.Succeeded, upgradeOperation.State)
   461  		assert.Equal(t, upgradeOperation.Avs.AvsInternalEvaluationStatus, internal.AvsEvaluationStatus{Current: internalStatus, Original: avs.StatusMaintenance})
   462  		assert.Equal(t, upgradeOperation.Avs.AvsExternalEvaluationStatus, internal.AvsEvaluationStatus{Current: externalStatus, Original: avs.StatusMaintenance})
   463  
   464  		storedOp, err := memoryStorage.Operations().GetUpgradeClusterOperationByID(upgradeOperation.Operation.ID)
   465  		assert.Equal(t, upgradeOperation, *storedOp)
   466  		assert.NoError(t, err)
   467  	})
   468  
   469  	t.Run("should refresh avs on fail (both monitors)", func(t *testing.T) {
   470  		// given
   471  		log := logrus.New()
   472  		memoryStorage := storage.NewMemoryStorage()
   473  		evalManager, client := createEvalManager(t, memoryStorage, log)
   474  		inputBuilder := &automock.CreatorForPlan{}
   475  
   476  		err := memoryStorage.Orchestrations().Insert(fixOrchestrationWithKymaVer())
   477  		require.NoError(t, err)
   478  
   479  		provisioningOperation := fixProvisioningOperation()
   480  		err = memoryStorage.Operations().InsertOperation(provisioningOperation)
   481  		require.NoError(t, err)
   482  
   483  		internalStatus, externalStatus := avs.StatusActive, avs.StatusInactive
   484  		avsData := createMonitors(t, client, internalStatus, externalStatus)
   485  		upgradeOperation := fixUpgradeClusterOperationWithAvs(avsData)
   486  
   487  		err = memoryStorage.Operations().InsertUpgradeClusterOperation(upgradeOperation)
   488  		require.NoError(t, err)
   489  
   490  		instance := fixInstanceRuntimeStatus()
   491  		err = memoryStorage.Instances().Insert(instance)
   492  		require.NoError(t, err)
   493  
   494  		provisionerClient := &provisionerAutomock.Client{}
   495  		provisionerClient.On("RuntimeOperationStatus", fixGlobalAccountID, fixProvisionerOperationID).Return(gqlschema.OperationStatus{
   496  			ID:        ptr.String(fixProvisionerOperationID),
   497  			Operation: "",
   498  			State:     gqlschema.OperationStateFailed,
   499  			Message:   nil,
   500  			RuntimeID: StringPtr(fixRuntimeID),
   501  		}, nil)
   502  
   503  		notificationTenants := []notification.NotificationTenant{
   504  			{
   505  				InstanceID: fixInstanceID,
   506  				State:      notification.FinishedMaintenanceState,
   507  				EndDate:    time.Now().Format("2006-01-02 15:04:05"),
   508  			},
   509  		}
   510  		notificationParas := notification.NotificationParams{
   511  			OrchestrationID: fixOrchestrationID,
   512  			Tenants:         notificationTenants,
   513  		}
   514  		notificationBuilder := &notificationAutomock.BundleBuilder{}
   515  		bundle := &notificationAutomock.Bundle{}
   516  		notificationBuilder.On("NewBundle", fixOrchestrationID, notificationParas).Return(bundle, nil).Once()
   517  		bundle.On("UpdateNotificationEvent").Return(nil).Once()
   518  
   519  		step := NewInitialisationStep(memoryStorage.Operations(), memoryStorage.Orchestrations(), provisionerClient, inputBuilder, evalManager, nil, notificationBuilder)
   520  
   521  		// when
   522  		upgradeOperation, repeat, err := step.Run(upgradeOperation, log)
   523  
   524  		// then
   525  		assert.NotNil(t, err)
   526  		assert.Equal(t, time.Duration(0), repeat)
   527  		assert.Equal(t, domain.Failed, upgradeOperation.State)
   528  		assert.Equal(t, upgradeOperation.Avs.AvsInternalEvaluationStatus, internal.AvsEvaluationStatus{Current: internalStatus, Original: avs.StatusMaintenance})
   529  		assert.Equal(t, upgradeOperation.Avs.AvsExternalEvaluationStatus, internal.AvsEvaluationStatus{Current: externalStatus, Original: avs.StatusMaintenance})
   530  
   531  		storedOp, err := memoryStorage.Operations().GetUpgradeClusterOperationByID(upgradeOperation.Operation.ID)
   532  		assert.Equal(t, upgradeOperation, *storedOp)
   533  		assert.NoError(t, err)
   534  	})
   535  
   536  	t.Run("should refresh avs on success (internal monitor)", func(t *testing.T) {
   537  		// given
   538  		log := logrus.New()
   539  		memoryStorage := storage.NewMemoryStorage()
   540  		evalManager, client := createEvalManager(t, memoryStorage, log)
   541  		inputBuilder := &automock.CreatorForPlan{}
   542  
   543  		err := memoryStorage.Orchestrations().Insert(fixOrchestrationWithKymaVer())
   544  		require.NoError(t, err)
   545  
   546  		provisioningOperation := fixProvisioningOperation()
   547  		err = memoryStorage.Operations().InsertOperation(provisioningOperation)
   548  		require.NoError(t, err)
   549  
   550  		internalStatus, externalStatus := avs.StatusActive, ""
   551  		avsData := createMonitors(t, client, internalStatus, externalStatus)
   552  		avsData.AVSEvaluationExternalId = 0
   553  		upgradeOperation := fixUpgradeClusterOperationWithAvs(avsData)
   554  
   555  		err = memoryStorage.Operations().InsertUpgradeClusterOperation(upgradeOperation)
   556  		require.NoError(t, err)
   557  
   558  		instance := fixInstanceRuntimeStatus()
   559  		err = memoryStorage.Instances().Insert(instance)
   560  		require.NoError(t, err)
   561  
   562  		provisionerClient := &provisionerAutomock.Client{}
   563  		provisionerClient.On("RuntimeOperationStatus", fixGlobalAccountID, fixProvisionerOperationID).Return(gqlschema.OperationStatus{
   564  			ID:        ptr.String(fixProvisionerOperationID),
   565  			Operation: "",
   566  			State:     gqlschema.OperationStateSucceeded,
   567  			Message:   nil,
   568  			RuntimeID: StringPtr(fixRuntimeID),
   569  		}, nil)
   570  
   571  		notificationTenants := []notification.NotificationTenant{
   572  			{
   573  				InstanceID: fixInstanceID,
   574  				State:      notification.FinishedMaintenanceState,
   575  				EndDate:    time.Now().Format("2006-01-02 15:04:05"),
   576  			},
   577  		}
   578  		notificationParas := notification.NotificationParams{
   579  			OrchestrationID: fixOrchestrationID,
   580  			Tenants:         notificationTenants,
   581  		}
   582  		notificationBuilder := &notificationAutomock.BundleBuilder{}
   583  		bundle := &notificationAutomock.Bundle{}
   584  		notificationBuilder.On("NewBundle", fixOrchestrationID, notificationParas).Return(bundle, nil).Once()
   585  		bundle.On("UpdateNotificationEvent").Return(nil).Once()
   586  
   587  		step := NewInitialisationStep(memoryStorage.Operations(), memoryStorage.Orchestrations(), provisionerClient, inputBuilder, evalManager, nil, notificationBuilder)
   588  
   589  		// when
   590  		upgradeOperation, repeat, err := step.Run(upgradeOperation, log)
   591  
   592  		// then
   593  		assert.NoError(t, err)
   594  		assert.Equal(t, time.Duration(0), repeat)
   595  		assert.Equal(t, domain.Succeeded, upgradeOperation.State)
   596  		assert.Equal(t, upgradeOperation.Avs.AvsInternalEvaluationStatus, internal.AvsEvaluationStatus{Current: internalStatus, Original: avs.StatusMaintenance})
   597  		assert.Equal(t, upgradeOperation.Avs.AvsExternalEvaluationStatus, internal.AvsEvaluationStatus{Current: "", Original: ""})
   598  
   599  		storedOp, err := memoryStorage.Operations().GetUpgradeClusterOperationByID(upgradeOperation.Operation.ID)
   600  		assert.Equal(t, upgradeOperation, *storedOp)
   601  		assert.NoError(t, err)
   602  	})
   603  
   604  	t.Run("should refresh avs on success (external monitor)", func(t *testing.T) {
   605  		// given
   606  		log := logrus.New()
   607  		memoryStorage := storage.NewMemoryStorage()
   608  		evalManager, client := createEvalManager(t, memoryStorage, log)
   609  		inputBuilder := &automock.CreatorForPlan{}
   610  
   611  		err := memoryStorage.Orchestrations().Insert(fixOrchestrationWithKymaVer())
   612  		require.NoError(t, err)
   613  
   614  		provisioningOperation := fixProvisioningOperation()
   615  		err = memoryStorage.Operations().InsertOperation(provisioningOperation)
   616  		require.NoError(t, err)
   617  
   618  		internalStatus, externalStatus := "", avs.StatusInactive
   619  		avsData := createMonitors(t, client, internalStatus, externalStatus)
   620  		avsData.AvsEvaluationInternalId = 0
   621  		upgradeOperation := fixUpgradeClusterOperationWithAvs(avsData)
   622  
   623  		err = memoryStorage.Operations().InsertUpgradeClusterOperation(upgradeOperation)
   624  		require.NoError(t, err)
   625  
   626  		instance := fixInstanceRuntimeStatus()
   627  		err = memoryStorage.Instances().Insert(instance)
   628  		require.NoError(t, err)
   629  
   630  		provisionerClient := &provisionerAutomock.Client{}
   631  		provisionerClient.On("RuntimeOperationStatus", fixGlobalAccountID, fixProvisionerOperationID).Return(gqlschema.OperationStatus{
   632  			ID:        ptr.String(fixProvisionerOperationID),
   633  			Operation: "",
   634  			State:     gqlschema.OperationStateSucceeded,
   635  			Message:   nil,
   636  			RuntimeID: StringPtr(fixRuntimeID),
   637  		}, nil)
   638  
   639  		notificationTenants := []notification.NotificationTenant{
   640  			{
   641  				InstanceID: fixInstanceID,
   642  				State:      notification.FinishedMaintenanceState,
   643  				EndDate:    time.Now().Format("2006-01-02 15:04:05"),
   644  			},
   645  		}
   646  		notificationParas := notification.NotificationParams{
   647  			OrchestrationID: fixOrchestrationID,
   648  			Tenants:         notificationTenants,
   649  		}
   650  		notificationBuilder := &notificationAutomock.BundleBuilder{}
   651  		bundle := &notificationAutomock.Bundle{}
   652  		notificationBuilder.On("NewBundle", fixOrchestrationID, notificationParas).Return(bundle, nil).Once()
   653  		bundle.On("UpdateNotificationEvent").Return(nil).Once()
   654  
   655  		step := NewInitialisationStep(memoryStorage.Operations(), memoryStorage.Orchestrations(), provisionerClient, inputBuilder, evalManager, nil, notificationBuilder)
   656  
   657  		// when
   658  		upgradeOperation, repeat, err := step.Run(upgradeOperation, log)
   659  
   660  		// then
   661  		assert.NoError(t, err)
   662  		assert.Equal(t, time.Duration(0), repeat)
   663  		assert.Equal(t, domain.Succeeded, upgradeOperation.State)
   664  		assert.Equal(t, upgradeOperation.Avs.AvsInternalEvaluationStatus, internal.AvsEvaluationStatus{Current: "", Original: ""})
   665  		assert.Equal(t, upgradeOperation.Avs.AvsExternalEvaluationStatus, internal.AvsEvaluationStatus{Current: externalStatus, Original: avs.StatusMaintenance})
   666  
   667  		storedOp, err := memoryStorage.Operations().GetUpgradeClusterOperationByID(upgradeOperation.Operation.ID)
   668  		assert.Equal(t, upgradeOperation, *storedOp)
   669  		assert.NoError(t, err)
   670  	})
   671  
   672  	t.Run("should refresh avs on success (no monitors)", func(t *testing.T) {
   673  		// given
   674  		log := logrus.New()
   675  		memoryStorage := storage.NewMemoryStorage()
   676  		evalManager, client := createEvalManager(t, memoryStorage, log)
   677  		inputBuilder := &automock.CreatorForPlan{}
   678  
   679  		err := memoryStorage.Orchestrations().Insert(fixOrchestrationWithKymaVer())
   680  		require.NoError(t, err)
   681  
   682  		provisioningOperation := fixProvisioningOperation()
   683  		err = memoryStorage.Operations().InsertOperation(provisioningOperation)
   684  		require.NoError(t, err)
   685  
   686  		internalStatus, externalStatus := "", ""
   687  		avsData := createMonitors(t, client, internalStatus, externalStatus)
   688  		avsData.AvsEvaluationInternalId = 0
   689  		avsData.AVSEvaluationExternalId = 0
   690  		upgradeOperation := fixUpgradeClusterOperationWithAvs(avsData)
   691  
   692  		err = memoryStorage.Operations().InsertUpgradeClusterOperation(upgradeOperation)
   693  		require.NoError(t, err)
   694  
   695  		instance := fixInstanceRuntimeStatus()
   696  		err = memoryStorage.Instances().Insert(instance)
   697  		require.NoError(t, err)
   698  
   699  		provisionerClient := &provisionerAutomock.Client{}
   700  		provisionerClient.On("RuntimeOperationStatus", fixGlobalAccountID, fixProvisionerOperationID).Return(gqlschema.OperationStatus{
   701  			ID:        ptr.String(fixProvisionerOperationID),
   702  			Operation: "",
   703  			State:     gqlschema.OperationStateSucceeded,
   704  			Message:   nil,
   705  			RuntimeID: StringPtr(fixRuntimeID),
   706  		}, nil)
   707  
   708  		notificationTenants := []notification.NotificationTenant{
   709  			{
   710  				InstanceID: fixInstanceID,
   711  				State:      notification.FinishedMaintenanceState,
   712  				EndDate:    time.Now().Format("2006-01-02 15:04:05"),
   713  			},
   714  		}
   715  		notificationParas := notification.NotificationParams{
   716  			OrchestrationID: fixOrchestrationID,
   717  			Tenants:         notificationTenants,
   718  		}
   719  		notificationBuilder := &notificationAutomock.BundleBuilder{}
   720  		bundle := &notificationAutomock.Bundle{}
   721  		notificationBuilder.On("NewBundle", fixOrchestrationID, notificationParas).Return(bundle, nil).Once()
   722  		bundle.On("UpdateNotificationEvent").Return(nil).Once()
   723  
   724  		step := NewInitialisationStep(memoryStorage.Operations(), memoryStorage.Orchestrations(), provisionerClient, inputBuilder, evalManager, nil, notificationBuilder)
   725  
   726  		// when
   727  		upgradeOperation, repeat, err := step.Run(upgradeOperation, log)
   728  
   729  		// then
   730  		assert.NoError(t, err)
   731  		assert.Equal(t, time.Duration(0), repeat)
   732  		assert.Equal(t, domain.Succeeded, upgradeOperation.State)
   733  		assert.Equal(t, upgradeOperation.Avs.AvsInternalEvaluationStatus, internal.AvsEvaluationStatus{Current: "", Original: ""})
   734  		assert.Equal(t, upgradeOperation.Avs.AvsExternalEvaluationStatus, internal.AvsEvaluationStatus{Current: "", Original: ""})
   735  
   736  		storedOp, err := memoryStorage.Operations().GetUpgradeClusterOperationByID(upgradeOperation.Operation.ID)
   737  		assert.Equal(t, upgradeOperation, *storedOp)
   738  		assert.NoError(t, err)
   739  	})
   740  
   741  	t.Run("should retry on client error (both monitors)", func(t *testing.T) {
   742  		// given
   743  		log := logrus.New()
   744  		memoryStorage := storage.NewMemoryStorage()
   745  		_, client := createEvalManager(t, memoryStorage, log)
   746  		evalManagerInvalid, _ := createEvalManagerWithValidity(t, memoryStorage, log, false)
   747  		inputBuilder := &automock.CreatorForPlan{}
   748  
   749  		err := memoryStorage.Orchestrations().Insert(fixOrchestrationWithKymaVer())
   750  		require.NoError(t, err)
   751  
   752  		provisioningOperation := fixProvisioningOperation()
   753  		err = memoryStorage.Operations().InsertOperation(provisioningOperation)
   754  		require.NoError(t, err)
   755  
   756  		internalStatus, externalStatus := avs.StatusInactive, avs.StatusActive
   757  		avsData := createMonitors(t, client, internalStatus, externalStatus)
   758  		upgradeOperation := fixUpgradeClusterOperationWithAvs(avsData)
   759  
   760  		err = memoryStorage.Operations().InsertUpgradeClusterOperation(upgradeOperation)
   761  		require.NoError(t, err)
   762  
   763  		instance := fixInstanceRuntimeStatus()
   764  		err = memoryStorage.Instances().Insert(instance)
   765  		require.NoError(t, err)
   766  
   767  		provisionerClient := &provisionerAutomock.Client{}
   768  		provisionerClient.On("RuntimeOperationStatus", fixGlobalAccountID, fixProvisionerOperationID).Return(
   769  			gqlschema.OperationStatus{
   770  				ID:        ptr.String(fixProvisionerOperationID),
   771  				Operation: "",
   772  				State:     gqlschema.OperationStateSucceeded,
   773  				Message:   nil,
   774  				RuntimeID: StringPtr(fixRuntimeID),
   775  			}, nil)
   776  
   777  		notificationTenants := []notification.NotificationTenant{
   778  			{
   779  				InstanceID: fixInstanceID,
   780  				State:      notification.FinishedMaintenanceState,
   781  				EndDate:    time.Now().Format("2006-01-02 15:04:05"),
   782  			},
   783  		}
   784  		notificationParas := notification.NotificationParams{
   785  			OrchestrationID: fixOrchestrationID,
   786  			Tenants:         notificationTenants,
   787  		}
   788  		notificationBuilder := &notificationAutomock.BundleBuilder{}
   789  		bundle := &notificationAutomock.Bundle{}
   790  		notificationBuilder.On("NewBundle", fixOrchestrationID, notificationParas).Return(bundle, nil).Once()
   791  		bundle.On("UpdateNotificationEvent").Return(nil).Once()
   792  
   793  		step := NewInitialisationStep(memoryStorage.Operations(), memoryStorage.Orchestrations(), provisionerClient, inputBuilder, evalManagerInvalid, nil, notificationBuilder)
   794  
   795  		// when
   796  		upgradeOperation, repeat, err := step.Run(upgradeOperation, log)
   797  
   798  		// then
   799  		assert.NoError(t, err)
   800  		assert.Equal(t, 10*time.Second, repeat)
   801  		assert.Equal(t, domain.InProgress, upgradeOperation.State)
   802  		assert.Equal(t, internal.AvsEvaluationStatus{Current: internalStatus, Original: internalStatus}, upgradeOperation.Avs.AvsInternalEvaluationStatus)
   803  		assert.Equal(t, internal.AvsEvaluationStatus{Current: externalStatus, Original: ""}, upgradeOperation.Avs.AvsExternalEvaluationStatus)
   804  	})
   805  
   806  	t.Run("should go through init and finish steps (both monitors)", func(t *testing.T) {
   807  		// given
   808  		log := logrus.New()
   809  		memoryStorage := storage.NewMemoryStorage()
   810  		evalManager, client := createEvalManager(t, memoryStorage, log)
   811  		evalManagerInvalid, _ := createEvalManagerWithValidity(t, memoryStorage, log, false)
   812  		inputBuilder := &automock.CreatorForPlan{}
   813  
   814  		err := memoryStorage.Orchestrations().Insert(fixOrchestrationWithKymaVer())
   815  		require.NoError(t, err)
   816  
   817  		provisioningOperation := fixProvisioningOperation()
   818  		err = memoryStorage.Operations().InsertOperation(provisioningOperation)
   819  		require.NoError(t, err)
   820  
   821  		internalStatus, externalStatus := avs.StatusInactive, avs.StatusActive
   822  		avsData := createMonitors(t, client, internalStatus, externalStatus)
   823  		upgradeOperation := fixUpgradeClusterOperationWithAvs(avsData)
   824  
   825  		err = memoryStorage.Operations().InsertUpgradeClusterOperation(upgradeOperation)
   826  		require.NoError(t, err)
   827  
   828  		instance := fixInstanceRuntimeStatus()
   829  		err = memoryStorage.Instances().Insert(instance)
   830  		require.NoError(t, err)
   831  
   832  		callCounter := 0
   833  		provisionerClient := &provisionerAutomock.Client{}
   834  		// for the first 2 step.Run calls, RuntimeOperationStatus will return OperationStateInProgress
   835  		// otherwise, OperationStateSucceeded
   836  		provisionerClient.On("RuntimeOperationStatus", fixGlobalAccountID, fixProvisionerOperationID).Return(
   837  			func(accountID string, operationID string) gqlschema.OperationStatus {
   838  				callCounter++
   839  				if callCounter <= 2 {
   840  					return gqlschema.OperationStatus{
   841  						ID:        ptr.String(fixProvisionerOperationID),
   842  						Operation: "",
   843  						State:     gqlschema.OperationStateInProgress,
   844  						Message:   nil,
   845  						RuntimeID: StringPtr(fixRuntimeID),
   846  					}
   847  				}
   848  
   849  				return gqlschema.OperationStatus{
   850  					ID:        ptr.String(fixProvisionerOperationID),
   851  					Operation: "",
   852  					State:     gqlschema.OperationStateSucceeded,
   853  					Message:   nil,
   854  					RuntimeID: StringPtr(fixRuntimeID),
   855  				}
   856  			}, nil)
   857  
   858  		notificationTenants := []notification.NotificationTenant{
   859  			{
   860  				InstanceID: fixInstanceID,
   861  				State:      notification.FinishedMaintenanceState,
   862  				EndDate:    time.Now().Format("2006-01-02 15:04:05"),
   863  			},
   864  		}
   865  		notificationParas := notification.NotificationParams{
   866  			OrchestrationID: fixOrchestrationID,
   867  			Tenants:         notificationTenants,
   868  		}
   869  		notificationBuilder := &notificationAutomock.BundleBuilder{}
   870  		bundle := &notificationAutomock.Bundle{}
   871  		notificationBuilder.On("NewBundle", fixOrchestrationID, notificationParas).Return(bundle, nil).Once()
   872  		bundle.On("UpdateNotificationEvent").Return(nil).Once()
   873  
   874  		step := NewInitialisationStep(memoryStorage.Operations(), memoryStorage.Orchestrations(), provisionerClient, inputBuilder, evalManagerInvalid, nil, notificationBuilder)
   875  
   876  		// when invalid client request, this should be delayed
   877  		upgradeOperation, repeat, err := step.Run(upgradeOperation, log)
   878  
   879  		// then
   880  		assert.NoError(t, err)
   881  		assert.Equal(t, 10*time.Second, repeat)
   882  		assert.Equal(t, domain.InProgress, upgradeOperation.State)
   883  		assert.Equal(t, internal.AvsEvaluationStatus{Current: internalStatus, Original: internalStatus}, upgradeOperation.Avs.AvsInternalEvaluationStatus)
   884  		assert.Equal(t, internal.AvsEvaluationStatus{Current: externalStatus, Original: ""}, upgradeOperation.Avs.AvsExternalEvaluationStatus)
   885  
   886  		// when valid client request and InProgress state from RuntimeOperationStatus, this should do init tasks
   887  		step.evaluationManager = evalManager
   888  		upgradeOperation, repeat, err = step.Run(upgradeOperation, log)
   889  
   890  		// then
   891  		assert.NoError(t, err)
   892  		assert.Equal(t, 1*time.Minute, repeat)
   893  		assert.Equal(t, domain.InProgress, upgradeOperation.State)
   894  		assert.Equal(t, upgradeOperation.Avs.AvsInternalEvaluationStatus, internal.AvsEvaluationStatus{Current: avs.StatusMaintenance, Original: internalStatus})
   895  		assert.Equal(t, upgradeOperation.Avs.AvsExternalEvaluationStatus, internal.AvsEvaluationStatus{Current: avs.StatusMaintenance, Original: externalStatus})
   896  
   897  		// when valid client request and Succeeded state from RuntimeOperationStatus, this should do finish tasks
   898  		upgradeOperation, repeat, err = step.Run(upgradeOperation, log)
   899  
   900  		// then
   901  		assert.NoError(t, err)
   902  		assert.Equal(t, time.Duration(0), repeat)
   903  		assert.Equal(t, domain.Succeeded, upgradeOperation.State)
   904  		assert.Equal(t, upgradeOperation.Avs.AvsInternalEvaluationStatus, internal.AvsEvaluationStatus{Current: internalStatus, Original: avs.StatusMaintenance})
   905  		assert.Equal(t, upgradeOperation.Avs.AvsExternalEvaluationStatus, internal.AvsEvaluationStatus{Current: externalStatus, Original: avs.StatusMaintenance})
   906  
   907  		storedOp, err := memoryStorage.Operations().GetUpgradeClusterOperationByID(upgradeOperation.Operation.ID)
   908  		assert.Equal(t, upgradeOperation, *storedOp)
   909  		assert.NoError(t, err)
   910  	})
   911  
   912  	t.Run("should set AvS evaluations statuses to maintenance", func(t *testing.T) {
   913  		// given
   914  		maintenanceModeDisabled := false
   915  		maintenanceModeAlwaysDisabledGAIDs := []string{fixMaintenanceModeAlwaysDisabledGlobalAccountID}
   916  
   917  		log := logrus.New()
   918  		memoryStorage := storage.NewMemoryStorage()
   919  		evalManager, client := createEvalManagerWithMaintenanceModeConfig(t, memoryStorage, maintenanceModeDisabled, maintenanceModeAlwaysDisabledGAIDs)
   920  		inputBuilder := &automock.CreatorForPlan{}
   921  
   922  		provisioningOperation := fixProvisioningOperation()
   923  		err := memoryStorage.Operations().InsertOperation(provisioningOperation)
   924  		require.NoError(t, err)
   925  
   926  		err = memoryStorage.Orchestrations().Insert(fixOrchestrationWithKymaVer())
   927  		require.NoError(t, err)
   928  
   929  		avsData := createMonitors(t, client, avs.StatusActive, avs.StatusActive)
   930  		upgradeOperation := fixUpgradeClusterOperationWithAvs(avsData)
   931  
   932  		err = memoryStorage.Operations().InsertUpgradeClusterOperation(upgradeOperation)
   933  		require.NoError(t, err)
   934  
   935  		instance := fixInstanceRuntimeStatus()
   936  		err = memoryStorage.Instances().Insert(instance)
   937  		require.NoError(t, err)
   938  
   939  		provisionerClient := &provisionerAutomock.Client{}
   940  		provisionerClient.On("RuntimeOperationStatus", fixGlobalAccountID, fixProvisionerOperationID).Return(gqlschema.OperationStatus{
   941  			ID:        ptr.String(fixProvisionerOperationID),
   942  			Operation: "",
   943  			State:     gqlschema.OperationStateInProgress,
   944  			Message:   nil,
   945  			RuntimeID: StringPtr(fixRuntimeID),
   946  		}, nil)
   947  
   948  		notificationTenants := []notification.NotificationTenant{
   949  			{
   950  				InstanceID: fixInstanceID,
   951  				State:      notification.FinishedMaintenanceState,
   952  				EndDate:    time.Now().Format("2006-01-02 15:04:05"),
   953  			},
   954  		}
   955  		notificationParams := notification.NotificationParams{
   956  			OrchestrationID: fixOrchestrationID,
   957  			Tenants:         notificationTenants,
   958  		}
   959  		notificationBuilder := &notificationAutomock.BundleBuilder{}
   960  		bundle := &notificationAutomock.Bundle{}
   961  		notificationBuilder.On("NewBundle", fixOrchestrationID, notificationParams).Return(bundle, nil).Once()
   962  		bundle.On("UpdateNotificationEvent").Return(nil).Once()
   963  
   964  		step := NewInitialisationStep(memoryStorage.Operations(), memoryStorage.Orchestrations(), provisionerClient,
   965  			inputBuilder, evalManager, nil, notificationBuilder)
   966  
   967  		// when
   968  		upgradeOperation, repeat, err := step.Run(upgradeOperation, log)
   969  
   970  		// then
   971  		assert.NoError(t, err)
   972  		assert.Equal(t, 1*time.Minute, repeat) // 1 min for StatusCheck
   973  		assert.Equal(t, domain.InProgress, upgradeOperation.State)
   974  		assert.Equal(t, upgradeOperation.Avs.AvsInternalEvaluationStatus, internal.AvsEvaluationStatus{Current: avs.StatusMaintenance, Original: avs.StatusActive})
   975  		assert.Equal(t, upgradeOperation.Avs.AvsExternalEvaluationStatus, internal.AvsEvaluationStatus{Current: avs.StatusMaintenance, Original: avs.StatusActive})
   976  
   977  		storedOp, err := memoryStorage.Operations().GetUpgradeClusterOperationByID(upgradeOperation.Operation.ID)
   978  		assert.Equal(t, upgradeOperation, *storedOp)
   979  		assert.NoError(t, err)
   980  	})
   981  
   982  	t.Run("should keep active AvS evaluations statuses for given GlobalAccount ID", func(t *testing.T) {
   983  		// given
   984  		maintenanceModeDisabled := false
   985  		maintenanceModeAlwaysDisabledGAIDs := []string{fixGlobalAccountID}
   986  
   987  		log := logrus.New()
   988  		memoryStorage := storage.NewMemoryStorage()
   989  		evalManager, client := createEvalManagerWithMaintenanceModeConfig(t, memoryStorage, maintenanceModeDisabled, maintenanceModeAlwaysDisabledGAIDs)
   990  		inputBuilder := &automock.CreatorForPlan{}
   991  
   992  		provisioningOperation := fixProvisioningOperation()
   993  		err := memoryStorage.Operations().InsertOperation(provisioningOperation)
   994  		require.NoError(t, err)
   995  
   996  		err = memoryStorage.Orchestrations().Insert(fixOrchestrationWithKymaVer())
   997  		require.NoError(t, err)
   998  
   999  		avsData := createMonitors(t, client, avs.StatusActive, avs.StatusActive)
  1000  		upgradeOperation := fixUpgradeClusterOperationWithAvs(avsData)
  1001  
  1002  		err = memoryStorage.Operations().InsertUpgradeClusterOperation(upgradeOperation)
  1003  		require.NoError(t, err)
  1004  
  1005  		instance := fixInstanceRuntimeStatus()
  1006  		err = memoryStorage.Instances().Insert(instance)
  1007  		require.NoError(t, err)
  1008  
  1009  		provisionerClient := &provisionerAutomock.Client{}
  1010  		provisionerClient.On("RuntimeOperationStatus", fixGlobalAccountID, fixProvisionerOperationID).Return(gqlschema.OperationStatus{
  1011  			ID:        ptr.String(fixProvisionerOperationID),
  1012  			Operation: "",
  1013  			State:     gqlschema.OperationStateInProgress,
  1014  			Message:   nil,
  1015  			RuntimeID: StringPtr(fixRuntimeID),
  1016  		}, nil)
  1017  
  1018  		notificationTenants := []notification.NotificationTenant{
  1019  			{
  1020  				InstanceID: fixInstanceID,
  1021  				State:      notification.FinishedMaintenanceState,
  1022  				EndDate:    time.Now().Format("2006-01-02 15:04:05"),
  1023  			},
  1024  		}
  1025  		notificationParams := notification.NotificationParams{
  1026  			OrchestrationID: fixOrchestrationID,
  1027  			Tenants:         notificationTenants,
  1028  		}
  1029  		notificationBuilder := &notificationAutomock.BundleBuilder{}
  1030  		bundle := &notificationAutomock.Bundle{}
  1031  		notificationBuilder.On("NewBundle", fixOrchestrationID, notificationParams).Return(bundle, nil).Once()
  1032  		bundle.On("UpdateNotificationEvent").Return(nil).Once()
  1033  
  1034  		step := NewInitialisationStep(memoryStorage.Operations(), memoryStorage.Orchestrations(), provisionerClient,
  1035  			inputBuilder, evalManager, nil, notificationBuilder)
  1036  
  1037  		// when
  1038  		upgradeOperation, repeat, err := step.Run(upgradeOperation, log)
  1039  
  1040  		// then
  1041  		assert.NoError(t, err)
  1042  		assert.Equal(t, 1*time.Minute, repeat) // 1 min for StatusCheck
  1043  		assert.Equal(t, domain.InProgress, upgradeOperation.State)
  1044  		assert.Equal(t, upgradeOperation.Avs.AvsInternalEvaluationStatus, internal.AvsEvaluationStatus{Current: avs.StatusActive, Original: ""})
  1045  		assert.Equal(t, upgradeOperation.Avs.AvsExternalEvaluationStatus, internal.AvsEvaluationStatus{Current: avs.StatusActive, Original: ""})
  1046  
  1047  		storedOp, err := memoryStorage.Operations().GetUpgradeClusterOperationByID(upgradeOperation.Operation.ID)
  1048  		assert.Equal(t, upgradeOperation, *storedOp)
  1049  		assert.NoError(t, err)
  1050  	})
  1051  
  1052  	t.Run("should keep active AvS evaluations statuses for all GA IDs", func(t *testing.T) {
  1053  		// given
  1054  		maintenanceModeDisabled := true
  1055  		maintenanceModeAlwaysDisabledGAIDs := []string{fixMaintenanceModeAlwaysDisabledGlobalAccountID}
  1056  
  1057  		log := logrus.New()
  1058  		memoryStorage := storage.NewMemoryStorage()
  1059  		evalManager, client := createEvalManagerWithMaintenanceModeConfig(t, memoryStorage, maintenanceModeDisabled, maintenanceModeAlwaysDisabledGAIDs)
  1060  		inputBuilder := &automock.CreatorForPlan{}
  1061  
  1062  		provisioningOperation := fixProvisioningOperation()
  1063  		err := memoryStorage.Operations().InsertOperation(provisioningOperation)
  1064  		require.NoError(t, err)
  1065  
  1066  		err = memoryStorage.Orchestrations().Insert(fixOrchestrationWithKymaVer())
  1067  		require.NoError(t, err)
  1068  
  1069  		avsData := createMonitors(t, client, avs.StatusActive, avs.StatusActive)
  1070  		upgradeOperation := fixUpgradeClusterOperationWithAvs(avsData)
  1071  
  1072  		err = memoryStorage.Operations().InsertUpgradeClusterOperation(upgradeOperation)
  1073  		require.NoError(t, err)
  1074  
  1075  		instance := fixInstanceRuntimeStatus()
  1076  		err = memoryStorage.Instances().Insert(instance)
  1077  		require.NoError(t, err)
  1078  
  1079  		provisionerClient := &provisionerAutomock.Client{}
  1080  		provisionerClient.On("RuntimeOperationStatus", fixGlobalAccountID, fixProvisionerOperationID).Return(gqlschema.OperationStatus{
  1081  			ID:        ptr.String(fixProvisionerOperationID),
  1082  			Operation: "",
  1083  			State:     gqlschema.OperationStateInProgress,
  1084  			Message:   nil,
  1085  			RuntimeID: StringPtr(fixRuntimeID),
  1086  		}, nil)
  1087  
  1088  		notificationTenants := []notification.NotificationTenant{
  1089  			{
  1090  				InstanceID: fixInstanceID,
  1091  				State:      notification.FinishedMaintenanceState,
  1092  				EndDate:    time.Now().Format("2006-01-02 15:04:05"),
  1093  			},
  1094  		}
  1095  		notificationParams := notification.NotificationParams{
  1096  			OrchestrationID: fixOrchestrationID,
  1097  			Tenants:         notificationTenants,
  1098  		}
  1099  		notificationBuilder := &notificationAutomock.BundleBuilder{}
  1100  		bundle := &notificationAutomock.Bundle{}
  1101  		notificationBuilder.On("NewBundle", fixOrchestrationID, notificationParams).Return(bundle, nil).Once()
  1102  		bundle.On("UpdateNotificationEvent").Return(nil).Once()
  1103  
  1104  		step := NewInitialisationStep(memoryStorage.Operations(), memoryStorage.Orchestrations(), provisionerClient,
  1105  			inputBuilder, evalManager, nil, notificationBuilder)
  1106  
  1107  		// when
  1108  		upgradeOperation, repeat, err := step.Run(upgradeOperation, log)
  1109  
  1110  		// then
  1111  		assert.NoError(t, err)
  1112  		assert.Equal(t, 1*time.Minute, repeat) // 1 min for StatusCheck
  1113  		assert.Equal(t, domain.InProgress, upgradeOperation.State)
  1114  		assert.Equal(t, upgradeOperation.Avs.AvsInternalEvaluationStatus, internal.AvsEvaluationStatus{Current: avs.StatusActive, Original: ""})
  1115  		assert.Equal(t, upgradeOperation.Avs.AvsExternalEvaluationStatus, internal.AvsEvaluationStatus{Current: avs.StatusActive, Original: ""})
  1116  
  1117  		storedOp, err := memoryStorage.Operations().GetUpgradeClusterOperationByID(upgradeOperation.Operation.ID)
  1118  		assert.Equal(t, upgradeOperation, *storedOp)
  1119  		assert.NoError(t, err)
  1120  	})
  1121  }
  1122  
  1123  func fixUpgradeClusterOperation() internal.UpgradeClusterOperation {
  1124  	return fixUpgradeClusterOperationWithAvs(internal.AvsLifecycleData{})
  1125  }
  1126  
  1127  func fixUpgradeClusterOperationWithAvs(avsData internal.AvsLifecycleData) internal.UpgradeClusterOperation {
  1128  	upgradeOperation := fixture.FixUpgradeClusterOperation(fixUpgradeOperationID, fixInstanceID)
  1129  	upgradeOperation.OrchestrationID = fixOrchestrationID
  1130  	upgradeOperation.ProvisionerOperationID = fixProvisionerOperationID
  1131  	upgradeOperation.State = orchestration.Pending
  1132  	upgradeOperation.Description = ""
  1133  	upgradeOperation.UpdatedAt = time.Now()
  1134  	upgradeOperation.InstanceDetails.Avs = avsData
  1135  	upgradeOperation.ProvisioningParameters = fixProvisioningParameters()
  1136  	upgradeOperation.RuntimeOperation.GlobalAccountID = fixGlobalAccountID
  1137  	upgradeOperation.RuntimeOperation.SubAccountID = fixSubAccountID
  1138  	upgradeOperation.InputCreator = nil
  1139  
  1140  	return upgradeOperation
  1141  }
  1142  
  1143  func fixProvisioningOperation() internal.Operation {
  1144  	provisioningOperation := fixture.FixProvisioningOperation(fixProvisioningOperationID, fixInstanceID)
  1145  	provisioningOperation.ProvisionerOperationID = fixProvisionerOperationID
  1146  	provisioningOperation.Description = ""
  1147  	provisioningOperation.ProvisioningParameters = fixProvisioningParameters()
  1148  
  1149  	return provisioningOperation
  1150  }
  1151  
  1152  func fixProvisioningParameters() internal.ProvisioningParameters {
  1153  	pp := fixture.FixProvisioningParameters("1")
  1154  	pp.PlanID = broker.AzurePlanID
  1155  	pp.ServiceID = ""
  1156  	pp.ErsContext.GlobalAccountID = fixGlobalAccountID
  1157  	pp.ErsContext.SubAccountID = fixSubAccountID
  1158  
  1159  	return pp
  1160  }
  1161  
  1162  func fixInstanceRuntimeStatus() internal.Instance {
  1163  	instance := fixture.FixInstance(fixInstanceID)
  1164  	instance.RuntimeID = fixRuntimeID
  1165  	instance.GlobalAccountID = fixGlobalAccountID
  1166  
  1167  	return instance
  1168  }
  1169  
  1170  func StringPtr(s string) *string {
  1171  	return &s
  1172  }
  1173  
  1174  // no forFreemiumPlan and forTrialPlan supported for the mock testing. planID by default is broker.AzurePlanID.
  1175  func fixGetHyperscalerProviderForPlanID(planID string) fixHyperscalerInputProvider {
  1176  	var provider fixHyperscalerInputProvider
  1177  	switch planID {
  1178  	case broker.GCPPlanID:
  1179  		provider = &cloudProvider.GcpInput{}
  1180  	case broker.OpenStackPlanID:
  1181  		provider = &cloudProvider.OpenStackInput{}
  1182  	case broker.AzurePlanID:
  1183  		provider = &cloudProvider.AzureInput{}
  1184  	case broker.AzureLitePlanID:
  1185  		provider = &cloudProvider.AzureLiteInput{}
  1186  	case broker.AWSPlanID:
  1187  		provider = &cloudProvider.AWSInput{}
  1188  		// insert cases for other providers like AWS or GCP
  1189  	default:
  1190  		return nil
  1191  	}
  1192  	return provider
  1193  }
  1194  
  1195  func fixOrchestrationWithKymaVer() internal.Orchestration {
  1196  	orch := internal.Orchestration{
  1197  		OrchestrationID: fixOrchestrationID,
  1198  		State:           orchestration.InProgress,
  1199  		Parameters: orchestration.Parameters{
  1200  			Kyma: &orchestration.KymaParameters{
  1201  				Version: fixKymaVersion,
  1202  			},
  1203  			Notification: true,
  1204  		},
  1205  	}
  1206  	return orch
  1207  }