github.com/kyma-project/kyma-environment-broker@v0.0.1/internal/orchestration/handlers/orchestration_handler_test.go (about)

     1  package handlers
     2  
     3  import (
     4  	"encoding/json"
     5  	"fmt"
     6  	"net/http"
     7  	"net/http/httptest"
     8  	"strings"
     9  	"testing"
    10  	"time"
    11  
    12  	"github.com/google/uuid"
    13  	"github.com/gorilla/mux"
    14  	reconcilerApi "github.com/kyma-incubator/reconciler/pkg/keb"
    15  	"github.com/kyma-project/control-plane/components/provisioner/pkg/gqlschema"
    16  	"github.com/kyma-project/kyma-environment-broker/common/orchestration"
    17  	"github.com/kyma-project/kyma-environment-broker/internal"
    18  	"github.com/kyma-project/kyma-environment-broker/internal/broker"
    19  	"github.com/kyma-project/kyma-environment-broker/internal/fixture"
    20  	"github.com/kyma-project/kyma-environment-broker/internal/process"
    21  	"github.com/kyma-project/kyma-environment-broker/internal/ptr"
    22  	"github.com/kyma-project/kyma-environment-broker/internal/storage"
    23  	"github.com/pivotal-cf/brokerapi/v8/domain"
    24  	"github.com/sirupsen/logrus"
    25  	"github.com/stretchr/testify/assert"
    26  	"github.com/stretchr/testify/require"
    27  )
    28  
    29  func TestStatusHandler_AttachRoutes(t *testing.T) {
    30  	fixID := "id-1"
    31  	t.Run("orchestrations", func(t *testing.T) {
    32  		// given
    33  		db := storage.NewMemoryStorage()
    34  
    35  		err := db.Orchestrations().Insert(internal.Orchestration{OrchestrationID: fixID})
    36  		require.NoError(t, err)
    37  		err = db.Orchestrations().Insert(internal.Orchestration{OrchestrationID: "id-2"})
    38  		require.NoError(t, err)
    39  
    40  		logs := logrus.New()
    41  		kymaHandler := NewOrchestrationStatusHandler(db.Operations(), db.Orchestrations(), db.RuntimeStates(), nil, nil, 100, logs)
    42  
    43  		req, err := http.NewRequest("GET", "/orchestrations?page_size=1", nil)
    44  		require.NoError(t, err)
    45  
    46  		rr := httptest.NewRecorder()
    47  		router := mux.NewRouter()
    48  		kymaHandler.AttachRoutes(router)
    49  
    50  		// when
    51  		router.ServeHTTP(rr, req)
    52  
    53  		// then
    54  		require.Equal(t, http.StatusOK, rr.Code)
    55  
    56  		var out orchestration.StatusResponseList
    57  
    58  		err = json.Unmarshal(rr.Body.Bytes(), &out)
    59  		require.NoError(t, err)
    60  		assert.Len(t, out.Data, 1)
    61  		assert.Equal(t, 2, out.TotalCount)
    62  		assert.Equal(t, 1, out.Count)
    63  
    64  		// given
    65  		urlPath := fmt.Sprintf("/orchestrations?page=2&page_size=1")
    66  		req, err = http.NewRequest(http.MethodGet, urlPath, nil)
    67  		require.NoError(t, err)
    68  		rr = httptest.NewRecorder()
    69  
    70  		// when
    71  		router.ServeHTTP(rr, req)
    72  
    73  		// then
    74  		require.Equal(t, http.StatusOK, rr.Code)
    75  
    76  		err = json.Unmarshal(rr.Body.Bytes(), &out)
    77  		require.NoError(t, err)
    78  		assert.Equal(t, 2, out.TotalCount)
    79  		assert.Equal(t, 1, out.Count)
    80  
    81  		// given
    82  		urlPath = fmt.Sprintf("/orchestrations/%s", fixID)
    83  		req, err = http.NewRequest(http.MethodGet, urlPath, nil)
    84  		require.NoError(t, err)
    85  		rr = httptest.NewRecorder()
    86  		err = db.Operations().InsertUpgradeKymaOperation(internal.UpgradeKymaOperation{
    87  			Operation: internal.Operation{
    88  				ID:              fixID,
    89  				InstanceID:      fixID,
    90  				OrchestrationID: fixID,
    91  				State:           domain.Succeeded,
    92  				ProvisioningParameters: internal.ProvisioningParameters{
    93  					PlanID: "4deee563-e5ec-4731-b9b1-53b42d855f0c",
    94  				},
    95  				RuntimeOperation: orchestration.RuntimeOperation{
    96  					ID: fixID,
    97  				},
    98  				Type: internal.OperationTypeUpgradeKyma,
    99  			},
   100  		})
   101  		err = db.Operations().InsertProvisioningOperation(internal.ProvisioningOperation{
   102  			Operation: internal.Operation{
   103  				ID:         "id-2",
   104  				InstanceID: fixID,
   105  			},
   106  		})
   107  		require.NoError(t, err)
   108  
   109  		dto := orchestration.StatusResponse{}
   110  
   111  		// when
   112  		router.ServeHTTP(rr, req)
   113  
   114  		// then
   115  		require.Equal(t, http.StatusOK, rr.Code)
   116  
   117  		err = json.Unmarshal(rr.Body.Bytes(), &dto)
   118  		require.NoError(t, err)
   119  		assert.Equal(t, dto.OrchestrationID, fixID)
   120  		assert.Len(t, dto.OperationStats, 6)
   121  		assert.Equal(t, 1, dto.OperationStats[orchestration.Succeeded])
   122  	})
   123  
   124  	t.Run("kyma upgrade operations", func(t *testing.T) {
   125  		// given
   126  		db := storage.NewMemoryStorage()
   127  		secondID := "id-2"
   128  
   129  		err := db.Orchestrations().Insert(internal.Orchestration{OrchestrationID: fixID, Type: orchestration.UpgradeKymaOrchestration})
   130  		require.NoError(t, err)
   131  		err = db.Operations().InsertUpgradeKymaOperation(internal.UpgradeKymaOperation{
   132  			Operation: internal.Operation{
   133  				ID:              fixID,
   134  				InstanceID:      fixID,
   135  				OrchestrationID: fixID,
   136  				ProvisioningParameters: internal.ProvisioningParameters{
   137  					PlanID: "4deee563-e5ec-4731-b9b1-53b42d855f0c",
   138  				},
   139  				RuntimeOperation: orchestration.RuntimeOperation{
   140  					ID: fixID,
   141  				},
   142  			},
   143  		})
   144  		err = db.Operations().InsertProvisioningOperation(internal.ProvisioningOperation{
   145  			Operation: internal.Operation{
   146  				ID:         secondID,
   147  				InstanceID: fixID,
   148  			},
   149  		})
   150  		require.NoError(t, err)
   151  
   152  		err = db.RuntimeStates().Insert(internal.RuntimeState{ID: secondID, OperationID: secondID})
   153  		require.NoError(t, err)
   154  		err = db.RuntimeStates().Insert(internal.RuntimeState{ID: fixID, OperationID: fixID})
   155  		require.NoError(t, err)
   156  
   157  		logs := logrus.New()
   158  		kymaHandler := NewOrchestrationStatusHandler(db.Operations(), db.Orchestrations(), db.RuntimeStates(), nil, nil, 100, logs)
   159  
   160  		urlPath := fmt.Sprintf("/orchestrations/%s/operations", fixID)
   161  		req, err := http.NewRequest("GET", urlPath, nil)
   162  		require.NoError(t, err)
   163  
   164  		rr := httptest.NewRecorder()
   165  		router := mux.NewRouter()
   166  		kymaHandler.AttachRoutes(router)
   167  
   168  		// when
   169  		router.ServeHTTP(rr, req)
   170  
   171  		// then
   172  		require.Equal(t, http.StatusOK, rr.Code)
   173  
   174  		var out orchestration.OperationResponseList
   175  
   176  		err = json.Unmarshal(rr.Body.Bytes(), &out)
   177  		require.NoError(t, err)
   178  		assert.Len(t, out.Data, 1)
   179  		assert.Equal(t, 1, out.TotalCount)
   180  		assert.Equal(t, 1, out.Count)
   181  
   182  		// given
   183  		urlPath = fmt.Sprintf("/orchestrations/%s/operations/%s", fixID, fixID)
   184  		req, err = http.NewRequest(http.MethodGet, urlPath, nil)
   185  		require.NoError(t, err)
   186  		rr = httptest.NewRecorder()
   187  
   188  		dto := orchestration.OperationDetailResponse{}
   189  
   190  		// when
   191  		router.ServeHTTP(rr, req)
   192  
   193  		// then
   194  		require.Equal(t, http.StatusOK, rr.Code)
   195  
   196  		err = json.Unmarshal(rr.Body.Bytes(), &dto)
   197  		require.NoError(t, err)
   198  		assert.Equal(t, dto.OrchestrationID, fixID)
   199  		assert.Equal(t, dto.OperationID, fixID)
   200  	})
   201  
   202  	t.Run("cluster upgrade operations", func(t *testing.T) {
   203  		// given
   204  		db := storage.NewMemoryStorage()
   205  
   206  		err := db.Orchestrations().Insert(internal.Orchestration{OrchestrationID: fixID, Type: orchestration.UpgradeClusterOrchestration})
   207  		require.NoError(t, err)
   208  		err = db.Operations().InsertUpgradeClusterOperation(internal.UpgradeClusterOperation{
   209  			Operation: internal.Operation{
   210  				ID:              fixID,
   211  				InstanceID:      fixID,
   212  				OrchestrationID: fixID,
   213  				ProvisioningParameters: internal.ProvisioningParameters{
   214  					PlanID: "4deee563-e5ec-4731-b9b1-53b42d855f0c",
   215  				},
   216  				RuntimeOperation: orchestration.RuntimeOperation{
   217  					ID: fixID,
   218  				},
   219  			},
   220  		})
   221  		require.NoError(t, err)
   222  
   223  		err = db.RuntimeStates().Insert(internal.RuntimeState{ID: fixID, OperationID: fixID})
   224  		require.NoError(t, err)
   225  
   226  		logs := logrus.New()
   227  		kymaHandler := NewOrchestrationStatusHandler(db.Operations(), db.Orchestrations(), db.RuntimeStates(), nil, nil, 100, logs)
   228  
   229  		urlPath := fmt.Sprintf("/orchestrations/%s/operations", fixID)
   230  		req, err := http.NewRequest("GET", urlPath, nil)
   231  		require.NoError(t, err)
   232  
   233  		rr := httptest.NewRecorder()
   234  		router := mux.NewRouter()
   235  		kymaHandler.AttachRoutes(router)
   236  
   237  		// when
   238  		router.ServeHTTP(rr, req)
   239  
   240  		// then
   241  		require.Equal(t, http.StatusOK, rr.Code)
   242  
   243  		var out orchestration.OperationResponseList
   244  
   245  		err = json.Unmarshal(rr.Body.Bytes(), &out)
   246  		require.NoError(t, err)
   247  		assert.Len(t, out.Data, 1)
   248  		assert.Equal(t, 1, out.TotalCount)
   249  		assert.Equal(t, 1, out.Count)
   250  
   251  		// given
   252  		urlPath = fmt.Sprintf("/orchestrations/%s/operations/%s", fixID, fixID)
   253  		req, err = http.NewRequest(http.MethodGet, urlPath, nil)
   254  		require.NoError(t, err)
   255  		rr = httptest.NewRecorder()
   256  
   257  		dto := orchestration.OperationDetailResponse{}
   258  
   259  		// when
   260  		router.ServeHTTP(rr, req)
   261  
   262  		// then
   263  		require.Equal(t, http.StatusOK, rr.Code)
   264  
   265  		err = json.Unmarshal(rr.Body.Bytes(), &dto)
   266  		require.NoError(t, err)
   267  		assert.Equal(t, dto.OrchestrationID, fixID)
   268  		assert.Equal(t, dto.OperationID, fixID)
   269  	})
   270  
   271  	t.Run("cancel orchestration", func(t *testing.T) {
   272  		// given
   273  		db := storage.NewMemoryStorage()
   274  
   275  		err := db.Orchestrations().Insert(internal.Orchestration{OrchestrationID: fixID, State: orchestration.InProgress})
   276  		require.NoError(t, err)
   277  
   278  		logs := logrus.New()
   279  		kymaHandler := NewOrchestrationStatusHandler(db.Operations(), db.Orchestrations(), db.RuntimeStates(), nil, nil, 100, logs)
   280  
   281  		req, err := http.NewRequest("PUT", fmt.Sprintf("/orchestrations/%s/cancel", fixID), nil)
   282  		require.NoError(t, err)
   283  
   284  		rr := httptest.NewRecorder()
   285  		router := mux.NewRouter()
   286  		kymaHandler.AttachRoutes(router)
   287  
   288  		// when
   289  		router.ServeHTTP(rr, req)
   290  
   291  		// then
   292  		require.Equal(t, http.StatusOK, rr.Code)
   293  
   294  		var out orchestration.UpgradeResponse
   295  
   296  		err = json.Unmarshal(rr.Body.Bytes(), &out)
   297  		require.NoError(t, err)
   298  		assert.Equal(t, out.OrchestrationID, fixID)
   299  
   300  		o, err := db.Orchestrations().GetByID(fixID)
   301  		require.NoError(t, err)
   302  		assert.Equal(t, orchestration.Canceling, o.State)
   303  	})
   304  
   305  	t.Run("Kyma 2.0 upgrade operation", func(t *testing.T) {
   306  		// given
   307  		db := storage.NewMemoryStorage()
   308  
   309  		instanceID := "instanceID"
   310  		provisioningOp1ID := "provisioningOp1ID"
   311  
   312  		provisioningOp1 := internal.ProvisioningOperation{
   313  			Operation: internal.Operation{
   314  				ID:         provisioningOp1ID,
   315  				InstanceID: instanceID,
   316  			},
   317  		}
   318  
   319  		err := db.Operations().InsertProvisioningOperation(provisioningOp1)
   320  		require.NoError(t, err)
   321  
   322  		orchestration1ID := "ochestration1ID"
   323  		orchestration1 := internal.Orchestration{
   324  			OrchestrationID: orchestration1ID,
   325  			Type:            orchestration.UpgradeKymaOrchestration,
   326  		}
   327  
   328  		err = db.Orchestrations().Insert(orchestration1)
   329  		require.NoError(t, err)
   330  
   331  		upgradeKymaOp1ID := "upgradeKymaOperation1ID"
   332  		upgradeKymaOp1 := internal.UpgradeKymaOperation{
   333  			Operation: internal.Operation{
   334  				ID:              upgradeKymaOp1ID,
   335  				InstanceID:      instanceID,
   336  				OrchestrationID: orchestration1ID,
   337  				ProvisioningParameters: internal.ProvisioningParameters{
   338  					PlanID: broker.AzurePlanID,
   339  				},
   340  				RuntimeOperation: orchestration.RuntimeOperation{
   341  					ID: upgradeKymaOp1ID,
   342  				},
   343  			},
   344  		}
   345  
   346  		err = db.Operations().InsertUpgradeKymaOperation(upgradeKymaOp1)
   347  		require.NoError(t, err)
   348  
   349  		runtimeStateWithClusterSetupID := "runtimeStateWithClusterSetupID"
   350  		runtimeStateWithClusterSetup := internal.RuntimeState{
   351  			ID:          runtimeStateWithClusterSetupID,
   352  			RuntimeID:   uuid.NewString(),
   353  			OperationID: upgradeKymaOp1ID,
   354  			ClusterSetup: &reconcilerApi.Cluster{
   355  				RuntimeID: uuid.NewString(),
   356  				KymaConfig: reconcilerApi.KymaConfig{
   357  					Version: "2.0.0",
   358  					Profile: string(gqlschema.KymaProfileProduction),
   359  					Components: []reconcilerApi.Component{
   360  						{
   361  							URL:       "component1URL.local",
   362  							Component: "component1",
   363  							Namespace: "test",
   364  							Configuration: []reconcilerApi.Configuration{
   365  								{
   366  									Key:    "key1",
   367  									Value:  "value1",
   368  									Secret: false,
   369  								},
   370  								{
   371  									Key:    "key2",
   372  									Value:  "value2",
   373  									Secret: true,
   374  								},
   375  							},
   376  						},
   377  					},
   378  					Administrators: []string{"admin1@test.com", "admin2@test.com"},
   379  				},
   380  			},
   381  		}
   382  
   383  		err = db.RuntimeStates().Insert(runtimeStateWithClusterSetup)
   384  		require.NoError(t, err)
   385  
   386  		logs := logrus.New()
   387  		kymaHandler := NewOrchestrationStatusHandler(db.Operations(), db.Orchestrations(), db.RuntimeStates(), nil, nil, 100, logs)
   388  
   389  		urlPath := fmt.Sprintf("/orchestrations/%s/operations", orchestration1ID)
   390  		req, err := http.NewRequest("GET", urlPath, nil)
   391  		require.NoError(t, err)
   392  
   393  		rr := httptest.NewRecorder()
   394  		router := mux.NewRouter()
   395  		kymaHandler.AttachRoutes(router)
   396  
   397  		// when
   398  		router.ServeHTTP(rr, req)
   399  
   400  		// then
   401  		require.Equal(t, http.StatusOK, rr.Code)
   402  
   403  		var opResponseList orchestration.OperationResponseList
   404  
   405  		err = json.Unmarshal(rr.Body.Bytes(), &opResponseList)
   406  		require.NoError(t, err)
   407  
   408  		assert.Len(t, opResponseList.Data, 1)
   409  		assert.Equal(t, 1, opResponseList.TotalCount)
   410  		assert.Equal(t, 1, opResponseList.Count)
   411  
   412  		// given
   413  		urlPath = fmt.Sprintf("/orchestrations/%s/operations/%s", orchestration1ID, upgradeKymaOp1ID)
   414  		req, err = http.NewRequest(http.MethodGet, urlPath, nil)
   415  		require.NoError(t, err)
   416  
   417  		rr = httptest.NewRecorder()
   418  
   419  		// when
   420  		router.ServeHTTP(rr, req)
   421  
   422  		// then
   423  		require.Equal(t, http.StatusOK, rr.Code)
   424  
   425  		var opDetailResponse orchestration.OperationDetailResponse
   426  		err = json.Unmarshal(rr.Body.Bytes(), &opDetailResponse)
   427  		require.NoError(t, err)
   428  
   429  		expectedKymaConfig := gqlschema.KymaConfigInput{
   430  			Version: "2.0.0",
   431  			Profile: (*gqlschema.KymaProfile)(ptr.String("Production")),
   432  			Components: []*gqlschema.ComponentConfigurationInput{
   433  				{
   434  					Component: "component1",
   435  					Namespace: "test",
   436  					SourceURL: ptr.String("component1URL.local"),
   437  					Configuration: []*gqlschema.ConfigEntryInput{
   438  						{
   439  							Key:    "key1",
   440  							Value:  "value1",
   441  							Secret: ptr.Bool(false),
   442  						},
   443  						{
   444  							Key:    "key2",
   445  							Value:  "value2",
   446  							Secret: ptr.Bool(true),
   447  						},
   448  					},
   449  				},
   450  			},
   451  		}
   452  
   453  		assert.Equal(t, opDetailResponse.OrchestrationID, orchestration1ID)
   454  		assert.Equal(t, opDetailResponse.OperationID, upgradeKymaOp1ID)
   455  		assert.NotNil(t, opDetailResponse.KymaConfig)
   456  		assertKymaConfigValues(t, expectedKymaConfig, *opDetailResponse.KymaConfig)
   457  	})
   458  }
   459  
   460  func TestStatusRetryHandler_AttachRoutes(t *testing.T) {
   461  	fixID := "id-1"
   462  	t.Run("retry failed cluster orchestration with specified operations", func(t *testing.T) {
   463  		// given
   464  		db := storage.NewMemoryStorage()
   465  
   466  		orchestrationID := "orchestration-" + fixID
   467  		operationIDs := []string{"id-0", "id-1", "id-2", "id-3", "id-10"}
   468  		err := db.Orchestrations().Insert(internal.Orchestration{OrchestrationID: orchestrationID, State: orchestration.Failed, Type: orchestration.UpgradeClusterOrchestration})
   469  		require.NoError(t, err)
   470  
   471  		err = fixFailedOrchestrationOperations(db, orchestrationID, orchestration.UpgradeClusterOrchestration)
   472  		require.NoError(t, err)
   473  
   474  		// same instance but different same type newer operation
   475  		err = db.Orchestrations().Insert(internal.Orchestration{OrchestrationID: "Orchestration-id-4", State: orchestration.Failed, Type: orchestration.UpgradeClusterOrchestration})
   476  		// err = db.Orchestrations().Insert(internal.Orchestration{OrchestrationID: "Orchestration-id-4", State: orchestration.Canceling, Type: orchestration.UpgradeClusterOrchestration})
   477  		require.NoError(t, err)
   478  		sameInstOp := fixture.FixUpgradeClusterOperation("id-4", "instance-id-0")
   479  		sameInstOp.CreatedAt = time.Now().Add(time.Hour * 2)
   480  		sameInstOp.State = orchestration.Failed
   481  		// sameInstOp.State = orchestration.Canceled
   482  		err = db.Operations().InsertUpgradeClusterOperation(sameInstOp)
   483  		require.NoError(t, err)
   484  
   485  		logs := logrus.New()
   486  		clusterQueue := process.NewQueue(&testExecutor{}, logs)
   487  		kymaHandler := NewOrchestrationStatusHandler(db.Operations(), db.Orchestrations(), db.RuntimeStates(), nil, clusterQueue, 100, logs)
   488  
   489  		for i, id := range operationIDs {
   490  			operationIDs[i] = "operation-id=" + id
   491  		}
   492  		req, err := http.NewRequest("POST", fmt.Sprintf("/orchestrations/%s/retry", orchestrationID), strings.NewReader(strings.Join(operationIDs, "&")))
   493  		require.NoError(t, err)
   494  		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
   495  
   496  		rr := httptest.NewRecorder()
   497  		router := mux.NewRouter()
   498  		kymaHandler.AttachRoutes(router)
   499  
   500  		// when
   501  		router.ServeHTTP(rr, req)
   502  
   503  		// then
   504  		require.Equal(t, http.StatusAccepted, rr.Code)
   505  
   506  		var out orchestration.RetryResponse
   507  		expectedOut := orchestration.RetryResponse{
   508  			OrchestrationID: orchestrationID,
   509  			RetryShoots:     []string{"Shoot-instance-id-2"},
   510  			// if "Orchestration-id-4" is failed
   511  			OldOperations: []string{"id-0"},
   512  			// if "id-4" is canceled
   513  			// OldOperations:     nil,
   514  			InvalidOperations: []string{"id-1", "id-3", "id-10"},
   515  			Msg:               "retry operations are queued for processing",
   516  		}
   517  
   518  		err = json.Unmarshal(rr.Body.Bytes(), &out)
   519  		require.NoError(t, err)
   520  		assert.Equal(t, expectedOut, out)
   521  
   522  		o, err := db.Orchestrations().GetByID(orchestrationID)
   523  		require.NoError(t, err)
   524  		assert.Equal(t, orchestration.Retrying, o.State)
   525  
   526  		op, err := db.Operations().GetOperationByID("id-0")
   527  		require.NoError(t, err)
   528  		// if "Orchestration-id-4" is canceling
   529  		// assert.Equal(t, orchestration.Retrying, string(op.State))
   530  		// if "Orchestration-id-4" is failed
   531  		assert.Equal(t, orchestration.Failed, string(op.State))
   532  
   533  		op, err = db.Operations().GetOperationByID("id-1")
   534  		require.NoError(t, err)
   535  		assert.Equal(t, orchestration.Succeeded, string(op.State))
   536  
   537  		op, err = db.Operations().GetOperationByID("id-2")
   538  		require.NoError(t, err)
   539  		assert.Equal(t, orchestration.Failed, string(op.State))
   540  
   541  		op, err = db.Operations().GetOperationByID("id-3")
   542  		require.NoError(t, err)
   543  		assert.Equal(t, orchestration.Succeeded, string(op.State))
   544  	})
   545  
   546  	t.Run("retry failed kyma orchestration with specified operations", func(t *testing.T) {
   547  		// given
   548  		db := storage.NewMemoryStorage()
   549  
   550  		orchestrationID := "orchestration-" + fixID
   551  		operationIDs := []string{"id-0", "id-1", "id-2", "id-3", "id-10"}
   552  		err := db.Orchestrations().Insert(internal.Orchestration{OrchestrationID: orchestrationID, State: orchestration.Failed, Type: orchestration.UpgradeKymaOrchestration})
   553  		require.NoError(t, err)
   554  
   555  		err = fixFailedOrchestrationOperations(db, orchestrationID, orchestration.UpgradeKymaOrchestration)
   556  		require.NoError(t, err)
   557  
   558  		// same instance but different same type newer operation
   559  		err = db.Orchestrations().Insert(internal.Orchestration{OrchestrationID: "Orchestration-id-4", State: orchestration.Failed, Type: orchestration.UpgradeKymaOrchestration})
   560  		require.NoError(t, err)
   561  		sameInstOp := fixture.FixUpgradeKymaOperation("id-4", "instance-id-0")
   562  		sameInstOp.CreatedAt = time.Now().Add(time.Hour * 2)
   563  		sameInstOp.State = orchestration.Failed
   564  		err = db.Operations().InsertUpgradeKymaOperation(sameInstOp)
   565  		require.NoError(t, err)
   566  
   567  		logs := logrus.New()
   568  		kymaQueue := process.NewQueue(&testExecutor{}, logs)
   569  		kymaHandler := NewOrchestrationStatusHandler(db.Operations(), db.Orchestrations(), db.RuntimeStates(), kymaQueue, nil, 100, logs)
   570  
   571  		for i, id := range operationIDs {
   572  			operationIDs[i] = "operation-id=" + id
   573  		}
   574  		req, err := http.NewRequest("POST", fmt.Sprintf("/orchestrations/%s/retry", orchestrationID), strings.NewReader(strings.Join(operationIDs, "&")))
   575  		require.NoError(t, err)
   576  		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
   577  
   578  		rr := httptest.NewRecorder()
   579  		router := mux.NewRouter()
   580  		kymaHandler.AttachRoutes(router)
   581  
   582  		// when
   583  		router.ServeHTTP(rr, req)
   584  
   585  		// then
   586  		require.Equal(t, http.StatusAccepted, rr.Code)
   587  
   588  		var out orchestration.RetryResponse
   589  		expectedOut := orchestration.RetryResponse{
   590  			OrchestrationID:   orchestrationID,
   591  			RetryShoots:       []string{"Shoot-instance-id-2"},
   592  			OldOperations:     []string{"id-0"},
   593  			InvalidOperations: []string{"id-1", "id-3", "id-10"},
   594  			Msg:               "retry operations are queued for processing",
   595  		}
   596  
   597  		err = json.Unmarshal(rr.Body.Bytes(), &out)
   598  		require.NoError(t, err)
   599  		assert.Equal(t, expectedOut, out)
   600  
   601  		o, err := db.Orchestrations().GetByID(orchestrationID)
   602  		require.NoError(t, err)
   603  		assert.Equal(t, orchestration.Retrying, o.State)
   604  
   605  		op, err := db.Operations().GetOperationByID("id-0")
   606  		require.NoError(t, err)
   607  		assert.Equal(t, orchestration.Failed, string(op.State))
   608  
   609  		op, err = db.Operations().GetOperationByID("id-1")
   610  		require.NoError(t, err)
   611  		assert.Equal(t, orchestration.Succeeded, string(op.State))
   612  
   613  		op, err = db.Operations().GetOperationByID("id-2")
   614  		require.NoError(t, err)
   615  		assert.Equal(t, orchestration.Failed, string(op.State))
   616  
   617  		op, err = db.Operations().GetOperationByID("id-3")
   618  		require.NoError(t, err)
   619  		assert.Equal(t, orchestration.Succeeded, string(op.State))
   620  	})
   621  
   622  	t.Run("retry failed cluster orchestration without specified operations", func(t *testing.T) {
   623  		// given
   624  		db := storage.NewMemoryStorage()
   625  
   626  		orchestrationID := "orchestration-" + fixID
   627  		err := db.Orchestrations().Insert(internal.Orchestration{OrchestrationID: orchestrationID, State: orchestration.Failed, Type: orchestration.UpgradeClusterOrchestration})
   628  		require.NoError(t, err)
   629  
   630  		err = fixFailedOrchestrationOperations(db, orchestrationID, orchestration.UpgradeClusterOrchestration)
   631  		require.NoError(t, err)
   632  
   633  		// same instance but different same type newer operation
   634  		// err = db.Orchestrations().Insert(internal.Orchestration{OrchestrationID: "Orchestration-id-4", State: orchestration.Failed, Type: orchestration.UpgradeClusterOrchestration})
   635  		err = db.Orchestrations().Insert(internal.Orchestration{OrchestrationID: "Orchestration-id-4", State: orchestration.Canceling, Type: orchestration.UpgradeClusterOrchestration})
   636  		require.NoError(t, err)
   637  		sameInstOp := fixture.FixUpgradeClusterOperation("id-4", "instance-id-0")
   638  		sameInstOp.CreatedAt = time.Now().Add(time.Hour * 2)
   639  		sameInstOp.State = orchestration.Canceled
   640  		// sameInstOp.State = orchestration.Failed
   641  		err = db.Operations().InsertUpgradeClusterOperation(sameInstOp)
   642  		require.NoError(t, err)
   643  
   644  		logs := logrus.New()
   645  		clusterQueue := process.NewQueue(&testExecutor{}, logs)
   646  		kymaHandler := NewOrchestrationStatusHandler(db.Operations(), db.Orchestrations(), db.RuntimeStates(), nil, clusterQueue, 100, logs)
   647  
   648  		req, err := http.NewRequest("POST", fmt.Sprintf("/orchestrations/%s/retry", orchestrationID), nil)
   649  		require.NoError(t, err)
   650  		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
   651  
   652  		rr := httptest.NewRecorder()
   653  		router := mux.NewRouter()
   654  		kymaHandler.AttachRoutes(router)
   655  
   656  		// when
   657  		router.ServeHTTP(rr, req)
   658  
   659  		// then
   660  		require.Equal(t, http.StatusAccepted, rr.Code)
   661  
   662  		var out orchestration.RetryResponse
   663  		expectedOut := orchestration.RetryResponse{
   664  			OrchestrationID: orchestrationID,
   665  			// if "Orchestration-id-4" is failed
   666  			// if "id-4" is canceled
   667  			RetryShoots:       []string{"Shoot-instance-id-0", "Shoot-instance-id-2"},
   668  			OldOperations:     nil,
   669  			InvalidOperations: nil,
   670  			Msg:               "retry operations are queued for processing",
   671  		}
   672  
   673  		err = json.Unmarshal(rr.Body.Bytes(), &out)
   674  		require.NoError(t, err)
   675  		assert.Equal(t, expectedOut, out)
   676  
   677  		o, err := db.Orchestrations().GetByID(orchestrationID)
   678  		require.NoError(t, err)
   679  		assert.Equal(t, orchestration.Retrying, o.State)
   680  
   681  		op, err := db.Operations().GetOperationByID("id-0")
   682  		require.NoError(t, err)
   683  		// if "id-4" is canceled
   684  		assert.Equal(t, orchestration.Failed, string(op.State))
   685  		// if "Orchestration-id-4" is failed
   686  		// assert.Equal(t, orchestration.Failed, string(op.State))
   687  
   688  		op, err = db.Operations().GetOperationByID("id-1")
   689  		require.NoError(t, err)
   690  		assert.Equal(t, orchestration.Succeeded, string(op.State))
   691  
   692  		op, err = db.Operations().GetOperationByID("id-2")
   693  		require.NoError(t, err)
   694  		assert.Equal(t, orchestration.Failed, string(op.State))
   695  
   696  		op, err = db.Operations().GetOperationByID("id-3")
   697  		require.NoError(t, err)
   698  		assert.Equal(t, orchestration.Succeeded, string(op.State))
   699  	})
   700  
   701  	t.Run("retry failed cluster orchestration with deprovisioned instance", func(t *testing.T) {
   702  		// given
   703  		db := storage.NewMemoryStorage()
   704  
   705  		orchestrationID := "orchestration-" + fixID
   706  		err := db.Orchestrations().Insert(internal.Orchestration{OrchestrationID: orchestrationID, State: orchestration.Failed, Type: orchestration.UpgradeClusterOrchestration})
   707  		require.NoError(t, err)
   708  
   709  		err = fixFailedOrchestrationOperations(db, orchestrationID, orchestration.UpgradeClusterOrchestration)
   710  		require.NoError(t, err)
   711  
   712  		// same instance but different same type newer canceled operation
   713  		err = db.Orchestrations().Insert(internal.Orchestration{OrchestrationID: "Orchestration-id-4", State: orchestration.Canceling, Type: orchestration.UpgradeClusterOrchestration})
   714  		require.NoError(t, err)
   715  		sameInstOp := fixture.FixUpgradeClusterOperation("id-4", "instance-id-0")
   716  		sameInstOp.CreatedAt = time.Now().Add(time.Hour * 2)
   717  		sameInstOp.State = orchestration.Canceled
   718  		err = db.Operations().InsertUpgradeClusterOperation(sameInstOp)
   719  		require.NoError(t, err)
   720  
   721  		// insert a deprovisioned instance
   722  		deprovisioningOperation := fixture.FixDeprovisioningOperation("id-5", "instance-id-2")
   723  		deprovisioningOperation.State = orchestration.InProgress
   724  		err = db.Operations().InsertDeprovisioningOperation(deprovisioningOperation)
   725  		require.NoError(t, err)
   726  
   727  		logs := logrus.New()
   728  		clusterQueue := process.NewQueue(&testExecutor{}, logs)
   729  		kymaHandler := NewOrchestrationStatusHandler(db.Operations(), db.Orchestrations(), db.RuntimeStates(), nil, clusterQueue, 100, logs)
   730  
   731  		req, err := http.NewRequest("POST", fmt.Sprintf("/orchestrations/%s/retry", orchestrationID), nil)
   732  		require.NoError(t, err)
   733  		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
   734  
   735  		rr := httptest.NewRecorder()
   736  		router := mux.NewRouter()
   737  		kymaHandler.AttachRoutes(router)
   738  
   739  		// when
   740  		router.ServeHTTP(rr, req)
   741  
   742  		// then
   743  		require.Equal(t, http.StatusAccepted, rr.Code)
   744  
   745  		var out orchestration.RetryResponse
   746  		expectedOut := orchestration.RetryResponse{
   747  			OrchestrationID:   orchestrationID,
   748  			RetryShoots:       []string{"Shoot-instance-id-0", "Shoot-instance-id-2"},
   749  			OldOperations:     nil,
   750  			InvalidOperations: nil,
   751  			Msg:               "retry operations are queued for processing",
   752  		}
   753  
   754  		err = json.Unmarshal(rr.Body.Bytes(), &out)
   755  		require.NoError(t, err)
   756  		assert.Equal(t, expectedOut, out)
   757  
   758  		o, err := db.Orchestrations().GetByID(orchestrationID)
   759  		require.NoError(t, err)
   760  		assert.Equal(t, orchestration.Retrying, o.State)
   761  
   762  		op, err := db.Operations().GetOperationByID("id-0")
   763  		require.NoError(t, err)
   764  		assert.Equal(t, orchestration.Failed, string(op.State))
   765  
   766  		op, err = db.Operations().GetOperationByID("id-1")
   767  		require.NoError(t, err)
   768  		assert.Equal(t, orchestration.Succeeded, string(op.State))
   769  
   770  		op, err = db.Operations().GetOperationByID("id-2")
   771  		require.NoError(t, err)
   772  		assert.Equal(t, orchestration.Failed, string(op.State))
   773  
   774  		op, err = db.Operations().GetOperationByID("id-3")
   775  		require.NoError(t, err)
   776  		assert.Equal(t, orchestration.Succeeded, string(op.State))
   777  	})
   778  
   779  	t.Run("retry failed kyma orchestration with deprovisioned instance", func(t *testing.T) {
   780  		// given
   781  		db := storage.NewMemoryStorage()
   782  
   783  		orchestrationID := "orchestration-" + fixID
   784  		err := db.Orchestrations().Insert(internal.Orchestration{OrchestrationID: orchestrationID, State: orchestration.Failed, Type: orchestration.UpgradeKymaOrchestration})
   785  		require.NoError(t, err)
   786  
   787  		err = fixFailedOrchestrationOperations(db, orchestrationID, orchestration.UpgradeKymaOrchestration)
   788  		require.NoError(t, err)
   789  
   790  		// insert a deprovisioned instance
   791  		deprovisioningOperation := fixture.FixDeprovisioningOperation("id-5", "instance-id-0")
   792  		deprovisioningOperation.State = orchestration.InProgress
   793  		err = db.Operations().InsertDeprovisioningOperation(deprovisioningOperation)
   794  		require.NoError(t, err)
   795  
   796  		logs := logrus.New()
   797  		kymaQueue := process.NewQueue(&testExecutor{}, logs)
   798  		kymaHandler := NewOrchestrationStatusHandler(db.Operations(), db.Orchestrations(), db.RuntimeStates(), kymaQueue, nil, 100, logs)
   799  
   800  		req, err := http.NewRequest("POST", fmt.Sprintf("/orchestrations/%s/retry", orchestrationID), nil)
   801  		require.NoError(t, err)
   802  		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
   803  
   804  		rr := httptest.NewRecorder()
   805  		router := mux.NewRouter()
   806  		kymaHandler.AttachRoutes(router)
   807  
   808  		// when
   809  		router.ServeHTTP(rr, req)
   810  
   811  		// then
   812  		require.Equal(t, http.StatusAccepted, rr.Code)
   813  
   814  		var out orchestration.RetryResponse
   815  		expectedOut := orchestration.RetryResponse{
   816  			OrchestrationID:   orchestrationID,
   817  			RetryShoots:       []string{"Shoot-instance-id-0", "Shoot-instance-id-2"},
   818  			OldOperations:     nil,
   819  			InvalidOperations: nil,
   820  			Msg:               "retry operations are queued for processing",
   821  		}
   822  
   823  		err = json.Unmarshal(rr.Body.Bytes(), &out)
   824  		require.NoError(t, err)
   825  		assert.Equal(t, expectedOut, out)
   826  
   827  		o, err := db.Orchestrations().GetByID(orchestrationID)
   828  		require.NoError(t, err)
   829  		assert.Equal(t, orchestration.Retrying, o.State)
   830  
   831  		op, err := db.Operations().GetOperationByID("id-0")
   832  		require.NoError(t, err)
   833  		assert.Equal(t, orchestration.Failed, string(op.State))
   834  
   835  		op, err = db.Operations().GetOperationByID("id-1")
   836  		require.NoError(t, err)
   837  		assert.Equal(t, orchestration.Succeeded, string(op.State))
   838  
   839  		op, err = db.Operations().GetOperationByID("id-2")
   840  		require.NoError(t, err)
   841  		assert.Equal(t, orchestration.Failed, string(op.State))
   842  
   843  		op, err = db.Operations().GetOperationByID("id-3")
   844  		require.NoError(t, err)
   845  		assert.Equal(t, orchestration.Succeeded, string(op.State))
   846  	})
   847  
   848  	t.Run("retry in progress cluster orchestration without specified operations", func(t *testing.T) {
   849  		// given
   850  		db := storage.NewMemoryStorage()
   851  
   852  		orchestrationID := "orchestration-" + fixID
   853  		err := db.Orchestrations().Insert(internal.Orchestration{OrchestrationID: orchestrationID, State: orchestration.InProgress, Type: orchestration.UpgradeClusterOrchestration})
   854  		require.NoError(t, err)
   855  
   856  		err = fixInProgressOrchestrationOperations(db, orchestrationID)
   857  		require.NoError(t, err)
   858  
   859  		logs := logrus.New()
   860  		clusterQueue := process.NewQueue(&testExecutor{}, logs)
   861  		kymaHandler := NewOrchestrationStatusHandler(db.Operations(), db.Orchestrations(), db.RuntimeStates(), nil, clusterQueue, 100, logs)
   862  
   863  		req, err := http.NewRequest("POST", fmt.Sprintf("/orchestrations/%s/retry", orchestrationID), nil)
   864  		require.NoError(t, err)
   865  		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
   866  
   867  		rr := httptest.NewRecorder()
   868  		router := mux.NewRouter()
   869  		kymaHandler.AttachRoutes(router)
   870  
   871  		// when
   872  		router.ServeHTTP(rr, req)
   873  
   874  		// then
   875  		require.Equal(t, http.StatusAccepted, rr.Code)
   876  
   877  		var out orchestration.RetryResponse
   878  		expectedOut := orchestration.RetryResponse{
   879  			OrchestrationID:   orchestrationID,
   880  			RetryShoots:       []string{"Shoot-instance-id-2"},
   881  			OldOperations:     nil,
   882  			InvalidOperations: nil,
   883  			Msg:               "retry operations are queued for processing",
   884  		}
   885  
   886  		err = json.Unmarshal(rr.Body.Bytes(), &out)
   887  		require.NoError(t, err)
   888  		assert.Equal(t, expectedOut, out)
   889  
   890  		o, err := db.Orchestrations().GetByID(orchestrationID)
   891  		require.NoError(t, err)
   892  		assert.Equal(t, orchestration.InProgress, o.State)
   893  
   894  		op, err := db.Operations().GetOperationByID("id-0")
   895  		require.NoError(t, err)
   896  		assert.Equal(t, orchestration.InProgress, string(op.State))
   897  
   898  		op, err = db.Operations().GetOperationByID("id-1")
   899  		require.NoError(t, err)
   900  		assert.Equal(t, orchestration.Pending, string(op.State))
   901  
   902  		op, err = db.Operations().GetOperationByID("id-2")
   903  		require.NoError(t, err)
   904  		assert.Equal(t, orchestration.Failed, string(op.State))
   905  
   906  		op, err = db.Operations().GetOperationByID("id-3")
   907  		require.NoError(t, err)
   908  		assert.Equal(t, orchestration.Succeeded, string(op.State))
   909  	})
   910  
   911  	t.Run("retry failed kyma orchestration - testing scheduling", func(t *testing.T) {
   912  		// given
   913  		db := storage.NewMemoryStorage()
   914  
   915  		orchestrationID := "orchestration-" + fixID
   916  		operationIDs := []string{"id-2"}
   917  		err := db.Orchestrations().Insert(internal.Orchestration{OrchestrationID: orchestrationID, State: orchestration.Failed, Type: orchestration.UpgradeKymaOrchestration, Parameters: orchestration.Parameters{Strategy: orchestration.StrategySpec{Schedule: time.Now().Format(time.RFC3339), MaintenanceWindow: true}}})
   918  		require.NoError(t, err)
   919  
   920  		err = fixFailedOrchestrationOperations(db, orchestrationID, orchestration.UpgradeKymaOrchestration)
   921  		require.NoError(t, err)
   922  
   923  		logs := logrus.New()
   924  		kymaQueue := process.NewQueue(&testExecutor{}, logs)
   925  		kymaHandler := NewOrchestrationStatusHandler(db.Operations(), db.Orchestrations(), db.RuntimeStates(), kymaQueue, nil, 100, logs)
   926  
   927  		for i, id := range operationIDs {
   928  			operationIDs[i] = "operation-id=" + id
   929  		}
   930  		str := strings.Join(operationIDs, "&")
   931  		req, err := http.NewRequest("POST", fmt.Sprintf("/orchestrations/%s/retry", orchestrationID), strings.NewReader(str+"&immediate=true"))
   932  		require.NoError(t, err)
   933  		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
   934  
   935  		rr := httptest.NewRecorder()
   936  		router := mux.NewRouter()
   937  		kymaHandler.AttachRoutes(router)
   938  
   939  		// when
   940  		router.ServeHTTP(rr, req)
   941  
   942  		// then
   943  		require.Equal(t, http.StatusAccepted, rr.Code)
   944  
   945  		var out orchestration.RetryResponse
   946  		expectedOut := orchestration.RetryResponse{
   947  			OrchestrationID:   orchestrationID,
   948  			RetryShoots:       []string{"Shoot-instance-id-2"},
   949  			OldOperations:     nil,
   950  			InvalidOperations: nil,
   951  			Msg:               "retry operations are queued for processing",
   952  		}
   953  
   954  		err = json.Unmarshal(rr.Body.Bytes(), &out)
   955  		require.NoError(t, err)
   956  		assert.Equal(t, expectedOut, out)
   957  
   958  		op, err := db.Operations().GetUpgradeKymaOperationByID("id-2")
   959  		require.NoError(t, err)
   960  		assert.NotEqual(t, op.MaintenanceWindowBegin, time.Time{})
   961  
   962  		o, err := db.Orchestrations().GetByID(orchestrationID)
   963  		require.NoError(t, err)
   964  		assert.Equal(t, orchestration.Retrying, o.State)
   965  
   966  	})
   967  }
   968  
   969  func assertKymaConfigValues(t *testing.T, expected, actual gqlschema.KymaConfigInput) {
   970  	assert.Equal(t, expected.Version, actual.Version)
   971  	assert.Equal(t, *expected.Profile, *actual.Profile)
   972  	if len(expected.Components) > 0 {
   973  		for i, cmp := range expected.Components {
   974  			if len(cmp.Configuration) > 0 {
   975  				for j, cfg := range cmp.Configuration {
   976  					assert.Equal(t, cfg.Value, actual.Components[i].Configuration[j].Value)
   977  					assert.Equal(t, cfg.Key, actual.Components[i].Configuration[j].Key)
   978  					assert.Equal(t, *cfg.Secret, *actual.Components[i].Configuration[j].Secret)
   979  				}
   980  			}
   981  			assert.Equal(t, cmp.Component, actual.Components[i].Component)
   982  			assert.Equal(t, cmp.Namespace, actual.Components[i].Namespace)
   983  			if cmp.SourceURL != nil {
   984  				assert.Equal(t, *cmp.SourceURL, *actual.Components[i].SourceURL)
   985  			}
   986  		}
   987  	}
   988  }
   989  
   990  func fixFailedOrchestrationOperations(db storage.BrokerStorage, orchestrationID string, t orchestration.Type) error {
   991  	operationIDs := []string{"id-0", "id-1", "id-2", "id-3"} // in order: failed, succeeded
   992  	switch t {
   993  	case orchestration.UpgradeClusterOrchestration:
   994  		operations := []internal.UpgradeClusterOperation{}
   995  
   996  		for i, id := range operationIDs {
   997  			operations = append(operations, fixture.FixUpgradeClusterOperation(id, "instance-"+id))
   998  			operations[i].OrchestrationID = orchestrationID
   999  			if i%2 == 0 {
  1000  				operations[i].State = orchestration.Failed
  1001  				continue
  1002  			}
  1003  		}
  1004  
  1005  		for _, op := range operations {
  1006  			err := db.Operations().InsertUpgradeClusterOperation(op)
  1007  			if err != nil {
  1008  				return err
  1009  			}
  1010  		}
  1011  	case orchestration.UpgradeKymaOrchestration:
  1012  		operations := []internal.UpgradeKymaOperation{}
  1013  
  1014  		for i, id := range operationIDs {
  1015  			operations = append(operations, fixture.FixUpgradeKymaOperation(id, "instance-"+id))
  1016  			operations[i].OrchestrationID = orchestrationID
  1017  			if i%2 == 0 {
  1018  				operations[i].State = orchestration.Failed
  1019  				continue
  1020  			}
  1021  		}
  1022  
  1023  		for _, op := range operations {
  1024  			err := db.Operations().InsertUpgradeKymaOperation(op)
  1025  			if err != nil {
  1026  				return err
  1027  			}
  1028  		}
  1029  	}
  1030  
  1031  	return nil
  1032  }
  1033  
  1034  func fixInProgressOrchestrationOperations(db storage.BrokerStorage, orchestrationID string) error {
  1035  	operationIDs := []string{"id-0", "id-1", "id-2", "id-3"} // in order: in progress, pending, failed, succeeded
  1036  	operations := []internal.UpgradeClusterOperation{}
  1037  
  1038  	for i, id := range operationIDs {
  1039  		operations = append(operations, fixture.FixUpgradeClusterOperation(id, "instance-"+id))
  1040  		operations[i].OrchestrationID = orchestrationID
  1041  		if (i+4)%4 == 0 {
  1042  			operations[i].State = orchestration.InProgress
  1043  			continue
  1044  		}
  1045  		if (i+4)%5 == 0 {
  1046  			operations[i].State = orchestration.Pending
  1047  			continue
  1048  		}
  1049  		if (i+4)%6 == 0 {
  1050  			operations[i].State = orchestration.Failed
  1051  		}
  1052  
  1053  	}
  1054  	for _, op := range operations {
  1055  		err := db.Operations().InsertUpgradeClusterOperation(op)
  1056  		if err != nil {
  1057  			return err
  1058  		}
  1059  	}
  1060  
  1061  	return nil
  1062  }