github.com/kubeshop/testkube@v1.17.23/pkg/scheduler/testsuite_scheduler.go (about)

     1  package scheduler
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"os"
     7  	"sync"
     8  	"time"
     9  
    10  	"github.com/pkg/errors"
    11  
    12  	testsuitesv3 "github.com/kubeshop/testkube-operator/api/testsuite/v3"
    13  	"github.com/kubeshop/testkube/pkg/api/v1/testkube"
    14  	"github.com/kubeshop/testkube/pkg/event/bus"
    15  	testsuiteexecutionsmapper "github.com/kubeshop/testkube/pkg/mapper/testsuiteexecutions"
    16  	testsuitesmapper "github.com/kubeshop/testkube/pkg/mapper/testsuites"
    17  
    18  	"github.com/kubeshop/testkube/pkg/telemetry"
    19  	"github.com/kubeshop/testkube/pkg/version"
    20  	"github.com/kubeshop/testkube/pkg/workerpool"
    21  )
    22  
    23  const (
    24  	// DefaultConcurrencyLevel is a default concurrency level for worker pool
    25  	DefaultConcurrencyLevel = 10
    26  )
    27  
    28  type testTuple struct {
    29  	test        testkube.Test
    30  	executionID string
    31  	stepRequest *testkube.TestSuiteStepExecutionRequest
    32  }
    33  
    34  func (s *Scheduler) PrepareTestSuiteRequests(work []testsuitesv3.TestSuite, request testkube.TestSuiteExecutionRequest) []workerpool.Request[
    35  	testkube.TestSuite,
    36  	testkube.TestSuiteExecutionRequest,
    37  	testkube.TestSuiteExecution,
    38  ] {
    39  	requests := make([]workerpool.Request[testkube.TestSuite, testkube.TestSuiteExecutionRequest, testkube.TestSuiteExecution], len(work))
    40  	for i := range work {
    41  		requests[i] = workerpool.Request[testkube.TestSuite, testkube.TestSuiteExecutionRequest, testkube.TestSuiteExecution]{
    42  			Object:  testsuitesmapper.MapCRToAPI(work[i]),
    43  			Options: request,
    44  			ExecFn:  s.executeTestSuite,
    45  		}
    46  	}
    47  
    48  	return requests
    49  }
    50  
    51  func (s *Scheduler) executeTestSuite(ctx context.Context, testSuite testkube.TestSuite, request testkube.TestSuiteExecutionRequest) (
    52  	testsuiteExecution testkube.TestSuiteExecution, err error) {
    53  	s.logger.Debugw("Got testsuite to execute", "test", testSuite)
    54  	secretUUID, err := s.testSuitesClient.GetCurrentSecretUUID(testSuite.Name)
    55  	if err != nil {
    56  		return testsuiteExecution, err
    57  	}
    58  
    59  	request.SecretUUID = secretUUID
    60  	if testSuite.ExecutionRequest != nil {
    61  		if request.Timeout == 0 && testSuite.ExecutionRequest.Timeout != 0 {
    62  			request.Timeout = testSuite.ExecutionRequest.Timeout
    63  		}
    64  
    65  		var fields = []struct {
    66  			source      string
    67  			destination *string
    68  		}{
    69  			{
    70  				testSuite.ExecutionRequest.Name,
    71  				&request.Name,
    72  			},
    73  			{
    74  				testSuite.ExecutionRequest.HttpProxy,
    75  				&request.HttpProxy,
    76  			},
    77  			{
    78  				testSuite.ExecutionRequest.HttpsProxy,
    79  				&request.HttpsProxy,
    80  			},
    81  			{
    82  				testSuite.ExecutionRequest.JobTemplate,
    83  				&request.JobTemplate,
    84  			},
    85  			{
    86  				testSuite.ExecutionRequest.JobTemplateReference,
    87  				&request.JobTemplateReference,
    88  			},
    89  			{
    90  				testSuite.ExecutionRequest.ScraperTemplate,
    91  				&request.ScraperTemplate,
    92  			},
    93  			{
    94  				testSuite.ExecutionRequest.ScraperTemplateReference,
    95  				&request.ScraperTemplateReference,
    96  			},
    97  			{
    98  				testSuite.ExecutionRequest.PvcTemplate,
    99  				&request.PvcTemplate,
   100  			},
   101  			{
   102  				testSuite.ExecutionRequest.PvcTemplateReference,
   103  				&request.PvcTemplateReference,
   104  			},
   105  		}
   106  
   107  		for _, field := range fields {
   108  			if *field.destination == "" && field.source != "" {
   109  				*field.destination = field.source
   110  			}
   111  		}
   112  	}
   113  
   114  	s.logger.Infow("Executing testsuite", "test", testSuite.Name, "request", request, "ExecutionRequest", testSuite.ExecutionRequest)
   115  
   116  	request.Number = s.getNextExecutionNumber("ts-" + testSuite.Name)
   117  	if request.Name == "" {
   118  		request.Name = fmt.Sprintf("ts-%s-%d", testSuite.Name, request.Number)
   119  	}
   120  
   121  	testsuiteExecution = testkube.NewStartedTestSuiteExecution(testSuite, request)
   122  	err = s.testsuiteResults.Insert(ctx, testsuiteExecution)
   123  	if err != nil {
   124  		s.logger.Infow("Inserting test execution", "error", err)
   125  	}
   126  
   127  	s.events.Notify(testkube.NewEventStartTestSuite(&testsuiteExecution))
   128  
   129  	var wg sync.WaitGroup
   130  	wg.Add(1)
   131  	go s.runSteps(ctx, &wg, &testsuiteExecution, request)
   132  
   133  	// wait for sync test suite execution
   134  	if request.Sync {
   135  		wg.Wait()
   136  	}
   137  
   138  	return testsuiteExecution, nil
   139  }
   140  
   141  func (s *Scheduler) runSteps(ctx context.Context, wg *sync.WaitGroup, testsuiteExecution *testkube.TestSuiteExecution, request testkube.TestSuiteExecutionRequest) {
   142  	defer s.runAfterEachStep(ctx, testsuiteExecution, wg)
   143  
   144  	s.logger.Infow("Running steps", "test", testsuiteExecution.Name)
   145  
   146  	statusChan := make(chan *testkube.TestSuiteExecutionStatus)
   147  	hasFailedSteps := false
   148  	cancelSteps := false
   149  	var batchStepResult *testkube.TestSuiteBatchStepExecutionResult
   150  
   151  	var abortionStatus *testkube.TestSuiteExecutionStatus
   152  
   153  	go s.timeoutCheck(ctx, testsuiteExecution, request.Timeout)
   154  
   155  	err := s.eventsBus.SubscribeTopic(bus.InternalSubscribeTopic, testsuiteExecution.Name, func(event testkube.Event) error {
   156  		s.logger.Infow("test suite abortion event in runSteps", "event", event)
   157  		if event.TestSuiteExecution != nil &&
   158  			event.TestSuiteExecution.Id == testsuiteExecution.Id &&
   159  			event.Type_ != nil &&
   160  			(*event.Type_ == testkube.END_TESTSUITE_ABORTED_EventType || *event.Type_ == testkube.END_TESTSUITE_TIMEOUT_EventType) {
   161  			s.logger.Infow("Aborting test suite execution", "execution", testsuiteExecution.Id)
   162  
   163  			status := testkube.TestSuiteExecutionStatusAborting
   164  			if *event.Type_ == testkube.END_TESTSUITE_TIMEOUT_EventType {
   165  				status = testkube.TestSuiteExecutionStatusTimeout
   166  			}
   167  			statusChan <- status
   168  		}
   169  		return nil
   170  	})
   171  
   172  	if err != nil {
   173  		s.logger.Errorw("error subscribing to event", "error", err)
   174  	}
   175  
   176  	for i := range testsuiteExecution.ExecuteStepResults {
   177  		batchStepResult = &testsuiteExecution.ExecuteStepResults[i]
   178  		s.logger.Debugw("Running batch step", "step", batchStepResult.Execute, "i", i)
   179  
   180  		select {
   181  		case status := <-statusChan:
   182  			abortionStatus = status
   183  			cancelSteps = true
   184  		default:
   185  		}
   186  
   187  		if cancelSteps {
   188  			s.logger.Infow("Aborting batch step", "step", batchStepResult.Execute, "i", i)
   189  			for j := range batchStepResult.Execute {
   190  				if batchStepResult.Execute[j].Execution != nil && batchStepResult.Execute[j].Execution.ExecutionResult != nil {
   191  					batchStepResult.Execute[j].Execution.ExecutionResult.Abort()
   192  				}
   193  			}
   194  
   195  			testsuiteExecution.Status = testkube.TestSuiteExecutionStatusAborting
   196  
   197  			for j := range batchStepResult.Execute {
   198  				if batchStepResult.Execute[j].Execution != nil && batchStepResult.Execute[j].Execution.ExecutionResult != nil {
   199  					batchStepResult.Execute[j].Execution.ExecutionResult.Abort()
   200  				}
   201  			}
   202  
   203  			continue
   204  		}
   205  
   206  		// start execution of given step
   207  		for j := range batchStepResult.Execute {
   208  			if batchStepResult.Execute[j].Execution != nil && batchStepResult.Execute[j].Execution.ExecutionResult != nil {
   209  				batchStepResult.Execute[j].Execution.ExecutionResult.InProgress()
   210  			}
   211  		}
   212  
   213  		err := s.testsuiteResults.Update(ctx, *testsuiteExecution)
   214  		if err != nil {
   215  			s.logger.Infow("Updating test execution", "error", err)
   216  		}
   217  
   218  		s.executeTestStep(ctx, *testsuiteExecution, request, batchStepResult, testsuiteExecution.ExecuteStepResults[:i])
   219  
   220  		var results []*testkube.ExecutionResult
   221  		for j := range batchStepResult.Execute {
   222  			if batchStepResult.Execute[j].Execution != nil && batchStepResult.Execute[j].Execution.ExecutionResult != nil {
   223  				results = append(results, batchStepResult.Execute[j].Execution.ExecutionResult)
   224  			}
   225  		}
   226  
   227  		s.logger.Debugw("Batch step execution result", "step", batchStepResult.Execute, "results", results)
   228  
   229  		err = s.testsuiteResults.Update(ctx, *testsuiteExecution)
   230  		if err != nil {
   231  			s.logger.Errorw("saving test suite execution results error", "error", err)
   232  
   233  			hasFailedSteps = true
   234  			continue
   235  		}
   236  
   237  		for j := range batchStepResult.Execute {
   238  			if batchStepResult.Execute[j].IsFailed() {
   239  				hasFailedSteps = true
   240  				if batchStepResult.Step != nil && batchStepResult.Step.StopOnFailure {
   241  					cancelSteps = true
   242  					break
   243  				}
   244  			}
   245  		}
   246  	}
   247  	s.logger.Infow("Finished running steps", "test", testsuiteExecution.Name, "hasFailedSteps", hasFailedSteps, "cancelSteps", cancelSteps, "status", testsuiteExecution.Status)
   248  
   249  	if testsuiteExecution.Status != nil && *testsuiteExecution.Status == testkube.ABORTING_TestSuiteExecutionStatus {
   250  		if abortionStatus != nil && *abortionStatus == testkube.TIMEOUT_TestSuiteExecutionStatus {
   251  			s.events.Notify(testkube.NewEventEndTestSuiteTimeout(testsuiteExecution))
   252  			testsuiteExecution.Status = testkube.TestSuiteExecutionStatusTimeout
   253  		} else {
   254  			s.events.Notify(testkube.NewEventEndTestSuiteAborted(testsuiteExecution))
   255  			testsuiteExecution.Status = testkube.TestSuiteExecutionStatusAborted
   256  		}
   257  	} else if hasFailedSteps {
   258  		testsuiteExecution.Status = testkube.TestSuiteExecutionStatusFailed
   259  		s.events.Notify(testkube.NewEventEndTestSuiteFailed(testsuiteExecution))
   260  	} else {
   261  		testsuiteExecution.Status = testkube.TestSuiteExecutionStatusPassed
   262  		s.events.Notify(testkube.NewEventEndTestSuiteSuccess(testsuiteExecution))
   263  	}
   264  
   265  	s.metrics.IncAndObserveExecuteTestSuite(*testsuiteExecution, s.dashboardURI)
   266  
   267  	err = s.testsuiteResults.Update(ctx, *testsuiteExecution)
   268  	if err != nil {
   269  		s.logger.Errorw("saving final test suite execution result error", "error", err)
   270  	}
   271  
   272  	s.eventsBus.Unsubscribe(testsuiteExecution.Name)
   273  }
   274  
   275  func (s *Scheduler) runAfterEachStep(ctx context.Context, execution *testkube.TestSuiteExecution, wg *sync.WaitGroup) {
   276  	execution.Stop()
   277  	err := s.testsuiteResults.EndExecution(ctx, *execution)
   278  	if err != nil {
   279  		s.logger.Errorw("error setting end time", "error", err.Error())
   280  	}
   281  
   282  	wg.Done()
   283  
   284  	if execution.TestSuite != nil {
   285  		testSuite, err := s.testSuitesClient.Get(execution.TestSuite.Name)
   286  		if err != nil {
   287  			s.logger.Errorw("getting test suite error", "error", err)
   288  		}
   289  
   290  		if testSuite != nil {
   291  			testSuite.Status = testsuitesmapper.MapExecutionToTestSuiteStatus(execution)
   292  			if err = s.testSuitesClient.UpdateStatus(testSuite); err != nil {
   293  				s.logger.Errorw("updating test suite error", "error", err)
   294  			}
   295  
   296  			if execution.TestSuiteExecutionName != "" {
   297  				testSuiteExecution, err := s.testSuiteExecutionsClient.Get(execution.TestSuiteExecutionName)
   298  				if err != nil {
   299  					s.logger.Errorw("getting test suite execution error", "error", err)
   300  				}
   301  
   302  				if testSuiteExecution != nil {
   303  					testSuiteExecution.Status = testsuiteexecutionsmapper.MapAPIToCRD(execution, testSuiteExecution.Generation)
   304  					if err = s.testSuiteExecutionsClient.UpdateStatus(testSuiteExecution); err != nil {
   305  						s.logger.Errorw("updating test suite execution error", "error", err)
   306  					}
   307  				}
   308  			}
   309  		}
   310  	}
   311  
   312  	telemetryEnabled, err := s.configMap.GetTelemetryEnabled(ctx)
   313  	if err != nil {
   314  		s.logger.Debugw("getting telemetry enabled error", "error", err)
   315  	}
   316  
   317  	if !telemetryEnabled {
   318  		return
   319  	}
   320  
   321  	clusterID, err := s.configMap.GetUniqueClusterId(ctx)
   322  	if err != nil {
   323  		s.logger.Debugw("getting cluster id error", "error", err)
   324  	}
   325  
   326  	host, err := os.Hostname()
   327  	if err != nil {
   328  		s.logger.Debugw("getting hostname error", "hostname", host, "error", err)
   329  	}
   330  
   331  	status := ""
   332  	if execution.Status != nil {
   333  		status = string(*execution.Status)
   334  	}
   335  
   336  	out, err := telemetry.SendRunEvent("testkube_api_run_test_suite", telemetry.RunParams{
   337  		AppVersion: version.Version,
   338  		Host:       host,
   339  		ClusterID:  clusterID,
   340  		DurationMs: execution.DurationMs,
   341  		Status:     status,
   342  	})
   343  
   344  	if err != nil {
   345  		s.logger.Debugw("sending run test suite telemetry event error", "error", err)
   346  	} else {
   347  		s.logger.Debugw("sending run test suite telemetry event", "output", out)
   348  	}
   349  }
   350  
   351  // timeoutCheck is checking if the testsuite has timed out
   352  func (s *Scheduler) timeoutCheck(ctx context.Context, testsuiteExecution *testkube.TestSuiteExecution, timeout int32) {
   353  	s.logger.Infow("timeout check started", "test", testsuiteExecution.Name, "timeout", timeout)
   354  
   355  	timer := time.NewTimer(time.Duration(timeout) * time.Second)
   356  
   357  	defer func() {
   358  		timer.Stop()
   359  	}()
   360  
   361  	for testsuiteExecution.Status == testkube.TestSuiteExecutionStatusRunning {
   362  		select {
   363  		case <-timer.C:
   364  			s.logger.Debugw("testsuite timeout occured", "test suite", testsuiteExecution.Name)
   365  
   366  			if timeout > 0 {
   367  				s.logger.Debugw("aborting test suite execution due to timeout", "execution", testsuiteExecution.Id)
   368  
   369  				err := s.eventsBus.PublishTopic(bus.InternalPublishTopic, testkube.NewEventEndTestSuiteTimeout(testsuiteExecution))
   370  				if err != nil {
   371  					s.logger.Errorw("error publishing event", "error", err)
   372  				}
   373  				return
   374  			}
   375  		case <-ctx.Done():
   376  			return
   377  		}
   378  	}
   379  
   380  	s.logger.Debugw("Timeout check, finished checking", "test", testsuiteExecution.Name)
   381  }
   382  
   383  func (s *Scheduler) executeTestStep(ctx context.Context, testsuiteExecution testkube.TestSuiteExecution,
   384  	request testkube.TestSuiteExecutionRequest, result *testkube.TestSuiteBatchStepExecutionResult,
   385  	previousSteps []testkube.TestSuiteBatchStepExecutionResult) {
   386  
   387  	var testSuiteName string
   388  	if testsuiteExecution.TestSuite != nil {
   389  		testSuiteName = testsuiteExecution.TestSuite.Name
   390  	}
   391  
   392  	ids := make(map[string]struct{})
   393  	testNames := make(map[string]struct{})
   394  	if result.Step != nil && result.Step.DownloadArtifacts != nil {
   395  		for _, testName := range result.Step.DownloadArtifacts.PreviousTestNames {
   396  			testNames[testName] = struct{}{}
   397  		}
   398  
   399  		for i := range previousSteps {
   400  			for j := range previousSteps[i].Execute {
   401  				if previousSteps[i].Execute[j].Execution != nil &&
   402  					previousSteps[i].Execute[j].Step != nil && previousSteps[i].Execute[j].Step.Test != "" {
   403  					if previousSteps[i].Execute[j].Execution.IsPassed() || previousSteps[i].Execute[j].Execution.IsFailed() {
   404  						if result.Step.DownloadArtifacts.AllPreviousSteps {
   405  							ids[previousSteps[i].Execute[j].Execution.Id] = struct{}{}
   406  						} else {
   407  							for _, n := range result.Step.DownloadArtifacts.PreviousStepNumbers {
   408  								if n == int32(i+1) {
   409  									ids[previousSteps[i].Execute[j].Execution.Id] = struct{}{}
   410  									break
   411  								}
   412  							}
   413  
   414  							if _, ok := testNames[previousSteps[i].Execute[j].Step.Test]; ok {
   415  								ids[previousSteps[i].Execute[j].Execution.Id] = struct{}{}
   416  							}
   417  						}
   418  					}
   419  				}
   420  			}
   421  		}
   422  	}
   423  
   424  	var testTuples []testTuple
   425  	var duration time.Duration
   426  	for i := range result.Execute {
   427  		step := result.Execute[i].Step
   428  		if step == nil {
   429  			continue
   430  		}
   431  
   432  		l := s.logger.With("type", step.Type(), "testSuiteName", testSuiteName, "name", step.FullName())
   433  
   434  		switch step.Type() {
   435  		case testkube.TestSuiteStepTypeExecuteTest:
   436  			executeTestStep := step.Test
   437  			if executeTestStep == "" {
   438  				continue
   439  			}
   440  
   441  			execution := result.Execute[i].Execution
   442  			if execution == nil {
   443  				continue
   444  			}
   445  
   446  			l.Info("executing test", "variables", testsuiteExecution.Variables, "request", request)
   447  
   448  			testTuples = append(testTuples, testTuple{
   449  				test:        testkube.Test{Name: executeTestStep, Namespace: testsuiteExecution.TestSuite.Namespace},
   450  				executionID: execution.Id,
   451  				stepRequest: step.ExecutionRequest,
   452  			})
   453  		case testkube.TestSuiteStepTypeDelay:
   454  			if step.Delay == "" {
   455  				continue
   456  			}
   457  
   458  			l.Infow("delaying execution", "step", step.FullName(), "delay", step.Delay)
   459  
   460  			delay, err := time.ParseDuration(step.Delay)
   461  			if err != nil {
   462  				result.Execute[i].Err(err)
   463  				continue
   464  			}
   465  
   466  			if delay > duration {
   467  				duration = delay
   468  			}
   469  		default:
   470  			result.Execute[i].Err(errors.Errorf("can't find handler for execution step type: '%v'", step.Type()))
   471  		}
   472  	}
   473  
   474  	concurrencyLevel := DefaultConcurrencyLevel
   475  	if request.ConcurrencyLevel != 0 {
   476  		concurrencyLevel = int(request.ConcurrencyLevel)
   477  	}
   478  
   479  	workerpoolService := workerpool.New[testkube.Test, testkube.ExecutionRequest, testkube.Execution](concurrencyLevel)
   480  
   481  	if len(testTuples) != 0 {
   482  		var executionIDs []string
   483  		for id := range ids {
   484  			executionIDs = append(executionIDs, id)
   485  		}
   486  
   487  		req := testkube.ExecutionRequest{
   488  			TestSuiteName:         testSuiteName,
   489  			Variables:             testsuiteExecution.Variables,
   490  			TestSuiteSecretUUID:   request.SecretUUID,
   491  			Sync:                  true,
   492  			HttpProxy:             request.HttpProxy,
   493  			HttpsProxy:            request.HttpsProxy,
   494  			ExecutionLabels:       request.ExecutionLabels,
   495  			ActiveDeadlineSeconds: int64(request.Timeout),
   496  			ContentRequest:        request.ContentRequest,
   497  			RunningContext: &testkube.RunningContext{
   498  				Type_:   string(testkube.RunningContextTypeTestSuite),
   499  				Context: testsuiteExecution.Name,
   500  			},
   501  			JobTemplate:                  request.JobTemplate,
   502  			JobTemplateReference:         request.JobTemplateReference,
   503  			ScraperTemplate:              request.ScraperTemplate,
   504  			ScraperTemplateReference:     request.ScraperTemplateReference,
   505  			PvcTemplate:                  request.PvcTemplate,
   506  			PvcTemplateReference:         request.PvcTemplateReference,
   507  			DownloadArtifactExecutionIDs: executionIDs,
   508  		}
   509  
   510  		requests := make([]workerpool.Request[testkube.Test, testkube.ExecutionRequest, testkube.Execution], len(testTuples))
   511  		for i := range testTuples {
   512  			req.Name = fmt.Sprintf("%s-%s", testSuiteName, testTuples[i].test.Name)
   513  			req.Id = testTuples[i].executionID
   514  			req = MergeStepRequest(testTuples[i].stepRequest, req)
   515  			requests[i] = workerpool.Request[testkube.Test, testkube.ExecutionRequest, testkube.Execution]{
   516  				Object:  testTuples[i].test,
   517  				Options: req,
   518  				ExecFn:  s.executeTest,
   519  			}
   520  		}
   521  
   522  		go workerpoolService.SendRequests(requests)
   523  		go workerpoolService.Run(ctx)
   524  	}
   525  
   526  	result.Start()
   527  	if err := s.testsuiteResults.Update(ctx, testsuiteExecution); err != nil {
   528  		s.logger.Errorw("saving test suite execution start time error", "error", err)
   529  	}
   530  
   531  	if duration != 0 {
   532  		s.delayWithAbortionCheck(duration, testsuiteExecution.Id, result)
   533  	}
   534  
   535  	if len(testTuples) != 0 {
   536  		for r := range workerpoolService.GetResponses() {
   537  			status := ""
   538  			if r.Result.ExecutionResult != nil && r.Result.ExecutionResult.Status != nil {
   539  				status = string(*r.Result.ExecutionResult.Status)
   540  			}
   541  
   542  			s.logger.Infow("execution result", "id", r.Result.Id, "status", status)
   543  			value := r.Result
   544  			for i := range result.Execute {
   545  				if result.Execute[i].Execution == nil {
   546  					continue
   547  				}
   548  
   549  				if result.Execute[i].Execution.Id == r.Result.Id {
   550  					result.Execute[i].Execution = &value
   551  
   552  					if err := s.testsuiteResults.Update(ctx, testsuiteExecution); err != nil {
   553  						s.logger.Errorw("saving test suite execution results error", "error", err)
   554  					}
   555  				}
   556  			}
   557  		}
   558  	}
   559  
   560  	result.Stop()
   561  	if err := s.testsuiteResults.Update(ctx, testsuiteExecution); err != nil {
   562  		s.logger.Errorw("saving test suite execution end time error", "error", err)
   563  	}
   564  }
   565  
   566  func (s *Scheduler) delayWithAbortionCheck(duration time.Duration, testSuiteId string, result *testkube.TestSuiteBatchStepExecutionResult) {
   567  	timer := time.NewTimer(duration)
   568  
   569  	defer func() {
   570  		timer.Stop()
   571  	}()
   572  
   573  	abortChan := make(chan bool)
   574  
   575  	err := s.eventsBus.SubscribeTopic(bus.InternalSubscribeTopic, testSuiteId, func(event testkube.Event) error {
   576  		s.logger.Infow("test suite abortion event in delay handling", "event", event)
   577  		if event.TestSuiteExecution != nil &&
   578  			event.TestSuiteExecution.Id == testSuiteId &&
   579  			event.Type_ != nil &&
   580  			*event.Type_ == testkube.END_TESTSUITE_ABORTED_EventType {
   581  
   582  			s.logger.Infow("delay aborted", "testSuiteId", testSuiteId, "duration", duration)
   583  			abortChan <- true
   584  		}
   585  		return nil
   586  	})
   587  
   588  	if err != nil {
   589  		s.logger.Errorw("error subscribing to event", "error", err)
   590  	}
   591  
   592  	for {
   593  		select {
   594  		case <-timer.C:
   595  			s.logger.Infow("delay finished", "testSuiteId", testSuiteId, "duration", duration)
   596  
   597  			for i := range result.Execute {
   598  				if result.Execute[i].Step != nil && result.Execute[i].Step.Delay != "" &&
   599  					result.Execute[i].Execution != nil && result.Execute[i].Execution.ExecutionResult != nil {
   600  					result.Execute[i].Execution.ExecutionResult.Success()
   601  				}
   602  			}
   603  
   604  			return
   605  		case <-abortChan:
   606  
   607  			for i := range result.Execute {
   608  				if result.Execute[i].Step != nil && result.Execute[i].Step.Delay != "" &&
   609  					result.Execute[i].Execution != nil && result.Execute[i].Execution.ExecutionResult != nil {
   610  					delay, err := time.ParseDuration(result.Execute[i].Step.Delay)
   611  					if err != nil {
   612  						result.Execute[i].Err(err)
   613  						continue
   614  					}
   615  
   616  					if delay < duration {
   617  						result.Execute[i].Execution.ExecutionResult.Success()
   618  						continue
   619  					}
   620  
   621  					result.Execute[i].Execution.ExecutionResult.Abort()
   622  				}
   623  			}
   624  			return
   625  		}
   626  	}
   627  }
   628  
   629  // MergeStepRequest inherits step request fields with execution request
   630  func MergeStepRequest(stepRequest *testkube.TestSuiteStepExecutionRequest, executionRequest testkube.ExecutionRequest) testkube.ExecutionRequest {
   631  	if stepRequest == nil {
   632  		return executionRequest
   633  	}
   634  	if stepRequest.ExecutionLabels != nil {
   635  		executionRequest.ExecutionLabels = stepRequest.ExecutionLabels
   636  	}
   637  
   638  	if stepRequest.Variables != nil {
   639  		executionRequest.Variables = mergeVariables(executionRequest.Variables, stepRequest.Variables)
   640  	}
   641  
   642  	if len(stepRequest.Args) != 0 {
   643  		if stepRequest.ArgsMode == string(testkube.ArgsModeTypeAppend) || stepRequest.ArgsMode == "" {
   644  			executionRequest.Args = append(executionRequest.Args, stepRequest.Args...)
   645  		}
   646  
   647  		if stepRequest.ArgsMode == string(testkube.ArgsModeTypeOverride) || stepRequest.ArgsMode == string(testkube.ArgsModeTypeReplace) {
   648  			executionRequest.Args = stepRequest.Args
   649  		}
   650  	}
   651  
   652  	if stepRequest.Command != nil {
   653  		executionRequest.Command = stepRequest.Command
   654  	}
   655  	executionRequest.HttpProxy = setStringField(executionRequest.HttpProxy, stepRequest.HttpProxy)
   656  	executionRequest.HttpsProxy = setStringField(executionRequest.HttpsProxy, stepRequest.HttpsProxy)
   657  	executionRequest.CronJobTemplate = setStringField(executionRequest.CronJobTemplate, stepRequest.CronJobTemplate)
   658  	executionRequest.CronJobTemplateReference = setStringField(executionRequest.CronJobTemplateReference, stepRequest.CronJobTemplateReference)
   659  	executionRequest.JobTemplate = setStringField(executionRequest.JobTemplate, stepRequest.JobTemplate)
   660  	executionRequest.JobTemplateReference = setStringField(executionRequest.JobTemplateReference, stepRequest.JobTemplateReference)
   661  	executionRequest.ScraperTemplate = setStringField(executionRequest.ScraperTemplate, stepRequest.ScraperTemplate)
   662  	executionRequest.ScraperTemplateReference = setStringField(executionRequest.ScraperTemplateReference, stepRequest.ScraperTemplateReference)
   663  	executionRequest.PvcTemplate = setStringField(executionRequest.PvcTemplate, stepRequest.PvcTemplate)
   664  	executionRequest.PvcTemplateReference = setStringField(executionRequest.PvcTemplate, stepRequest.PvcTemplateReference)
   665  
   666  	if stepRequest.RunningContext != nil {
   667  		executionRequest.RunningContext = &testkube.RunningContext{
   668  			Type_:   string(stepRequest.RunningContext.Type_),
   669  			Context: stepRequest.RunningContext.Context,
   670  		}
   671  	}
   672  
   673  	return executionRequest
   674  }
   675  
   676  func setStringField(oldValue string, newValue string) string {
   677  	if newValue != "" {
   678  		return newValue
   679  	}
   680  	return oldValue
   681  }