github.com/kubevela/workflow@v0.6.0/controllers/workflow_test.go (about)

     1  /*
     2  Copyright 2022 The KubeVela Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package controllers
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"path/filepath"
    23  	sysruntime "runtime"
    24  	"strings"
    25  	"testing"
    26  	"time"
    27  
    28  	. "github.com/onsi/ginkgo"
    29  	. "github.com/onsi/gomega"
    30  	appsv1 "k8s.io/api/apps/v1"
    31  	corev1 "k8s.io/api/core/v1"
    32  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    33  	"k8s.io/apimachinery/pkg/runtime"
    34  	"k8s.io/apimachinery/pkg/types"
    35  	utilfeature "k8s.io/apiserver/pkg/util/feature"
    36  	featuregatetesting "k8s.io/component-base/featuregate/testing"
    37  	"sigs.k8s.io/controller-runtime/pkg/client"
    38  	"sigs.k8s.io/controller-runtime/pkg/reconcile"
    39  
    40  	"github.com/kubevela/pkg/util/test/definition"
    41  
    42  	"github.com/kubevela/workflow/api/v1alpha1"
    43  	"github.com/kubevela/workflow/pkg/debug"
    44  	"github.com/kubevela/workflow/pkg/features"
    45  	wfTypes "github.com/kubevela/workflow/pkg/types"
    46  	"github.com/kubevela/workflow/pkg/utils"
    47  )
    48  
    49  var _ = Describe("Test Workflow", func() {
    50  	ctx := context.Background()
    51  	namespace := "test-ns"
    52  
    53  	wrTemplate := &v1alpha1.WorkflowRun{
    54  		TypeMeta: metav1.TypeMeta{
    55  			Kind:       "WorkflowRun",
    56  			APIVersion: "core.oam.dev/v1alpha1",
    57  		},
    58  		ObjectMeta: metav1.ObjectMeta{
    59  			Name:      "wr",
    60  			Namespace: namespace,
    61  		},
    62  		Spec: v1alpha1.WorkflowRunSpec{
    63  			WorkflowSpec: &v1alpha1.WorkflowSpec{
    64  				Steps: []v1alpha1.WorkflowStep{
    65  					{
    66  						WorkflowStepBase: v1alpha1.WorkflowStepBase{
    67  							Name: "step-1",
    68  							Type: "suspend",
    69  						},
    70  					},
    71  				},
    72  			},
    73  		},
    74  	}
    75  	testDefinitions := []string{"test-apply", "apply-object", "failed-render", "suspend-and-deploy", "multi-suspend", "save-process-context"}
    76  
    77  	BeforeEach(func() {
    78  		setupNamespace(ctx, namespace)
    79  		setupTestDefinitions(ctx, testDefinitions, namespace)
    80  		By("[TEST] Set up definitions before integration test")
    81  	})
    82  
    83  	AfterEach(func() {
    84  		Expect(k8sClient.DeleteAllOf(ctx, &v1alpha1.WorkflowRun{}, client.InNamespace(namespace))).Should(Succeed())
    85  		Expect(k8sClient.DeleteAllOf(ctx, &appsv1.Deployment{}, client.InNamespace(namespace))).Should(Succeed())
    86  		Expect(k8sClient.DeleteAllOf(ctx, &corev1.ConfigMap{}, client.InNamespace(namespace))).Should(Succeed())
    87  	})
    88  
    89  	It("get steps from workflow ref", func() {
    90  		workflow := &v1alpha1.Workflow{
    91  			TypeMeta: metav1.TypeMeta{
    92  				Kind:       "Workflow",
    93  				APIVersion: "core.oam.dev/v1alpha1",
    94  			},
    95  			ObjectMeta: metav1.ObjectMeta{
    96  				Name:      "workflow",
    97  				Namespace: namespace,
    98  			},
    99  			Mode: &v1alpha1.WorkflowExecuteMode{
   100  				Steps: v1alpha1.WorkflowModeDAG,
   101  			},
   102  			WorkflowSpec: v1alpha1.WorkflowSpec{
   103  				Steps: []v1alpha1.WorkflowStep{
   104  					{
   105  						WorkflowStepBase: v1alpha1.WorkflowStepBase{
   106  							Name: "step-1",
   107  							Type: "suspend",
   108  						},
   109  					},
   110  				},
   111  			},
   112  		}
   113  		wr := wrTemplate.DeepCopy()
   114  		wr.Spec = v1alpha1.WorkflowRunSpec{
   115  			WorkflowRef: "workflow",
   116  		}
   117  		Expect(k8sClient.Create(ctx, workflow)).Should(BeNil())
   118  		Expect(k8sClient.Create(ctx, wr)).Should(BeNil())
   119  
   120  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   121  
   122  		wrObj := &v1alpha1.WorkflowRun{}
   123  		Expect(k8sClient.Get(ctx, client.ObjectKey{
   124  			Name:      wr.Name,
   125  			Namespace: wr.Namespace,
   126  		}, wrObj)).Should(BeNil())
   127  
   128  		Expect(wrObj.Status.Suspend).Should(BeTrue())
   129  		Expect(wrObj.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateSuspending))
   130  		Expect(wrObj.Status.Mode.Steps).Should(BeEquivalentTo(v1alpha1.WorkflowModeDAG))
   131  		Expect(wrObj.Status.Mode.SubSteps).Should(BeEquivalentTo(v1alpha1.WorkflowModeDAG))
   132  
   133  		wr2 := wrTemplate.DeepCopy()
   134  		wr2.Name = "wr-template-with-mode"
   135  		wr2.Spec = v1alpha1.WorkflowRunSpec{
   136  			WorkflowRef: "workflow",
   137  			Mode: &v1alpha1.WorkflowExecuteMode{
   138  				Steps:    v1alpha1.WorkflowModeStep,
   139  				SubSteps: v1alpha1.WorkflowModeStep,
   140  			},
   141  		}
   142  		Expect(k8sClient.Create(ctx, wr2)).Should(BeNil())
   143  
   144  		tryReconcile(reconciler, wr2.Name, wr2.Namespace)
   145  
   146  		wrObj = &v1alpha1.WorkflowRun{}
   147  		Expect(k8sClient.Get(ctx, client.ObjectKey{
   148  			Name:      wr2.Name,
   149  			Namespace: wr2.Namespace,
   150  		}, wrObj)).Should(BeNil())
   151  
   152  		Expect(wrObj.Status.Suspend).Should(BeTrue())
   153  		Expect(wrObj.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateSuspending))
   154  		Expect(wrObj.Status.Mode.Steps).Should(BeEquivalentTo(v1alpha1.WorkflowModeStep))
   155  		Expect(wrObj.Status.Mode.SubSteps).Should(BeEquivalentTo(v1alpha1.WorkflowModeStep))
   156  	})
   157  
   158  	It("get failed to generate", func() {
   159  		wr := wrTemplate.DeepCopy()
   160  		wr.Name = "failed-generate"
   161  		wr.Spec.WorkflowSpec.Steps = []v1alpha1.WorkflowStep{
   162  			{
   163  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
   164  					Name: "failed-generate",
   165  					Type: "invalid",
   166  				},
   167  			}}
   168  		Expect(k8sClient.Create(ctx, wr)).Should(BeNil())
   169  
   170  		err := reconcileWithReturn(reconciler, wr.Name, wr.Namespace)
   171  		Expect(err).ShouldNot(BeNil())
   172  
   173  		events, err := recorder.GetEventsWithName(wr.Name)
   174  		Expect(err).Should(BeNil())
   175  		Expect(len(events)).Should(Equal(1))
   176  		Expect(events[0].EventType).Should(Equal(corev1.EventTypeWarning))
   177  		Expect(events[0].Reason).Should(Equal(v1alpha1.ReasonGenerate))
   178  		Expect(events[0].Message).Should(ContainSubstring(v1alpha1.MessageFailedGenerate))
   179  	})
   180  
   181  	It("should create workflow context ConfigMap", func() {
   182  		wr := wrTemplate.DeepCopy()
   183  		Expect(k8sClient.Create(ctx, wr)).Should(BeNil())
   184  
   185  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   186  
   187  		cm := &corev1.ConfigMap{}
   188  		Expect(k8sClient.Get(ctx, client.ObjectKey{
   189  			Name:      "workflow-wr-context",
   190  			Namespace: namespace,
   191  		}, cm)).Should(BeNil())
   192  	})
   193  
   194  	It("test workflow suspend", func() {
   195  		wr := wrTemplate.DeepCopy()
   196  		wr.Name = "test-wr-suspend"
   197  		Expect(k8sClient.Create(ctx, wr)).Should(BeNil())
   198  
   199  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   200  
   201  		wrObj := &v1alpha1.WorkflowRun{}
   202  		Expect(k8sClient.Get(ctx, client.ObjectKey{
   203  			Name:      wr.Name,
   204  			Namespace: wr.Namespace,
   205  		}, wrObj)).Should(BeNil())
   206  
   207  		Expect(wrObj.Status.Suspend).Should(BeTrue())
   208  		Expect(wrObj.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateSuspending))
   209  		Expect(wrObj.Status.Steps[0].Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStepPhaseSuspending))
   210  		Expect(wrObj.Status.Steps[0].ID).ShouldNot(BeEquivalentTo(""))
   211  		// resume
   212  		Expect(utils.ResumeWorkflow(ctx, k8sClient, wrObj, "")).Should(BeNil())
   213  		Expect(wrObj.Status.Suspend).Should(BeFalse())
   214  
   215  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   216  
   217  		Expect(k8sClient.Get(ctx, client.ObjectKey{
   218  			Name:      wr.Name,
   219  			Namespace: wr.Namespace,
   220  		}, wrObj)).Should(BeNil())
   221  		Expect(wrObj.Status.Suspend).Should(BeFalse())
   222  		Expect(wrObj.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateSucceeded))
   223  	})
   224  
   225  	It("test workflow suspend in sub steps", func() {
   226  		wr := wrTemplate.DeepCopy()
   227  		wr.Name = "test-wr-sub-suspend"
   228  		wr.Spec.WorkflowSpec.Steps = []v1alpha1.WorkflowStep{
   229  			{
   230  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
   231  					Name: "group",
   232  					Type: "step-group",
   233  				},
   234  				SubSteps: []v1alpha1.WorkflowStepBase{
   235  					{
   236  						Name: "suspend",
   237  						Type: "suspend",
   238  					},
   239  					{
   240  						Name:       "step1",
   241  						Type:       "test-apply",
   242  						Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
   243  					},
   244  				},
   245  			}}
   246  		Expect(k8sClient.Create(ctx, wr)).Should(BeNil())
   247  
   248  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   249  
   250  		wrObj := &v1alpha1.WorkflowRun{}
   251  		Expect(k8sClient.Get(ctx, client.ObjectKey{
   252  			Name:      wr.Name,
   253  			Namespace: wr.Namespace,
   254  		}, wrObj)).Should(BeNil())
   255  
   256  		Expect(wrObj.Status.Suspend).Should(BeTrue())
   257  		Expect(wrObj.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateSuspending))
   258  		Expect(wrObj.Status.Steps[0].SubStepsStatus[0].Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStepPhaseSuspending))
   259  		Expect(wrObj.Status.Steps[0].SubStepsStatus[0].ID).ShouldNot(BeEquivalentTo(""))
   260  		// resume
   261  		Expect(utils.ResumeWorkflow(ctx, k8sClient, wrObj, "")).Should(BeNil())
   262  		Expect(wrObj.Status.Suspend).Should(BeFalse())
   263  		expDeployment := &appsv1.Deployment{}
   264  		step1Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step1"}
   265  		Expect(k8sClient.Get(ctx, step1Key, expDeployment)).Should(BeNil())
   266  		expDeployment.Status.Replicas = 1
   267  		expDeployment.Status.ReadyReplicas = 1
   268  		expDeployment.Status.Conditions = []appsv1.DeploymentCondition{{
   269  			Message: "hello",
   270  		}}
   271  		Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
   272  
   273  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   274  
   275  		Expect(k8sClient.Get(ctx, client.ObjectKey{
   276  			Name:      wr.Name,
   277  			Namespace: wr.Namespace,
   278  		}, wrObj)).Should(BeNil())
   279  		Expect(wrObj.Status.Suspend).Should(BeFalse())
   280  		Expect(wrObj.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateSucceeded))
   281  	})
   282  
   283  	It("test workflow terminate a suspend workflow", func() {
   284  		wr := wrTemplate.DeepCopy()
   285  		wr.Name = "test-terminate-suspend-wr"
   286  		wr.Spec.WorkflowSpec.Steps = []v1alpha1.WorkflowStep{
   287  			{
   288  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
   289  					Name: "suspend",
   290  					Type: "suspend",
   291  				},
   292  			},
   293  			{
   294  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
   295  					Name: "suspend-1",
   296  					Type: "suspend",
   297  				},
   298  			}}
   299  		Expect(k8sClient.Create(ctx, wr)).Should(BeNil())
   300  
   301  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   302  
   303  		wrObj := &v1alpha1.WorkflowRun{}
   304  		Expect(k8sClient.Get(ctx, client.ObjectKey{
   305  			Name:      wr.Name,
   306  			Namespace: wr.Namespace,
   307  		}, wrObj)).Should(BeNil())
   308  
   309  		Expect(wrObj.Status.Suspend).Should(BeTrue())
   310  		Expect(wrObj.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateSuspending))
   311  
   312  		// terminate the workflow
   313  		Expect(utils.TerminateWorkflow(ctx, k8sClient, wrObj)).Should(BeNil())
   314  
   315  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   316  
   317  		Expect(k8sClient.Get(ctx, client.ObjectKey{
   318  			Name:      wr.Name,
   319  			Namespace: wr.Namespace,
   320  		}, wrObj)).Should(BeNil())
   321  
   322  		Expect(wrObj.Status.Suspend).Should(BeFalse())
   323  		Expect(wrObj.Status.Terminated).Should(BeTrue())
   324  		Expect(wrObj.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateTerminated))
   325  	})
   326  
   327  	It("test input/output in step mode", func() {
   328  		wr := wrTemplate.DeepCopy()
   329  		wr.Name = "wr-with-inout"
   330  		wr.Spec.WorkflowSpec.Steps = []v1alpha1.WorkflowStep{
   331  			{
   332  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
   333  					Name:       "step1",
   334  					Type:       "test-apply",
   335  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
   336  					Outputs: v1alpha1.StepOutputs{
   337  						{
   338  							Name:      "message",
   339  							ValueFrom: `"message: " +output.value.status.conditions[0].message`,
   340  						},
   341  					},
   342  				},
   343  			},
   344  			{
   345  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
   346  					Name:       "step2",
   347  					Type:       "test-apply",
   348  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox","message":"test"}`)},
   349  					Inputs: v1alpha1.StepInputs{
   350  						{
   351  							From:         "message",
   352  							ParameterKey: "message",
   353  						},
   354  					},
   355  				},
   356  			}}
   357  
   358  		Expect(k8sClient.Create(context.Background(), wr)).Should(BeNil())
   359  		wrKey := types.NamespacedName{Namespace: wr.Namespace, Name: wr.Name}
   360  
   361  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   362  
   363  		checkRun := &v1alpha1.WorkflowRun{}
   364  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
   365  		Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateExecuting))
   366  		expDeployment := &appsv1.Deployment{}
   367  		step2Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step2"}
   368  		Expect(k8sClient.Get(ctx, step2Key, expDeployment)).Should(utils.NotFoundMatcher{})
   369  		step1Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step1"}
   370  		Expect(k8sClient.Get(ctx, step1Key, expDeployment)).Should(BeNil())
   371  
   372  		expDeployment.Status.Replicas = 1
   373  		expDeployment.Status.ReadyReplicas = 1
   374  		expDeployment.Status.Conditions = []appsv1.DeploymentCondition{{
   375  			Message: "hello",
   376  		}}
   377  		Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
   378  
   379  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   380  
   381  		Expect(k8sClient.Get(ctx, step2Key, expDeployment)).Should(BeNil())
   382  		Expect(expDeployment.Spec.Template.Spec.Containers[0].Env[0].Value).Should(Equal("message: hello"))
   383  		expDeployment.Status.Replicas = 1
   384  		expDeployment.Status.ReadyReplicas = 1
   385  		Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
   386  
   387  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   388  
   389  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
   390  		Expect(checkRun.Status.Mode.Steps).Should(BeEquivalentTo(v1alpha1.WorkflowModeStep))
   391  		Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateSucceeded))
   392  	})
   393  
   394  	It("test input/output in dag mode", func() {
   395  		wr := wrTemplate.DeepCopy()
   396  		wr.Name = "wr-with-inout"
   397  		wr.Spec.WorkflowSpec.Steps = []v1alpha1.WorkflowStep{
   398  			{
   399  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
   400  					Name:       "step2",
   401  					Type:       "test-apply",
   402  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
   403  					Inputs: v1alpha1.StepInputs{
   404  						{
   405  							From:         "message",
   406  							ParameterKey: "message",
   407  						},
   408  					},
   409  				},
   410  			},
   411  			{
   412  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
   413  					Name:       "step1",
   414  					Type:       "test-apply",
   415  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
   416  					Outputs: v1alpha1.StepOutputs{
   417  						{
   418  							Name:      "message",
   419  							ValueFrom: `"message: " +output.value.status.conditions[0].message`,
   420  						},
   421  					},
   422  				},
   423  			},
   424  		}
   425  		wr.Spec.Mode = &v1alpha1.WorkflowExecuteMode{
   426  			Steps: v1alpha1.WorkflowModeDAG,
   427  		}
   428  
   429  		Expect(k8sClient.Create(context.Background(), wr)).Should(BeNil())
   430  		wrKey := types.NamespacedName{Namespace: wr.Namespace, Name: wr.Name}
   431  
   432  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   433  
   434  		checkRun := &v1alpha1.WorkflowRun{}
   435  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
   436  		Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateExecuting))
   437  		expDeployment := &appsv1.Deployment{}
   438  		step2Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step2"}
   439  		Expect(k8sClient.Get(ctx, step2Key, expDeployment)).Should(utils.NotFoundMatcher{})
   440  		step1Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step1"}
   441  		Expect(k8sClient.Get(ctx, step1Key, expDeployment)).Should(BeNil())
   442  		expDeployment.Status.Replicas = 1
   443  		expDeployment.Status.ReadyReplicas = 1
   444  		expDeployment.Status.Conditions = []appsv1.DeploymentCondition{{
   445  			Message: "hello",
   446  		}}
   447  		Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
   448  
   449  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   450  
   451  		Expect(k8sClient.Get(ctx, step2Key, expDeployment)).Should(BeNil())
   452  		Expect(expDeployment.Spec.Template.Spec.Containers[0].Env[0].Value).Should(Equal("message: hello"))
   453  		expDeployment.Status.Replicas = 1
   454  		expDeployment.Status.ReadyReplicas = 1
   455  		Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
   456  
   457  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   458  
   459  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
   460  		Expect(checkRun.Status.Mode.Steps).Should(BeEquivalentTo(v1alpha1.WorkflowModeDAG))
   461  		Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateSucceeded))
   462  	})
   463  
   464  	It("test depends on in step mode", func() {
   465  		wr := wrTemplate.DeepCopy()
   466  		wr.Name = "wr-depends-on"
   467  		wr.Spec.WorkflowSpec.Steps = []v1alpha1.WorkflowStep{
   468  			{
   469  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
   470  					Name:       "step1",
   471  					Type:       "test-apply",
   472  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
   473  				},
   474  			},
   475  			{
   476  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
   477  					Name:       "step2",
   478  					Type:       "test-apply",
   479  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
   480  					DependsOn:  []string{"step1"},
   481  				},
   482  			},
   483  		}
   484  
   485  		Expect(k8sClient.Create(context.Background(), wr)).Should(BeNil())
   486  		wrKey := types.NamespacedName{Namespace: wr.Namespace, Name: wr.Name}
   487  
   488  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   489  
   490  		expDeployment := &appsv1.Deployment{}
   491  		step1Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step1"}
   492  		step2Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step2"}
   493  		Expect(k8sClient.Get(ctx, step2Key, expDeployment)).Should(utils.NotFoundMatcher{})
   494  
   495  		checkRun := &v1alpha1.WorkflowRun{}
   496  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
   497  		Expect(k8sClient.Get(ctx, step1Key, expDeployment)).Should(BeNil())
   498  
   499  		expDeployment.Status.Replicas = 1
   500  		expDeployment.Status.ReadyReplicas = 1
   501  		Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
   502  
   503  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   504  
   505  		expDeployment = &appsv1.Deployment{}
   506  		Expect(k8sClient.Get(ctx, step2Key, expDeployment)).Should(BeNil())
   507  		expDeployment.Status.Replicas = 1
   508  		expDeployment.Status.ReadyReplicas = 1
   509  		Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
   510  
   511  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   512  
   513  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
   514  		Expect(checkRun.Status.Mode.Steps).Should(BeEquivalentTo(v1alpha1.WorkflowModeStep))
   515  		Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateSucceeded))
   516  	})
   517  
   518  	It("test depends on in dag mode", func() {
   519  		wr := wrTemplate.DeepCopy()
   520  		wr.Name = "wr-depends-on"
   521  		wr.Spec.WorkflowSpec.Steps = []v1alpha1.WorkflowStep{
   522  			{
   523  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
   524  					Name:       "step2",
   525  					Type:       "test-apply",
   526  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
   527  					DependsOn:  []string{"step1"},
   528  				},
   529  			},
   530  			{
   531  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
   532  					Name:       "step1",
   533  					Type:       "test-apply",
   534  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
   535  				},
   536  			},
   537  			{
   538  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
   539  					Name: "step3",
   540  					Type: "suspend",
   541  					If:   "false",
   542  				},
   543  			},
   544  			{
   545  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
   546  					Name:       "step4",
   547  					Type:       "test-apply",
   548  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
   549  					DependsOn:  []string{"step3"},
   550  				},
   551  			},
   552  		}
   553  		wr.Spec.Mode = &v1alpha1.WorkflowExecuteMode{
   554  			Steps: v1alpha1.WorkflowModeDAG,
   555  		}
   556  
   557  		Expect(k8sClient.Create(context.Background(), wr)).Should(BeNil())
   558  		wrKey := types.NamespacedName{Namespace: wr.Namespace, Name: wr.Name}
   559  
   560  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   561  
   562  		expDeployment := &appsv1.Deployment{}
   563  		step1Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step1"}
   564  		step2Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step2"}
   565  		step4Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step4"}
   566  		Expect(k8sClient.Get(ctx, step2Key, expDeployment)).Should(utils.NotFoundMatcher{})
   567  
   568  		checkRun := &v1alpha1.WorkflowRun{}
   569  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
   570  		Expect(k8sClient.Get(ctx, step1Key, expDeployment)).Should(BeNil())
   571  
   572  		expDeployment.Status.Replicas = 1
   573  		expDeployment.Status.ReadyReplicas = 1
   574  		Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
   575  
   576  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   577  
   578  		expDeployment = &appsv1.Deployment{}
   579  		Expect(k8sClient.Get(ctx, step2Key, expDeployment)).Should(BeNil())
   580  		expDeployment.Status.Replicas = 1
   581  		expDeployment.Status.ReadyReplicas = 1
   582  		Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
   583  
   584  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   585  
   586  		Expect(k8sClient.Get(ctx, step4Key, expDeployment)).Should(utils.NotFoundMatcher{})
   587  
   588  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
   589  		Expect(checkRun.Status.Mode.Steps).Should(BeEquivalentTo(v1alpha1.WorkflowModeDAG))
   590  		Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateSucceeded))
   591  	})
   592  
   593  	It("test failed after retries in step mode with suspend on failure", func() {
   594  		defer featuregatetesting.SetFeatureGateDuringTest(&testing.T{}, utilfeature.DefaultFeatureGate, features.EnableSuspendOnFailure, true)()
   595  		wr := wrTemplate.DeepCopy()
   596  		wr.Name = "wr-failed-after-retries"
   597  		wr.Spec.WorkflowSpec.Steps = []v1alpha1.WorkflowStep{
   598  			{
   599  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
   600  					Name:       "step1",
   601  					Type:       "test-apply",
   602  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
   603  				},
   604  			},
   605  			{
   606  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
   607  					Name:       "step2-failed",
   608  					Type:       "apply-object",
   609  					Properties: &runtime.RawExtension{Raw: []byte(`{"value":[{"apiVersion":"v1","kind":"invalid","metadata":{"name":"test1"}}]}`)},
   610  				},
   611  			},
   612  		}
   613  
   614  		Expect(k8sClient.Create(ctx, wr)).Should(BeNil())
   615  		wrKey := types.NamespacedName{Namespace: wr.Namespace, Name: wr.Name}
   616  		checkRun := &v1alpha1.WorkflowRun{}
   617  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
   618  
   619  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   620  
   621  		expDeployment := &appsv1.Deployment{}
   622  		step1Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step1"}
   623  		Expect(k8sClient.Get(ctx, step1Key, expDeployment)).Should(BeNil())
   624  
   625  		expDeployment.Status.Replicas = 1
   626  		expDeployment.Status.ReadyReplicas = 1
   627  		Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
   628  
   629  		By("verify the first ten reconciles")
   630  		for i := 0; i < wfTypes.MaxWorkflowStepErrorRetryTimes; i++ {
   631  			tryReconcile(reconciler, wr.Name, wr.Namespace)
   632  			Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
   633  			Expect(checkRun.Status.Message).Should(BeEquivalentTo(""))
   634  			Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateExecuting))
   635  			Expect(checkRun.Status.Steps[1].Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStepPhaseFailed))
   636  		}
   637  
   638  		By("workflowrun should be suspended after failed max reconciles")
   639  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   640  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
   641  		Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateSuspending))
   642  		Expect(checkRun.Status.Message).Should(BeEquivalentTo(wfTypes.MessageSuspendFailedAfterRetries))
   643  		Expect(checkRun.Status.Steps[1].Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStepPhaseFailed))
   644  		Expect(checkRun.Status.Steps[1].Reason).Should(BeEquivalentTo(wfTypes.StatusReasonFailedAfterRetries))
   645  
   646  		By("resume the suspended workflow run")
   647  		Expect(utils.ResumeWorkflow(ctx, k8sClient, checkRun, "")).Should(BeNil())
   648  
   649  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   650  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
   651  		Expect(checkRun.Status.Message).Should(BeEquivalentTo(""))
   652  		Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateExecuting))
   653  		Expect(checkRun.Status.Steps[1].Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStepPhaseFailed))
   654  	})
   655  
   656  	It("test reconcile with patch status at once", func() {
   657  		defer featuregatetesting.SetFeatureGateDuringTest(&testing.T{}, utilfeature.DefaultFeatureGate, features.EnableSuspendOnFailure, true)()
   658  		defer featuregatetesting.SetFeatureGateDuringTest(&testing.T{}, utilfeature.DefaultFeatureGate, features.EnablePatchStatusAtOnce, true)()
   659  		wr := wrTemplate.DeepCopy()
   660  		wr.Name = "wr-failed-after-retries"
   661  		wr.Spec.WorkflowSpec.Steps = []v1alpha1.WorkflowStep{
   662  			{
   663  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
   664  					Name:       "step1",
   665  					Type:       "test-apply",
   666  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
   667  				},
   668  			},
   669  			{
   670  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
   671  					Name:       "step2-failed",
   672  					Type:       "apply-object",
   673  					Properties: &runtime.RawExtension{Raw: []byte(`{"value":[{"apiVersion":"v1","kind":"invalid","metadata":{"name":"test1"}}]}`)},
   674  				},
   675  			},
   676  		}
   677  
   678  		Expect(k8sClient.Create(ctx, wr)).Should(BeNil())
   679  		wrKey := types.NamespacedName{Namespace: wr.Namespace, Name: wr.Name}
   680  		checkRun := &v1alpha1.WorkflowRun{}
   681  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
   682  
   683  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   684  
   685  		expDeployment := &appsv1.Deployment{}
   686  		step1Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step1"}
   687  		Expect(k8sClient.Get(ctx, step1Key, expDeployment)).Should(BeNil())
   688  
   689  		expDeployment.Status.Replicas = 1
   690  		expDeployment.Status.ReadyReplicas = 1
   691  		Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
   692  
   693  		By("verify the first ten reconciles")
   694  		for i := 0; i < wfTypes.MaxWorkflowStepErrorRetryTimes; i++ {
   695  			tryReconcile(reconciler, wr.Name, wr.Namespace)
   696  			Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
   697  			Expect(checkRun.Status.Message).Should(BeEquivalentTo(""))
   698  			Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateExecuting))
   699  			Expect(checkRun.Status.Steps[1].Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStepPhaseFailed))
   700  		}
   701  
   702  		By("workflowrun should be suspended after failed max reconciles")
   703  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   704  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
   705  		Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateSuspending))
   706  		Expect(checkRun.Status.Message).Should(BeEquivalentTo(wfTypes.MessageSuspendFailedAfterRetries))
   707  		Expect(checkRun.Status.Steps[1].Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStepPhaseFailed))
   708  		Expect(checkRun.Status.Steps[1].Reason).Should(BeEquivalentTo(wfTypes.StatusReasonFailedAfterRetries))
   709  
   710  		By("resume the suspended workflow run")
   711  		Expect(utils.ResumeWorkflow(ctx, k8sClient, checkRun, "")).Should(BeNil())
   712  
   713  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   714  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
   715  		Expect(checkRun.Status.Message).Should(BeEquivalentTo(""))
   716  		Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateExecuting))
   717  		Expect(checkRun.Status.Steps[1].Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStepPhaseFailed))
   718  	})
   719  
   720  	It("test failed after retries in dag mode with running step and suspend on failure", func() {
   721  		defer featuregatetesting.SetFeatureGateDuringTest(&testing.T{}, utilfeature.DefaultFeatureGate, features.EnableSuspendOnFailure, true)()
   722  		wr := wrTemplate.DeepCopy()
   723  		wr.Name = "wr-failed-after-retries"
   724  		wr.Spec.WorkflowSpec.Steps = []v1alpha1.WorkflowStep{
   725  			{
   726  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
   727  					Name:       "step1",
   728  					Type:       "test-apply",
   729  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
   730  				},
   731  			},
   732  			{
   733  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
   734  					Name:       "step2-failed",
   735  					Type:       "apply-object",
   736  					Properties: &runtime.RawExtension{Raw: []byte(`{"value":[{"apiVersion":"v1","kind":"invalid","metadata":{"name":"test1"}}]}`)},
   737  				},
   738  			},
   739  		}
   740  		wr.Spec.Mode = &v1alpha1.WorkflowExecuteMode{
   741  			Steps: v1alpha1.WorkflowModeDAG,
   742  		}
   743  
   744  		Expect(k8sClient.Create(ctx, wr)).Should(BeNil())
   745  		wrKey := types.NamespacedName{Namespace: wr.Namespace, Name: wr.Name}
   746  		checkRun := &v1alpha1.WorkflowRun{}
   747  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
   748  
   749  		By("verify the first ten reconciles")
   750  		for i := 0; i < wfTypes.MaxWorkflowStepErrorRetryTimes; i++ {
   751  			tryReconcile(reconciler, wr.Name, wr.Namespace)
   752  			Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
   753  			Expect(checkRun.Status.Message).Should(BeEquivalentTo(""))
   754  			Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateExecuting))
   755  			Expect(checkRun.Status.Steps[1].Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStepPhaseFailed))
   756  		}
   757  
   758  		By("workflowrun should not be suspended after failed max reconciles because of running step")
   759  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   760  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
   761  		Expect(checkRun.Status.Message).Should(BeEquivalentTo(""))
   762  		Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateExecuting))
   763  		Expect(checkRun.Status.Steps[1].Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStepPhaseFailed))
   764  		Expect(checkRun.Status.Steps[1].Reason).Should(BeEquivalentTo(wfTypes.StatusReasonFailedAfterRetries))
   765  	})
   766  
   767  	It("test failed render", func() {
   768  		wr := wrTemplate.DeepCopy()
   769  		wr.Name = "wr-failed-render"
   770  		wr.Spec.WorkflowSpec.Steps = []v1alpha1.WorkflowStep{
   771  			{
   772  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
   773  					Name:       "step1",
   774  					Type:       "failed-render",
   775  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
   776  				},
   777  			},
   778  			{
   779  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
   780  					Name:       "step2",
   781  					Type:       "test-apply",
   782  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
   783  				},
   784  			},
   785  		}
   786  
   787  		Expect(k8sClient.Create(ctx, wr)).Should(BeNil())
   788  		wrKey := types.NamespacedName{Namespace: wr.Namespace, Name: wr.Name}
   789  		checkRun := &v1alpha1.WorkflowRun{}
   790  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
   791  
   792  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   793  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
   794  		Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateFailed))
   795  		Expect(checkRun.Status.Steps[0].Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStepPhaseFailed))
   796  		Expect(checkRun.Status.Steps[0].Reason).Should(BeEquivalentTo(wfTypes.StatusReasonRendering))
   797  		Expect(checkRun.Status.Steps[1].Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStepPhaseSkipped))
   798  	})
   799  
   800  	It("test workflow run with mode", func() {
   801  		wr := wrTemplate.DeepCopy()
   802  		wr.Name = "wr-mode"
   803  		wr.Spec.WorkflowSpec.Steps = []v1alpha1.WorkflowStep{
   804  			{
   805  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
   806  					Name:       "step1",
   807  					Type:       "test-apply",
   808  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
   809  				},
   810  			},
   811  			{
   812  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
   813  					Name: "group",
   814  					Type: "step-group",
   815  				},
   816  				SubSteps: []v1alpha1.WorkflowStepBase{
   817  					{
   818  						Name:       "step2",
   819  						Type:       "test-apply",
   820  						Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
   821  					},
   822  					{
   823  						Name:       "step3",
   824  						Type:       "test-apply",
   825  						Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
   826  					},
   827  				},
   828  			},
   829  		}
   830  		wr.Spec.Mode = &v1alpha1.WorkflowExecuteMode{
   831  			Steps:    v1alpha1.WorkflowModeDAG,
   832  			SubSteps: v1alpha1.WorkflowModeStep,
   833  		}
   834  
   835  		Expect(k8sClient.Create(context.Background(), wr)).Should(BeNil())
   836  		wrKey := types.NamespacedName{Namespace: wr.Namespace, Name: wr.Name}
   837  
   838  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   839  
   840  		expDeployment := &appsv1.Deployment{}
   841  		step3Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step3"}
   842  		Expect(k8sClient.Get(ctx, step3Key, expDeployment)).Should(utils.NotFoundMatcher{})
   843  
   844  		checkRun := &v1alpha1.WorkflowRun{}
   845  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
   846  		step1Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step1"}
   847  		Expect(k8sClient.Get(ctx, step1Key, expDeployment)).Should(BeNil())
   848  		expDeployment.Status.Replicas = 1
   849  		expDeployment.Status.ReadyReplicas = 1
   850  		Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
   851  		step2Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step2"}
   852  		Expect(k8sClient.Get(ctx, step2Key, expDeployment)).Should(BeNil())
   853  		expDeployment.Status.Replicas = 1
   854  		expDeployment.Status.ReadyReplicas = 1
   855  		Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
   856  
   857  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   858  
   859  		Expect(k8sClient.Get(ctx, step3Key, expDeployment)).Should(BeNil())
   860  		expDeployment.Status.Replicas = 1
   861  		expDeployment.Status.ReadyReplicas = 1
   862  		Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
   863  
   864  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   865  
   866  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
   867  		Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateSucceeded))
   868  		Expect(checkRun.Status.Mode).Should(BeEquivalentTo(v1alpha1.WorkflowExecuteMode{
   869  			Steps:    v1alpha1.WorkflowModeDAG,
   870  			SubSteps: v1alpha1.WorkflowModeStep,
   871  		}))
   872  	})
   873  
   874  	It("test workflow run with mode in step groups", func() {
   875  		wr := wrTemplate.DeepCopy()
   876  		wr.Name = "wr-group-mode"
   877  		wr.Spec.WorkflowSpec.Steps = []v1alpha1.WorkflowStep{
   878  			{
   879  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
   880  					Name:       "step1",
   881  					Type:       "test-apply",
   882  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
   883  				},
   884  			},
   885  			{
   886  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
   887  					Name: "group",
   888  					Type: "step-group",
   889  				},
   890  				Mode: v1alpha1.WorkflowModeStep,
   891  				SubSteps: []v1alpha1.WorkflowStepBase{
   892  					{
   893  						Name:       "step2",
   894  						Type:       "test-apply",
   895  						Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
   896  					},
   897  					{
   898  						Name:       "step3",
   899  						Type:       "test-apply",
   900  						Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
   901  					},
   902  				},
   903  			},
   904  		}
   905  		wr.Spec.Mode = &v1alpha1.WorkflowExecuteMode{
   906  			Steps: v1alpha1.WorkflowModeDAG,
   907  		}
   908  
   909  		Expect(k8sClient.Create(context.Background(), wr)).Should(BeNil())
   910  		wrKey := types.NamespacedName{Namespace: wr.Namespace, Name: wr.Name}
   911  
   912  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   913  
   914  		expDeployment := &appsv1.Deployment{}
   915  		step3Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step3"}
   916  		Expect(k8sClient.Get(ctx, step3Key, expDeployment)).Should(utils.NotFoundMatcher{})
   917  
   918  		checkRun := &v1alpha1.WorkflowRun{}
   919  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
   920  		step1Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step1"}
   921  		Expect(k8sClient.Get(ctx, step1Key, expDeployment)).Should(BeNil())
   922  		expDeployment.Status.Replicas = 1
   923  		expDeployment.Status.ReadyReplicas = 1
   924  		Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
   925  		step2Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step2"}
   926  		Expect(k8sClient.Get(ctx, step2Key, expDeployment)).Should(BeNil())
   927  		expDeployment.Status.Replicas = 1
   928  		expDeployment.Status.ReadyReplicas = 1
   929  		Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
   930  
   931  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   932  
   933  		Expect(k8sClient.Get(ctx, step3Key, expDeployment)).Should(BeNil())
   934  		expDeployment.Status.Replicas = 1
   935  		expDeployment.Status.ReadyReplicas = 1
   936  		Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
   937  
   938  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   939  
   940  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
   941  		Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateSucceeded))
   942  		Expect(checkRun.Status.Mode).Should(BeEquivalentTo(v1alpha1.WorkflowExecuteMode{
   943  			Steps:    v1alpha1.WorkflowModeDAG,
   944  			SubSteps: v1alpha1.WorkflowModeDAG,
   945  		}))
   946  	})
   947  
   948  	It("test sub steps", func() {
   949  		wr := wrTemplate.DeepCopy()
   950  		wr.Name = "wr-substeps"
   951  		wr.Spec.WorkflowSpec.Steps = []v1alpha1.WorkflowStep{
   952  			{
   953  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
   954  					Name:       "step1",
   955  					Type:       "test-apply",
   956  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
   957  				},
   958  			},
   959  			{
   960  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
   961  					Name: "group",
   962  					Type: "step-group",
   963  				},
   964  				SubSteps: []v1alpha1.WorkflowStepBase{
   965  					{
   966  						Name:       "step2",
   967  						Type:       "test-apply",
   968  						DependsOn:  []string{"step3"},
   969  						Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
   970  					},
   971  					{
   972  						Name:       "step3",
   973  						Type:       "test-apply",
   974  						Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
   975  					},
   976  				},
   977  			},
   978  		}
   979  
   980  		Expect(k8sClient.Create(context.Background(), wr)).Should(BeNil())
   981  		wrKey := types.NamespacedName{Namespace: wr.Namespace, Name: wr.Name}
   982  
   983  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   984  
   985  		expDeployment := &appsv1.Deployment{}
   986  		step2Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step2"}
   987  		Expect(k8sClient.Get(ctx, step2Key, expDeployment)).Should(utils.NotFoundMatcher{})
   988  		step3Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step3"}
   989  		Expect(k8sClient.Get(ctx, step3Key, expDeployment)).Should(utils.NotFoundMatcher{})
   990  		checkRun := &v1alpha1.WorkflowRun{}
   991  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
   992  		step1Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step1"}
   993  		Expect(k8sClient.Get(ctx, step1Key, expDeployment)).Should(BeNil())
   994  		expDeployment.Status.Replicas = 1
   995  		expDeployment.Status.ReadyReplicas = 1
   996  		Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
   997  
   998  		tryReconcile(reconciler, wr.Name, wr.Namespace)
   999  
  1000  		Expect(k8sClient.Get(ctx, step2Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1001  		Expect(k8sClient.Get(ctx, step3Key, expDeployment)).Should(BeNil())
  1002  		expDeployment.Status.Replicas = 1
  1003  		expDeployment.Status.ReadyReplicas = 1
  1004  		Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
  1005  
  1006  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1007  
  1008  		Expect(k8sClient.Get(ctx, step2Key, expDeployment)).Should(BeNil())
  1009  		expDeployment.Status.Replicas = 1
  1010  		expDeployment.Status.ReadyReplicas = 1
  1011  		Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
  1012  
  1013  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1014  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
  1015  		Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateSucceeded))
  1016  	})
  1017  
  1018  	It("test failed step's outputs", func() {
  1019  		wr := wrTemplate.DeepCopy()
  1020  		wr.Name = "wr-timeout-output"
  1021  		wr.Spec.WorkflowSpec.Steps = []v1alpha1.WorkflowStep{
  1022  			{
  1023  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
  1024  					Name:    "step1",
  1025  					Type:    "test-apply",
  1026  					Timeout: "1s",
  1027  					Outputs: v1alpha1.StepOutputs{
  1028  						{
  1029  							Name:      "output",
  1030  							ValueFrom: "context.name",
  1031  						},
  1032  					},
  1033  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
  1034  				},
  1035  			},
  1036  			{
  1037  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
  1038  					Name: "step2",
  1039  					Inputs: v1alpha1.StepInputs{
  1040  						{
  1041  							From:         "output",
  1042  							ParameterKey: "",
  1043  						},
  1044  					},
  1045  					If:         `inputs.output == "wr-timeout-output"`,
  1046  					Type:       "test-apply",
  1047  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
  1048  				},
  1049  			},
  1050  		}
  1051  
  1052  		Expect(k8sClient.Create(context.Background(), wr)).Should(BeNil())
  1053  		wrKey := types.NamespacedName{Namespace: wr.Namespace, Name: wr.Name}
  1054  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1055  
  1056  		expDeployment := &appsv1.Deployment{}
  1057  		step1Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step1"}
  1058  		Expect(k8sClient.Get(ctx, step1Key, expDeployment)).Should(BeNil())
  1059  		step2Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step2"}
  1060  		Expect(k8sClient.Get(ctx, step2Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1061  
  1062  		time.Sleep(time.Second)
  1063  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1064  
  1065  		Expect(k8sClient.Get(ctx, step2Key, expDeployment)).Should(BeNil())
  1066  		expDeployment.Status.Replicas = 1
  1067  		expDeployment.Status.ReadyReplicas = 1
  1068  		Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
  1069  
  1070  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1071  
  1072  		checkRun := &v1alpha1.WorkflowRun{}
  1073  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
  1074  		Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateFailed))
  1075  	})
  1076  
  1077  	It("test if always", func() {
  1078  		wr := wrTemplate.DeepCopy()
  1079  		wr.Name = "wr-if-always"
  1080  		wr.Spec.WorkflowSpec.Steps = []v1alpha1.WorkflowStep{
  1081  			{
  1082  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
  1083  					Name:       "step-failed",
  1084  					Type:       "apply-object",
  1085  					Properties: &runtime.RawExtension{Raw: []byte(`{"value":[{"apiVersion":"v1","kind":"invalid","metadata":{"name":"test1"}}]}`)},
  1086  				},
  1087  			},
  1088  			{
  1089  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
  1090  					Name:       "step1",
  1091  					Type:       "test-apply",
  1092  					If:         "always",
  1093  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
  1094  				},
  1095  			},
  1096  			{
  1097  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
  1098  					Name:       "step2",
  1099  					Type:       "test-apply",
  1100  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
  1101  				},
  1102  			},
  1103  		}
  1104  		Expect(k8sClient.Create(context.Background(), wr)).Should(BeNil())
  1105  		wrKey := types.NamespacedName{Namespace: wr.Namespace, Name: wr.Name}
  1106  
  1107  		By("verify the first ten reconciles")
  1108  		for i := 0; i < wfTypes.MaxWorkflowStepErrorRetryTimes; i++ {
  1109  			tryReconcile(reconciler, wr.Name, wr.Namespace)
  1110  		}
  1111  
  1112  		expDeployment := &appsv1.Deployment{}
  1113  		step1Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step1"}
  1114  		Expect(k8sClient.Get(ctx, step1Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1115  		step2Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step2"}
  1116  		Expect(k8sClient.Get(ctx, step2Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1117  
  1118  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1119  
  1120  		Expect(k8sClient.Get(ctx, step1Key, expDeployment)).Should(BeNil())
  1121  		expDeployment.Status.Replicas = 1
  1122  		expDeployment.Status.ReadyReplicas = 1
  1123  		Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
  1124  		Expect(k8sClient.Get(ctx, step2Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1125  
  1126  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1127  
  1128  		Expect(k8sClient.Get(ctx, step2Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1129  
  1130  		checkRun := &v1alpha1.WorkflowRun{}
  1131  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
  1132  		Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateFailed))
  1133  	})
  1134  
  1135  	It("test if always in sub steps", func() {
  1136  		wr := wrTemplate.DeepCopy()
  1137  		wr.Name = "wr-if-always-substeps"
  1138  		wr.Spec.WorkflowSpec.Steps = []v1alpha1.WorkflowStep{
  1139  			{
  1140  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
  1141  					Name: "group",
  1142  					Type: "step-group",
  1143  				},
  1144  				SubSteps: []v1alpha1.WorkflowStepBase{
  1145  					{
  1146  						Name:       "sub1",
  1147  						Type:       "test-apply",
  1148  						If:         "always",
  1149  						DependsOn:  []string{"sub2-failed"},
  1150  						Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
  1151  					},
  1152  					{
  1153  						Name:       "sub2-failed",
  1154  						Type:       "apply-object",
  1155  						Properties: &runtime.RawExtension{Raw: []byte(`{"value":[{"apiVersion":"v1","kind":"invalid","metadata":{"name":"test1"}}]}`)},
  1156  					},
  1157  					{
  1158  						Name:       "sub3",
  1159  						Type:       "test-apply",
  1160  						DependsOn:  []string{"sub1"},
  1161  						Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
  1162  					},
  1163  				},
  1164  			},
  1165  			{
  1166  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
  1167  					Name:       "step2",
  1168  					Type:       "test-apply",
  1169  					If:         "always",
  1170  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
  1171  				},
  1172  			},
  1173  			{
  1174  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
  1175  					Name:       "step3",
  1176  					Type:       "test-apply",
  1177  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
  1178  				},
  1179  			},
  1180  		}
  1181  
  1182  		Expect(k8sClient.Create(context.Background(), wr)).Should(BeNil())
  1183  		wrKey := types.NamespacedName{Namespace: wr.Namespace, Name: wr.Name}
  1184  
  1185  		By("verify the first ten reconciles")
  1186  		for i := 0; i < wfTypes.MaxWorkflowStepErrorRetryTimes; i++ {
  1187  			tryReconcile(reconciler, wr.Name, wr.Namespace)
  1188  		}
  1189  
  1190  		expDeployment := &appsv1.Deployment{}
  1191  		sub1Key := types.NamespacedName{Namespace: wr.Namespace, Name: "sub1"}
  1192  		Expect(k8sClient.Get(ctx, sub1Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1193  		sub3Key := types.NamespacedName{Namespace: wr.Namespace, Name: "sub3"}
  1194  		Expect(k8sClient.Get(ctx, sub3Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1195  		step2Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step2"}
  1196  		Expect(k8sClient.Get(ctx, step2Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1197  		step3Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step3"}
  1198  		Expect(k8sClient.Get(ctx, step3Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1199  
  1200  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1201  
  1202  		Expect(k8sClient.Get(ctx, sub1Key, expDeployment)).Should(BeNil())
  1203  		expDeployment.Status.Replicas = 1
  1204  		expDeployment.Status.ReadyReplicas = 1
  1205  		Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
  1206  		Expect(k8sClient.Get(ctx, sub3Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1207  		Expect(k8sClient.Get(ctx, step2Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1208  		Expect(k8sClient.Get(ctx, step3Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1209  
  1210  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1211  
  1212  		Expect(k8sClient.Get(ctx, sub3Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1213  		Expect(k8sClient.Get(ctx, step2Key, expDeployment)).Should(BeNil())
  1214  		expDeployment.Status.Replicas = 1
  1215  		expDeployment.Status.ReadyReplicas = 1
  1216  		Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
  1217  		Expect(k8sClient.Get(ctx, step3Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1218  
  1219  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1220  
  1221  		Expect(k8sClient.Get(ctx, sub3Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1222  		Expect(k8sClient.Get(ctx, step3Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1223  
  1224  		checkRun := &v1alpha1.WorkflowRun{}
  1225  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
  1226  		Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateFailed))
  1227  	})
  1228  
  1229  	It("test if expressions", func() {
  1230  		wr := wrTemplate.DeepCopy()
  1231  		wr.Name = "wr-if-expressions"
  1232  		wr.Spec.Context = &runtime.RawExtension{Raw: []byte(`{"mycontext":{"a":1,"b":2,"c":["hello", "world"]}}`)}
  1233  		wr.Spec.WorkflowSpec.Steps = []v1alpha1.WorkflowStep{
  1234  			{
  1235  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
  1236  					Name:    "suspend",
  1237  					Type:    "suspend",
  1238  					Timeout: "1s",
  1239  					Outputs: v1alpha1.StepOutputs{
  1240  						{
  1241  							Name:      "suspend-output",
  1242  							ValueFrom: "context.name",
  1243  						},
  1244  						{
  1245  							Name:      "custom-output",
  1246  							ValueFrom: "context.mycontext.a",
  1247  						},
  1248  					},
  1249  				},
  1250  			},
  1251  			{
  1252  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
  1253  					Name:       "step2",
  1254  					Type:       "test-apply",
  1255  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
  1256  					Inputs: v1alpha1.StepInputs{
  1257  						{
  1258  							From:         "suspend-output",
  1259  							ParameterKey: "",
  1260  						},
  1261  						{
  1262  							From:         "custom-output",
  1263  							ParameterKey: "",
  1264  						},
  1265  						{
  1266  							From:         "context.mycontext.c[1]",
  1267  							ParameterKey: "",
  1268  						},
  1269  					},
  1270  					If: `status.suspend.timeout && inputs["suspend-output"] == "wr-if-expressions" && inputs["custom-output"] == 1 && context.mycontext.b == 2 && context.mycontext.c[0] == "hello" && inputs["context.mycontext.c[1]"] == "world"`,
  1271  				},
  1272  			},
  1273  			{
  1274  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
  1275  					Name:       "step3",
  1276  					If:         "status.suspend.succeeded",
  1277  					Type:       "test-apply",
  1278  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
  1279  				},
  1280  			},
  1281  		}
  1282  
  1283  		Expect(k8sClient.Create(context.Background(), wr)).Should(BeNil())
  1284  		wrKey := types.NamespacedName{Namespace: wr.Namespace, Name: wr.Name}
  1285  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1286  
  1287  		checkRun := &v1alpha1.WorkflowRun{}
  1288  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
  1289  		Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateSuspending))
  1290  
  1291  		expDeployment := &appsv1.Deployment{}
  1292  		step2Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step2"}
  1293  		Expect(k8sClient.Get(ctx, step2Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1294  		step3Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step3"}
  1295  		Expect(k8sClient.Get(ctx, step3Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1296  
  1297  		time.Sleep(time.Second)
  1298  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1299  		Expect(k8sClient.Get(ctx, step2Key, expDeployment)).Should(BeNil())
  1300  		expDeployment.Status.Replicas = 1
  1301  		expDeployment.Status.ReadyReplicas = 1
  1302  		Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
  1303  		Expect(k8sClient.Get(ctx, step3Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1304  
  1305  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1306  
  1307  		Expect(k8sClient.Get(ctx, step3Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1308  
  1309  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
  1310  		Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateFailed))
  1311  	})
  1312  
  1313  	It("test if expressions in sub steps", func() {
  1314  		wr := wrTemplate.DeepCopy()
  1315  		wr.Name = "wr-if-expressions-substeps"
  1316  		wr.Spec.Context = &runtime.RawExtension{Raw: []byte(`{"mycontext":{"a":1,"b":2,"c":["hello", "world"]}}`)}
  1317  		wr.Spec.WorkflowSpec.Steps = []v1alpha1.WorkflowStep{
  1318  			{
  1319  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
  1320  					Name: "group",
  1321  					Type: "step-group",
  1322  					Inputs: v1alpha1.StepInputs{
  1323  						{
  1324  							From:         "context.mycontext.c[1]",
  1325  							ParameterKey: "",
  1326  						},
  1327  					},
  1328  					If: `context.mycontext.b == 2`,
  1329  				},
  1330  				SubSteps: []v1alpha1.WorkflowStepBase{
  1331  					{
  1332  						Name:       "sub1",
  1333  						Type:       "test-apply",
  1334  						If:         "status.sub2.timeout",
  1335  						DependsOn:  []string{"sub2"},
  1336  						Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
  1337  					},
  1338  					{
  1339  						Name:       "sub2",
  1340  						Type:       "suspend",
  1341  						Properties: &runtime.RawExtension{Raw: []byte(`{"duration":"1s"}`)},
  1342  						If:         `inputs["context.mycontext.c[0]"] == "hello" && context.mycontext.c[1] == "world"`,
  1343  						Outputs: v1alpha1.StepOutputs{
  1344  							{
  1345  								Name:      "suspend-output",
  1346  								ValueFrom: "context.name",
  1347  							},
  1348  							{
  1349  								Name:      "suspend-custom-output",
  1350  								ValueFrom: "context.mycontext.a",
  1351  							},
  1352  						},
  1353  						Inputs: v1alpha1.StepInputs{
  1354  							{
  1355  								From:         "context.mycontext.c[0]",
  1356  								ParameterKey: "",
  1357  							},
  1358  						},
  1359  					},
  1360  					{
  1361  						Name:      "sub3",
  1362  						Type:      "test-apply",
  1363  						DependsOn: []string{"sub1"},
  1364  						Inputs: v1alpha1.StepInputs{
  1365  							{
  1366  								From:         "suspend-output",
  1367  								ParameterKey: "",
  1368  							},
  1369  						},
  1370  						If:         `status.sub1.timeout || inputs["suspend-output"] == "wr-if-expressions-substeps"`,
  1371  						Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
  1372  					},
  1373  				},
  1374  			},
  1375  			{
  1376  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
  1377  					Name:       "step2",
  1378  					Type:       "test-apply",
  1379  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
  1380  					If:         "status.group.failed",
  1381  				},
  1382  			},
  1383  			{
  1384  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
  1385  					Name:       "step3",
  1386  					Type:       "test-apply",
  1387  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
  1388  				},
  1389  			},
  1390  			{
  1391  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
  1392  					Name: "group2",
  1393  					Type: "step-group",
  1394  				},
  1395  				SubSteps: []v1alpha1.WorkflowStepBase{
  1396  					{
  1397  						Name:    "group2-sub",
  1398  						Type:    "suspend",
  1399  						Timeout: "1s",
  1400  					},
  1401  				},
  1402  			},
  1403  		}
  1404  		Expect(k8sClient.Create(context.Background(), wr)).Should(BeNil())
  1405  		wrKey := types.NamespacedName{Namespace: wr.Namespace, Name: wr.Name}
  1406  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1407  
  1408  		checkRun := &v1alpha1.WorkflowRun{}
  1409  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
  1410  		Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateSuspending))
  1411  
  1412  		expDeployment := &appsv1.Deployment{}
  1413  		sub1Key := types.NamespacedName{Namespace: wr.Namespace, Name: "sub1"}
  1414  		Expect(k8sClient.Get(ctx, sub1Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1415  		sub3Key := types.NamespacedName{Namespace: wr.Namespace, Name: "sub3"}
  1416  		Expect(k8sClient.Get(ctx, sub3Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1417  		step2Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step2"}
  1418  		Expect(k8sClient.Get(ctx, step2Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1419  		step3Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step3"}
  1420  		Expect(k8sClient.Get(ctx, step3Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1421  
  1422  		time.Sleep(time.Second)
  1423  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1424  
  1425  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
  1426  		Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateExecuting))
  1427  		Expect(k8sClient.Get(ctx, sub1Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1428  		Expect(k8sClient.Get(ctx, sub3Key, expDeployment)).Should(BeNil())
  1429  		expDeployment.Status.Replicas = 1
  1430  		expDeployment.Status.ReadyReplicas = 1
  1431  		Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
  1432  		Expect(k8sClient.Get(ctx, step2Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1433  		Expect(k8sClient.Get(ctx, step3Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1434  
  1435  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1436  
  1437  		Expect(k8sClient.Get(ctx, step2Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1438  		Expect(k8sClient.Get(ctx, step3Key, expDeployment)).Should(BeNil())
  1439  		expDeployment.Status.Replicas = 1
  1440  		expDeployment.Status.ReadyReplicas = 1
  1441  		Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
  1442  
  1443  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1444  
  1445  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
  1446  		Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateSuspending))
  1447  
  1448  		time.Sleep(time.Second)
  1449  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1450  
  1451  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
  1452  		Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateFailed))
  1453  	})
  1454  
  1455  	It("test suspend and deploy", func() {
  1456  		wr := wrTemplate.DeepCopy()
  1457  		wr.Name = "wr-suspend-and-deploy"
  1458  		wr.Spec.WorkflowSpec.Steps = []v1alpha1.WorkflowStep{
  1459  			{
  1460  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
  1461  					Name:       "step1",
  1462  					Type:       "suspend-and-deploy",
  1463  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
  1464  				},
  1465  			},
  1466  		}
  1467  		Expect(k8sClient.Create(context.Background(), wr)).Should(BeNil())
  1468  		wrKey := types.NamespacedName{Namespace: wr.Namespace, Name: wr.Name}
  1469  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1470  
  1471  		checkRun := &v1alpha1.WorkflowRun{}
  1472  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
  1473  		Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateSuspending))
  1474  
  1475  		expDeployment := &appsv1.Deployment{}
  1476  		step1Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step1"}
  1477  		Expect(k8sClient.Get(ctx, step1Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1478  
  1479  		time.Sleep(time.Second)
  1480  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1481  
  1482  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
  1483  		Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateExecuting))
  1484  		Expect(k8sClient.Get(ctx, step1Key, expDeployment)).Should(BeNil())
  1485  		expDeployment.Status.Replicas = 1
  1486  		expDeployment.Status.ReadyReplicas = 1
  1487  		Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
  1488  
  1489  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1490  
  1491  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
  1492  		Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateSucceeded))
  1493  	})
  1494  
  1495  	It("test multiple suspend", func() {
  1496  		wr := wrTemplate.DeepCopy()
  1497  		wr.Name = "wr-multi-suspend"
  1498  		wr.Spec.WorkflowSpec.Steps = []v1alpha1.WorkflowStep{
  1499  			{
  1500  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
  1501  					Name: "step1",
  1502  					Type: "multi-suspend",
  1503  				},
  1504  			},
  1505  		}
  1506  		Expect(k8sClient.Create(context.Background(), wr)).Should(BeNil())
  1507  		wrKey := types.NamespacedName{Namespace: wr.Namespace, Name: wr.Name}
  1508  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1509  
  1510  		checkRun := &v1alpha1.WorkflowRun{}
  1511  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
  1512  		Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateSuspending))
  1513  
  1514  		Expect(utils.ResumeWorkflow(ctx, k8sClient, checkRun, "")).Should(BeNil())
  1515  		Expect(checkRun.Status.Suspend).Should(BeFalse())
  1516  
  1517  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1518  
  1519  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
  1520  		// suspended by the second suspend
  1521  		Expect(checkRun.Status.Suspend).Should(BeTrue())
  1522  
  1523  		Expect(utils.ResumeWorkflow(ctx, k8sClient, checkRun, "")).Should(BeNil())
  1524  		Expect(checkRun.Status.Suspend).Should(BeFalse())
  1525  
  1526  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1527  
  1528  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
  1529  		Expect(checkRun.Status.Suspend).Should(BeFalse())
  1530  		Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateSucceeded))
  1531  	})
  1532  
  1533  	It("test timeout", func() {
  1534  		wr := wrTemplate.DeepCopy()
  1535  		wr.Name = "wr-timeout"
  1536  		wr.Spec.WorkflowSpec.Steps = []v1alpha1.WorkflowStep{
  1537  			{
  1538  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
  1539  					Name:       "step1",
  1540  					Type:       "test-apply",
  1541  					Timeout:    "1s",
  1542  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
  1543  				},
  1544  			},
  1545  			{
  1546  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
  1547  					Name:       "step2",
  1548  					Type:       "test-apply",
  1549  					If:         "always",
  1550  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
  1551  				},
  1552  			},
  1553  			{
  1554  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
  1555  					Name:       "step3",
  1556  					Type:       "test-apply",
  1557  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
  1558  				},
  1559  			},
  1560  		}
  1561  
  1562  		Expect(k8sClient.Create(context.Background(), wr)).Should(BeNil())
  1563  		wrKey := types.NamespacedName{Namespace: wr.Namespace, Name: wr.Name}
  1564  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1565  
  1566  		expDeployment := &appsv1.Deployment{}
  1567  		step1Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step1"}
  1568  		Expect(k8sClient.Get(ctx, step1Key, expDeployment)).Should(BeNil())
  1569  		step2Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step2"}
  1570  		Expect(k8sClient.Get(ctx, step2Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1571  		step3Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step3"}
  1572  		Expect(k8sClient.Get(ctx, step3Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1573  
  1574  		time.Sleep(time.Second)
  1575  
  1576  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1577  
  1578  		Expect(k8sClient.Get(ctx, step2Key, expDeployment)).Should(BeNil())
  1579  		expDeployment.Status.Replicas = 1
  1580  		expDeployment.Status.ReadyReplicas = 1
  1581  		Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
  1582  		Expect(k8sClient.Get(ctx, step3Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1583  
  1584  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1585  		Expect(k8sClient.Get(ctx, step3Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1586  
  1587  		checkRun := &v1alpha1.WorkflowRun{}
  1588  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
  1589  		Expect(checkRun.Status.Steps[0].Reason).Should(Equal(wfTypes.StatusReasonTimeout))
  1590  		Expect(checkRun.Status.Steps[1].Phase).Should(Equal(v1alpha1.WorkflowStepPhaseSucceeded))
  1591  		Expect(checkRun.Status.Steps[2].Reason).Should(Equal(wfTypes.StatusReasonSkip))
  1592  		Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateFailed))
  1593  	})
  1594  
  1595  	It("test timeout in sub steps", func() {
  1596  		wr := wrTemplate.DeepCopy()
  1597  		wr.Name = "wr-timeout-substeps"
  1598  		wr.Spec.WorkflowSpec.Steps = []v1alpha1.WorkflowStep{
  1599  			{
  1600  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
  1601  					Name: "group",
  1602  					Type: "step-group",
  1603  				},
  1604  				SubSteps: []v1alpha1.WorkflowStepBase{
  1605  					{
  1606  						Name:       "sub1",
  1607  						Type:       "test-apply",
  1608  						If:         "always",
  1609  						DependsOn:  []string{"sub2"},
  1610  						Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
  1611  					},
  1612  					{
  1613  						Name:       "sub2",
  1614  						Type:       "test-apply",
  1615  						Timeout:    "1s",
  1616  						Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
  1617  					},
  1618  					{
  1619  						Name:       "sub3",
  1620  						Type:       "test-apply",
  1621  						DependsOn:  []string{"sub1"},
  1622  						Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
  1623  					},
  1624  				},
  1625  			},
  1626  			{
  1627  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
  1628  					Name:       "step2",
  1629  					Type:       "test-apply",
  1630  					If:         "always",
  1631  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
  1632  				},
  1633  			},
  1634  			{
  1635  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
  1636  					Name:       "step3",
  1637  					Type:       "test-apply",
  1638  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
  1639  				},
  1640  			},
  1641  			{
  1642  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
  1643  					Name:    "group2",
  1644  					If:      "always",
  1645  					Type:    "step-group",
  1646  					Timeout: "1s",
  1647  				},
  1648  				SubSteps: []v1alpha1.WorkflowStepBase{
  1649  					{
  1650  						Name: "group2-sub",
  1651  						Type: "suspend",
  1652  					},
  1653  				},
  1654  			},
  1655  		}
  1656  
  1657  		Expect(k8sClient.Create(context.Background(), wr)).Should(BeNil())
  1658  		wrKey := types.NamespacedName{Namespace: wr.Namespace, Name: wr.Name}
  1659  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1660  
  1661  		expDeployment := &appsv1.Deployment{}
  1662  		sub1Key := types.NamespacedName{Namespace: wr.Namespace, Name: "sub1"}
  1663  		Expect(k8sClient.Get(ctx, sub1Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1664  		sub2Key := types.NamespacedName{Namespace: wr.Namespace, Name: "sub2"}
  1665  		Expect(k8sClient.Get(ctx, sub2Key, expDeployment)).Should(BeNil())
  1666  		sub3Key := types.NamespacedName{Namespace: wr.Namespace, Name: "sub3"}
  1667  		Expect(k8sClient.Get(ctx, sub3Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1668  		step2Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step2"}
  1669  		Expect(k8sClient.Get(ctx, step2Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1670  		step3Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step3"}
  1671  		Expect(k8sClient.Get(ctx, step3Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1672  
  1673  		time.Sleep(time.Second)
  1674  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1675  
  1676  		Expect(k8sClient.Get(ctx, sub1Key, expDeployment)).Should(BeNil())
  1677  		expDeployment.Status.Replicas = 1
  1678  		expDeployment.Status.ReadyReplicas = 1
  1679  		Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
  1680  		Expect(k8sClient.Get(ctx, sub3Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1681  		Expect(k8sClient.Get(ctx, step2Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1682  		Expect(k8sClient.Get(ctx, step3Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1683  
  1684  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1685  
  1686  		Expect(k8sClient.Get(ctx, sub3Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1687  		Expect(k8sClient.Get(ctx, step2Key, expDeployment)).Should(BeNil())
  1688  		expDeployment.Status.Replicas = 1
  1689  		expDeployment.Status.ReadyReplicas = 1
  1690  		Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
  1691  		Expect(k8sClient.Get(ctx, step3Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1692  
  1693  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1694  
  1695  		Expect(k8sClient.Get(ctx, step3Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1696  
  1697  		checkRun := &v1alpha1.WorkflowRun{}
  1698  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
  1699  		Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateSuspending))
  1700  
  1701  		time.Sleep(time.Second)
  1702  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1703  
  1704  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
  1705  		Expect(checkRun.Status.Steps[0].Reason).Should(Equal(wfTypes.StatusReasonTimeout))
  1706  		Expect(checkRun.Status.Steps[1].Phase).Should(Equal(v1alpha1.WorkflowStepPhaseSucceeded))
  1707  		Expect(checkRun.Status.Steps[2].Reason).Should(Equal(wfTypes.StatusReasonSkip))
  1708  		Expect(checkRun.Status.Steps[3].Reason).Should(Equal(wfTypes.StatusReasonTimeout))
  1709  		Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateFailed))
  1710  	})
  1711  
  1712  	It("test terminate manually", func() {
  1713  		wr := wrTemplate.DeepCopy()
  1714  		wr.Name = "wr-terminate-manually"
  1715  		wr.Spec.WorkflowSpec.Steps = []v1alpha1.WorkflowStep{
  1716  			{
  1717  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
  1718  					Name: "step1",
  1719  					Type: "suspend",
  1720  				},
  1721  			},
  1722  			{
  1723  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
  1724  					Name:       "step2",
  1725  					Type:       "test-apply",
  1726  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
  1727  				},
  1728  			},
  1729  		}
  1730  
  1731  		Expect(k8sClient.Create(context.Background(), wr)).Should(BeNil())
  1732  		wrKey := types.NamespacedName{Namespace: wr.Namespace, Name: wr.Name}
  1733  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1734  
  1735  		expDeployment := &appsv1.Deployment{}
  1736  		step2Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step2"}
  1737  		Expect(k8sClient.Get(ctx, step2Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1738  
  1739  		// terminate manually
  1740  		checkRun := &v1alpha1.WorkflowRun{}
  1741  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
  1742  		Expect(utils.TerminateWorkflow(ctx, k8sClient, checkRun)).Should(BeNil())
  1743  
  1744  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1745  
  1746  		Expect(k8sClient.Get(ctx, step2Key, expDeployment)).Should(utils.NotFoundMatcher{})
  1747  
  1748  		Expect(k8sClient.Get(ctx, wrKey, checkRun)).Should(BeNil())
  1749  		Expect(checkRun.Status.Steps[1].Phase).Should(Equal(v1alpha1.WorkflowStepPhaseSkipped))
  1750  		Expect(checkRun.Status.Phase).Should(BeEquivalentTo(v1alpha1.WorkflowStateTerminated))
  1751  	})
  1752  
  1753  	It("test debug", func() {
  1754  		wr := wrTemplate.DeepCopy()
  1755  		wr.Name = "wr-debug"
  1756  		wr.Spec.WorkflowSpec.Steps = []v1alpha1.WorkflowStep{
  1757  			{
  1758  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
  1759  					Name:       "step1",
  1760  					Type:       "test-apply",
  1761  					Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
  1762  				},
  1763  			},
  1764  			{
  1765  				WorkflowStepBase: v1alpha1.WorkflowStepBase{
  1766  					Name: "step2",
  1767  					Type: "step-group",
  1768  				},
  1769  				SubSteps: []v1alpha1.WorkflowStepBase{
  1770  					{
  1771  						Name:       "step2-sub",
  1772  						Type:       "test-apply",
  1773  						Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
  1774  					},
  1775  				},
  1776  			},
  1777  		}
  1778  		wr.Annotations = map[string]string{
  1779  			wfTypes.AnnotationWorkflowRunDebug: "true",
  1780  		}
  1781  		Expect(k8sClient.Create(ctx, wr)).Should(BeNil())
  1782  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1783  
  1784  		expDeployment := &appsv1.Deployment{}
  1785  		step1Key := types.NamespacedName{Namespace: wr.Namespace, Name: "step1"}
  1786  		Expect(k8sClient.Get(ctx, step1Key, expDeployment)).Should(BeNil())
  1787  		expDeployment.Status.Replicas = 1
  1788  		expDeployment.Status.ReadyReplicas = 1
  1789  		Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
  1790  
  1791  		wrKey := client.ObjectKey{
  1792  			Name:      wr.Name,
  1793  			Namespace: wr.Namespace,
  1794  		}
  1795  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1796  
  1797  		subKey := types.NamespacedName{Namespace: wr.Namespace, Name: "step2-sub"}
  1798  		Expect(k8sClient.Get(ctx, subKey, expDeployment)).Should(BeNil())
  1799  		expDeployment.Status.Replicas = 1
  1800  		expDeployment.Status.ReadyReplicas = 1
  1801  		Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
  1802  
  1803  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1804  
  1805  		By("Check WorkflowRun running successfully")
  1806  		curRun := &v1alpha1.WorkflowRun{}
  1807  		Expect(k8sClient.Get(ctx, wrKey, curRun)).Should(BeNil())
  1808  		Expect(curRun.Status.Phase).Should(Equal(v1alpha1.WorkflowStateSucceeded))
  1809  
  1810  		By("Check debug Config Map is created")
  1811  		debugCM := &corev1.ConfigMap{}
  1812  		Expect(k8sClient.Get(ctx, types.NamespacedName{
  1813  			Name:      debug.GenerateContextName(wr.Name, curRun.Status.Steps[0].ID, string(curRun.UID)),
  1814  			Namespace: wr.Namespace,
  1815  		}, debugCM)).Should(BeNil())
  1816  		Expect(k8sClient.Get(ctx, types.NamespacedName{
  1817  			Name:      debug.GenerateContextName(wr.Name, curRun.Status.Steps[1].SubStepsStatus[0].ID, string(curRun.UID)),
  1818  			Namespace: wr.Namespace,
  1819  		}, debugCM)).Should(BeNil())
  1820  	})
  1821  
  1822  	It("test step context data", func() {
  1823  		wr := wrTemplate.DeepCopy()
  1824  		wr.Name = "test-step-context-data"
  1825  		wr.Spec.WorkflowSpec.Steps = []v1alpha1.WorkflowStep{{
  1826  			WorkflowStepBase: v1alpha1.WorkflowStepBase{
  1827  				Name: "group1",
  1828  				Type: "step-group",
  1829  			},
  1830  			SubSteps: []v1alpha1.WorkflowStepBase{
  1831  				{
  1832  					Name:       "step1",
  1833  					Type:       "save-process-context",
  1834  					Properties: &runtime.RawExtension{Raw: []byte(`{"name":"process-context-step1"}`)},
  1835  				},
  1836  				{
  1837  					Name:       "step2",
  1838  					Type:       "save-process-context",
  1839  					Properties: &runtime.RawExtension{Raw: []byte(`{"name":"process-context-step2"}`)},
  1840  				},
  1841  			},
  1842  		}, {
  1843  			WorkflowStepBase: v1alpha1.WorkflowStepBase{
  1844  				Name: "group2",
  1845  				Type: "step-group",
  1846  			},
  1847  			SubSteps: []v1alpha1.WorkflowStepBase{
  1848  				{
  1849  					Name:       "step3",
  1850  					Type:       "save-process-context",
  1851  					Properties: &runtime.RawExtension{Raw: []byte(`{"name":"process-context-step3"}`)},
  1852  				},
  1853  				{
  1854  					Name:       "step4",
  1855  					Type:       "save-process-context",
  1856  					Properties: &runtime.RawExtension{Raw: []byte(`{"name":"process-context-step4"}`)},
  1857  				},
  1858  			},
  1859  		}, {
  1860  			WorkflowStepBase: v1alpha1.WorkflowStepBase{
  1861  				Name:       "step5",
  1862  				Type:       "save-process-context",
  1863  				Properties: &runtime.RawExtension{Raw: []byte(`{"name":"process-context-step5"}`)},
  1864  			},
  1865  		}}
  1866  		Expect(k8sClient.Create(ctx, wr)).Should(BeNil())
  1867  
  1868  		tryReconcile(reconciler, wr.Name, wr.Namespace)
  1869  		wrObj := &v1alpha1.WorkflowRun{}
  1870  		Expect(k8sClient.Get(ctx, client.ObjectKey{
  1871  			Name:      wr.Name,
  1872  			Namespace: wr.Namespace,
  1873  		}, wrObj)).Should(BeNil())
  1874  		cmList := new(corev1.ConfigMapList)
  1875  		labels := &metav1.LabelSelector{
  1876  			MatchLabels: map[string]string{
  1877  				"process.context.data": "true",
  1878  			},
  1879  		}
  1880  		selector, err := metav1.LabelSelectorAsSelector(labels)
  1881  		Expect(err).Should(BeNil())
  1882  		Expect(k8sClient.List(ctx, cmList, &client.ListOptions{
  1883  			LabelSelector: selector,
  1884  		})).Should(BeNil())
  1885  
  1886  		processCtxMap := make(map[string]map[string]string)
  1887  		for _, cm := range cmList.Items {
  1888  			processCtxMap[cm.Name] = cm.Data
  1889  		}
  1890  		step1Ctx := processCtxMap["process-context-step1"]
  1891  		step2Ctx := processCtxMap["process-context-step2"]
  1892  		step3Ctx := processCtxMap["process-context-step3"]
  1893  		step4Ctx := processCtxMap["process-context-step4"]
  1894  		step5Ctx := processCtxMap["process-context-step5"]
  1895  
  1896  		By("check context.stepName")
  1897  		Expect(step1Ctx["stepName"]).Should(Equal("step1"))
  1898  		Expect(step2Ctx["stepName"]).Should(Equal("step2"))
  1899  		Expect(step3Ctx["stepName"]).Should(Equal("step3"))
  1900  		Expect(step4Ctx["stepName"]).Should(Equal("step4"))
  1901  		Expect(step5Ctx["stepName"]).Should(Equal("step5"))
  1902  
  1903  		By("check context.stepGroupName")
  1904  		Expect(step1Ctx["stepGroupName"]).Should(Equal("group1"))
  1905  		Expect(step2Ctx["stepGroupName"]).Should(Equal("group1"))
  1906  		Expect(step3Ctx["stepGroupName"]).Should(Equal("group2"))
  1907  		Expect(step4Ctx["stepGroupName"]).Should(Equal("group2"))
  1908  		Expect(step5Ctx["stepGroupName"]).Should(Equal(""))
  1909  
  1910  		By("check context.spanID")
  1911  		spanID := strings.Split(step1Ctx["spanID"], ".")[0]
  1912  		for _, pCtx := range processCtxMap {
  1913  			Expect(pCtx["spanID"]).Should(ContainSubstring(spanID))
  1914  		}
  1915  	})
  1916  })
  1917  
  1918  func reconcileWithReturn(r *WorkflowRunReconciler, name, ns string) error {
  1919  	wrKey := client.ObjectKey{
  1920  		Name:      name,
  1921  		Namespace: ns,
  1922  	}
  1923  	_, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: wrKey})
  1924  	return err
  1925  }
  1926  
  1927  func tryReconcile(r *WorkflowRunReconciler, name, ns string) {
  1928  	wrKey := client.ObjectKey{
  1929  		Name:      name,
  1930  		Namespace: ns,
  1931  	}
  1932  	_, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: wrKey})
  1933  	if err != nil {
  1934  		By(fmt.Sprintf("reconcile err: %+v ", err))
  1935  	}
  1936  	Expect(err).Should(BeNil())
  1937  }
  1938  
  1939  func setupNamespace(ctx context.Context, namespace string) {
  1940  	ns := &corev1.Namespace{
  1941  		ObjectMeta: metav1.ObjectMeta{Name: namespace},
  1942  	}
  1943  	Expect(k8sClient.Create(ctx, ns)).Should(SatisfyAny(BeNil(), &utils.AlreadyExistMatcher{}))
  1944  }
  1945  
  1946  func setupTestDefinitions(ctx context.Context, defs []string, namespace string) {
  1947  	_, file, _, _ := sysruntime.Caller(0)
  1948  	for _, def := range defs {
  1949  		Expect(definition.InstallDefinitionFromYAML(ctx, k8sClient, filepath.Join(filepath.Dir(filepath.Dir(file)), fmt.Sprintf("./controllers/testdata/%s.yaml", def)), func(s string) string {
  1950  			return strings.ReplaceAll(s, "vela-system", namespace)
  1951  		})).Should(SatisfyAny(BeNil(), &utils.AlreadyExistMatcher{}))
  1952  	}
  1953  }