github.com/Jeffail/benthos/v3@v3.65.0/lib/processor/workflow_deprecated_test.go (about)

     1  package processor
     2  
     3  import (
     4  	"reflect"
     5  	"testing"
     6  
     7  	"github.com/Jeffail/benthos/v3/lib/condition"
     8  	"github.com/Jeffail/benthos/v3/lib/log"
     9  	"github.com/Jeffail/benthos/v3/lib/message"
    10  	"github.com/Jeffail/benthos/v3/lib/metrics"
    11  )
    12  
    13  func TestWorkflowCircular(t *testing.T) {
    14  	conf := NewConfig()
    15  	conf.Type = "workflow"
    16  	conf.Workflow.Stages["foo"] = createProcMapConf("tmp.baz", "tmp.foo")
    17  	conf.Workflow.Stages["bar"] = createProcMapConf("tmp.foo", "tmp.bar")
    18  	conf.Workflow.Stages["baz"] = createProcMapConf("tmp.bar", "tmp.baz")
    19  
    20  	_, err := New(conf, nil, log.Noop(), metrics.Noop())
    21  	if err == nil {
    22  		t.Error("expected error from circular deps")
    23  	}
    24  }
    25  
    26  func TestWorkflowBadNames(t *testing.T) {
    27  	conf := NewConfig()
    28  	conf.Type = "workflow"
    29  	conf.Workflow.Stages["foo,bar"] = createProcMapConf("tmp.baz", "tmp.foo")
    30  
    31  	_, err := New(conf, nil, log.Noop(), metrics.Noop())
    32  	if err == nil {
    33  		t.Error("expected error from bad name")
    34  	}
    35  
    36  	conf = NewConfig()
    37  	conf.Type = "workflow"
    38  	conf.Workflow.Stages["foo$"] = createProcMapConf("tmp.baz", "tmp.foo")
    39  
    40  	if _, err = New(conf, nil, log.Noop(), metrics.Noop()); err == nil {
    41  		t.Error("expected error from bad name")
    42  	}
    43  }
    44  
    45  func TestWorkflowGoodNames(t *testing.T) {
    46  	conf := NewConfig()
    47  	conf.Type = "workflow"
    48  	conf.Workflow.Stages["foo_bar"] = createProcMapConf("tmp.baz", "tmp.foo")
    49  	conf.Workflow.Stages["FOO-BAR"] = createProcMapConf("tmp.baz", "tmp.foo")
    50  	conf.Workflow.Stages["FOO-9"] = createProcMapConf("tmp.baz", "tmp.foo")
    51  	conf.Workflow.Stages["FOO-10"] = createProcMapConf("tmp.baz", "tmp.foo")
    52  
    53  	if _, err := New(conf, nil, log.Noop(), metrics.Noop()); err != nil {
    54  		t.Error(err)
    55  	}
    56  }
    57  
    58  func TestWorkflowSimple(t *testing.T) {
    59  	conf := NewConfig()
    60  	conf.Type = "workflow"
    61  	conf.Workflow.MetaPath = "0meta"
    62  	conf.Workflow.Stages["foo"] = createProcMapConf("root", "tmp.foo")
    63  	conf.Workflow.Stages["bar"] = createProcMapConf("tmp.foo", "tmp.bar")
    64  	conf.Workflow.Stages["baz"] = createProcMapConf("tmp.bar", "tmp.baz")
    65  
    66  	c, err := New(conf, nil, log.Noop(), metrics.Noop())
    67  	if err != nil {
    68  		t.Fatal(err)
    69  	}
    70  	exp := [][]byte{
    71  		[]byte(`{"0meta":{"failed":["bar","baz","foo"],"skipped":[],"succeeded":[]},"oops":"no root"}`),
    72  		[]byte(`{"0meta":{"failed":[],"skipped":[],"succeeded":["bar","baz","foo"]},"root":"foobarbaz","tmp":{"bar":"foobarbaz","baz":"foobarbaz","foo":"foobarbaz"}}`),
    73  		[]byte(`{"0meta":{"failed":[],"skipped":[],"succeeded":["bar","baz","foo"]},"root":"foobarbaz","tmp":{"also":"here","bar":"foobarbaz","baz":"foobarbaz","foo":"foobarbaz"}}`),
    74  	}
    75  
    76  	msg, res := c.ProcessMessage(message.New([][]byte{
    77  		[]byte(`{"oops":"no root"}`),
    78  		[]byte(`{"root":"foobarbaz"}`),
    79  		[]byte(`{"root":"foobarbaz","tmp":{"also":"here"}}`),
    80  	}))
    81  	if res != nil {
    82  		t.Error(res.Error())
    83  	}
    84  	if act := message.GetAllBytes(msg[0]); !reflect.DeepEqual(act, exp) {
    85  		t.Errorf("Wrong result: %s != %s", act, exp)
    86  	}
    87  }
    88  
    89  func TestWorkflowNoMeta(t *testing.T) {
    90  	conf := NewConfig()
    91  	conf.Type = "workflow"
    92  	conf.Workflow.MetaPath = ""
    93  	conf.Workflow.Stages["foo"] = createProcMapConf("root", "tmp.foo")
    94  	conf.Workflow.Stages["bar"] = createProcMapConf("tmp.foo", "tmp.bar")
    95  	conf.Workflow.Stages["baz"] = createProcMapConf("tmp.bar", "tmp.baz")
    96  
    97  	c, err := New(conf, nil, log.Noop(), metrics.Noop())
    98  	if err != nil {
    99  		t.Fatal(err)
   100  	}
   101  	exp := [][]byte{
   102  		[]byte(`{"oops":"no root"}`),
   103  		[]byte(`{"root":"foobarbaz","tmp":{"bar":"foobarbaz","baz":"foobarbaz","foo":"foobarbaz"}}`),
   104  		[]byte(`{"root":"foobarbaz","tmp":{"also":"here","bar":"foobarbaz","baz":"foobarbaz","foo":"foobarbaz"}}`),
   105  	}
   106  
   107  	msg, res := c.ProcessMessage(message.New([][]byte{
   108  		[]byte(`{"oops":"no root"}`),
   109  		[]byte(`{"root":"foobarbaz"}`),
   110  		[]byte(`{"root":"foobarbaz","tmp":{"also":"here"}}`),
   111  	}))
   112  	if res != nil {
   113  		t.Error(res.Error())
   114  	}
   115  	if act := message.GetAllBytes(msg[0]); !reflect.DeepEqual(act, exp) {
   116  		t.Errorf("Wrong result: %s != %s", act, exp)
   117  	}
   118  }
   119  
   120  func TestWorkflowSimpleFromPrevious(t *testing.T) {
   121  	conf := NewConfig()
   122  	conf.Type = "workflow"
   123  	conf.Workflow.MetaPath = "0meta"
   124  	conf.Workflow.Stages["foo"] = createProcMapConf("root", "tmp.foo")
   125  	conf.Workflow.Stages["bar"] = createProcMapConf("tmp.foo", "tmp.bar")
   126  	conf.Workflow.Stages["baz"] = createProcMapConf("tmp.bar", "tmp.baz")
   127  
   128  	c, err := New(conf, nil, log.Noop(), metrics.Noop())
   129  	if err != nil {
   130  		t.Fatal(err)
   131  	}
   132  	exp := [][]byte{
   133  		[]byte(`{"0meta":{"failed":["baz","foo"],"previous":{"failed":["baz","foo"],"skipped":["bar"],"succeeded":[]},"skipped":["bar"],"succeeded":[]},"oops":"no root"}`),
   134  		[]byte(`{"0meta":{"failed":[],"previous":{"failed":[],"skipped":["baz"],"succeeded":[]},"skipped":["baz"],"succeeded":["bar","foo"]},"root":"foobarbaz","tmp":{"bar":"foobarbaz","foo":"foobarbaz"}}`),
   135  		[]byte(`{"0meta":{"failed":[],"previous":{"failed":[],"skipped":[],"succeeded":["baz"]},"skipped":["baz"],"succeeded":["bar","foo"]},"root":"foobarbaz","tmp":{"also":"here","bar":"foobarbaz","foo":"foobarbaz"}}`),
   136  	}
   137  
   138  	msg, res := c.ProcessMessage(message.New([][]byte{
   139  		[]byte(`{"0meta":{"failed":["baz","foo"],"skipped":["bar"],"succeeded":[]},"oops":"no root"}`),
   140  		[]byte(`{"0meta":{"failed":[],"skipped":["baz"],"succeeded":[]},"root":"foobarbaz"}`),
   141  		[]byte(`{"0meta":{"failed":[],"skipped":[],"succeeded":["baz"]},"root":"foobarbaz","tmp":{"also":"here"}}`),
   142  	}))
   143  	if res != nil {
   144  		t.Error(res.Error())
   145  	}
   146  	if act := message.GetAllBytes(msg[0]); !reflect.DeepEqual(act, exp) {
   147  		t.Errorf("Wrong result: %s != %s", act, exp)
   148  	}
   149  }
   150  
   151  func TestWorkflowParallel(t *testing.T) {
   152  	condConf := condition.NewConfig()
   153  	condConf.Type = condition.TypeText
   154  	condConf.Text.Operator = "contains"
   155  	condConf.Text.Arg = "foo"
   156  
   157  	procConf := NewConfig()
   158  	procConf.Type = TypeMetadata
   159  	procConf.Metadata.Operator = "set"
   160  	procConf.Metadata.Key = "A"
   161  	procConf.Metadata.Value = "foo: ${!json(\"foo\")}"
   162  
   163  	fooConf := NewProcessMapConfig()
   164  	fooConf.Conditions = []condition.Config{condConf}
   165  	fooConf.Premap["."] = "."
   166  	fooConf.Postmap["tmp.A"] = "."
   167  	fooConf.Processors = []Config{procConf}
   168  
   169  	condConf = condition.NewConfig()
   170  	condConf.Type = condition.TypeText
   171  	condConf.Text.Operator = "contains"
   172  	condConf.Text.Arg = "bar"
   173  
   174  	procConf = NewConfig()
   175  	procConf.Type = TypeMetadata
   176  	procConf.Metadata.Operator = "set"
   177  	procConf.Metadata.Key = "A"
   178  	procConf.Metadata.Value = "bar: ${!json(\"bar\")}"
   179  
   180  	barConf := NewProcessMapConfig()
   181  	barConf.Conditions = []condition.Config{condConf}
   182  	barConf.Premap["."] = "."
   183  	barConf.Postmap["tmp.A"] = "."
   184  	barConf.Processors = []Config{procConf}
   185  
   186  	condConf = condition.NewConfig()
   187  	condConf.Type = condition.TypeText
   188  	condConf.Text.Operator = "contains"
   189  	condConf.Text.Arg = "baz"
   190  
   191  	procConf = NewConfig()
   192  	procConf.Type = TypeMetadata
   193  	procConf.Metadata.Operator = "set"
   194  	procConf.Metadata.Key = "B"
   195  	procConf.Metadata.Value = "${!meta(\"A\")}"
   196  
   197  	bazConf := NewProcessMapConfig()
   198  	bazConf.Conditions = []condition.Config{condConf}
   199  	bazConf.Premap["."] = "tmp.A"
   200  	bazConf.Postmap["tmp.B"] = "."
   201  	bazConf.Processors = []Config{procConf}
   202  
   203  	condConf = condition.NewConfig()
   204  	condConf.Type = condition.TypeText
   205  	condConf.Text.Operator = "contains"
   206  	condConf.Text.Arg = "qux"
   207  
   208  	procConf = NewConfig()
   209  	procConf.Type = TypeMetadata
   210  	procConf.Metadata.Operator = "set"
   211  	procConf.Metadata.Key = "B"
   212  	procConf.Metadata.Value = "${!meta(\"A\")}"
   213  
   214  	quxConf := NewProcessMapConfig()
   215  	quxConf.Conditions = []condition.Config{condConf}
   216  	quxConf.Premap["."] = "tmp.A"
   217  	quxConf.Postmap["tmp.B"] = "."
   218  	quxConf.Processors = []Config{procConf}
   219  
   220  	conf := NewConfig()
   221  	conf.Type = TypeWorkflow
   222  	conf.Workflow.MetaPath = "0meta"
   223  	conf.Workflow.Stages["foo"] = DepProcessMapConfig{
   224  		ProcessMapConfig: fooConf,
   225  	}
   226  	conf.Workflow.Stages["bar"] = DepProcessMapConfig{
   227  		ProcessMapConfig: barConf,
   228  	}
   229  	conf.Workflow.Stages["baz"] = DepProcessMapConfig{
   230  		ProcessMapConfig: bazConf,
   231  	}
   232  	conf.Workflow.Stages["qux"] = DepProcessMapConfig{
   233  		ProcessMapConfig: quxConf,
   234  	}
   235  
   236  	c, err := New(conf, nil, log.Noop(), metrics.Noop())
   237  	if err != nil {
   238  		t.Fatal(err)
   239  	}
   240  	expParts := [][]byte{
   241  		[]byte(`{"0meta":{"failed":[],"skipped":["bar","baz"],"succeeded":["foo","qux"]},"foo":"1","qux":"2","tmp":{"A":{"foo":"1","qux":"2"},"B":{"foo":"1","qux":"2"}}}`),
   242  		[]byte(`{"0meta":{"failed":[],"skipped":["baz","foo"],"succeeded":["bar","qux"]},"bar":"3","qux":"4","tmp":{"A":{"bar":"3","qux":"4"},"B":{"bar":"3","qux":"4"}}}`),
   243  		[]byte(`{"0meta":{"failed":[],"skipped":["bar","qux"],"succeeded":["baz","foo"]},"baz":"6","foo":"5","tmp":{"A":{"baz":"6","foo":"5"},"B":{"baz":"6","foo":"5"}}}`),
   244  		[]byte(`{"0meta":{"failed":[],"skipped":["foo","qux"],"succeeded":["bar","baz"]},"bar":"7","baz":"8","tmp":{"A":{"bar":"7","baz":"8"},"B":{"bar":"7","baz":"8"}}}`),
   245  		[]byte(`{"0meta":{"failed":[],"skipped":["bar","baz","qux"],"succeeded":["foo"]},"foo":"9","tmp":{"A":{"foo":"9"}}}`),
   246  		[]byte(`{"0meta":{"failed":[],"skipped":["baz","foo","qux"],"succeeded":["bar"]},"bar":"10","tmp":{"A":{"bar":"10"}}}`),
   247  		[]byte(`{"0meta":{"failed":[],"skipped":["bar","baz"],"succeeded":["foo","qux"]},"foo":"11","qux":"12","tmp":{"A":{"foo":"11","qux":"12"},"B":{"foo":"11","qux":"12"}}}`),
   248  		[]byte(`{"0meta":{"failed":[],"skipped":["baz","foo"],"succeeded":["bar","qux"]},"bar":"13","qux":"14","tmp":{"A":{"bar":"13","qux":"14"},"B":{"bar":"13","qux":"14"}}}`),
   249  		[]byte(`{"0meta":{"failed":[],"skipped":["bar","qux"],"succeeded":["baz","foo"]},"baz":"16","foo":"15","tmp":{"A":{"baz":"16","foo":"15"},"B":{"baz":"16","foo":"15"}}}`),
   250  		[]byte(`{"0meta":{"failed":[],"skipped":["foo","qux"],"succeeded":["bar","baz"]},"bar":"17","baz":"18","tmp":{"A":{"bar":"17","baz":"18"},"B":{"bar":"17","baz":"18"}}}`),
   251  	}
   252  
   253  	msg, res := c.ProcessMessage(message.New([][]byte{
   254  		[]byte(`{"foo":"1","qux":"2"}`),
   255  		[]byte(`{"bar":"3","qux":"4"}`),
   256  		[]byte(`{"foo":"5","baz":"6"}`),
   257  		[]byte(`{"bar":"7","baz":"8"}`),
   258  		[]byte(`{"foo":"9"}`),
   259  		[]byte(`{"bar":"10"}`),
   260  		[]byte(`{"foo":"11","qux":"12"}`),
   261  		[]byte(`{"bar":"13","qux":"14"}`),
   262  		[]byte(`{"foo":"15","baz":"16"}`),
   263  		[]byte(`{"bar":"17","baz":"18"}`),
   264  		[]byte(`{"baz":"19"}`),
   265  		[]byte(`{"qux":"20"}`),
   266  	}))
   267  	if res != nil {
   268  		t.Error(res.Error())
   269  	}
   270  
   271  	actParts := message.GetAllBytes(msg[0])
   272  	for i, exp := range expParts {
   273  		if len(actParts) <= i {
   274  			t.Errorf("Missing result part index '%v': %s", i, exp)
   275  		}
   276  		if actStr, expStr := string(actParts[i]), string(exp); actStr != expStr {
   277  			t.Errorf("Wrong part result: %v != %v", actStr, expStr)
   278  		}
   279  	}
   280  }
   281  
   282  func TestWorkflowRoot(t *testing.T) {
   283  	conf := NewConfig()
   284  	conf.Type = "workflow"
   285  	conf.Workflow.MetaPath = "0meta"
   286  	conf.Workflow.Stages["foo"] = createProcMapConf("root", "tmp.foo")
   287  	conf.Workflow.Stages["bar"] = createProcMapConf("", "tmp.bar")
   288  	conf.Workflow.Stages["baz"] = createProcMapConf("tmp.bar", "tmp.baz")
   289  
   290  	c, err := New(conf, nil, log.Noop(), metrics.Noop())
   291  	if err != nil {
   292  		t.Fatal(err)
   293  	}
   294  	exp := [][]byte{
   295  		[]byte(`{"0meta":{"failed":["foo"],"skipped":[],"succeeded":["bar","baz"]},"oops":"no root","tmp":{"bar":{"oops":"no root"},"baz":{"oops":"no root"}}}`),
   296  		[]byte(`{"0meta":{"failed":[],"skipped":[],"succeeded":["bar","baz","foo"]},"root":"foobarbaz","tmp":{"bar":{"root":"foobarbaz"},"baz":{"root":"foobarbaz"},"foo":"foobarbaz"}}`),
   297  		[]byte(`{"0meta":{"failed":[],"skipped":[],"succeeded":["bar","baz","foo"]},"root":"foobarbaz","tmp":{"also":"here","bar":{"root":"foobarbaz","tmp":{"also":"here"}},"baz":{"root":"foobarbaz","tmp":{"also":"here"}},"foo":"foobarbaz"}}`),
   298  	}
   299  
   300  	msg, res := c.ProcessMessage(message.New([][]byte{
   301  		[]byte(`{"oops":"no root"}`),
   302  		[]byte(`{"root":"foobarbaz"}`),
   303  		[]byte(`{"root":"foobarbaz","tmp":{"also":"here"}}`),
   304  	}))
   305  	if res != nil {
   306  		t.Error(res.Error())
   307  	}
   308  	if act := message.GetAllBytes(msg[0]); !reflect.DeepEqual(act, exp) {
   309  		t.Errorf("Wrong result: %s != %s", act, exp)
   310  	}
   311  }
   312  
   313  func TestWorkflowDiamond(t *testing.T) {
   314  	conf := NewConfig()
   315  	conf.Type = "workflow"
   316  	conf.Workflow.MetaPath = "0meta"
   317  	conf.Workflow.Stages["foo"] = createProcMapConf(".", "foo_result")
   318  	conf.Workflow.Stages["bar"] = createProcMapConf("root.path", "bar_result")
   319  	conf.Workflow.Stages["baz"] = createProcMapConf(".", "baz_result", "foo_result", "bar_result")
   320  
   321  	c, err := New(conf, nil, log.Noop(), metrics.Noop())
   322  	if err != nil {
   323  		t.Fatal(err)
   324  	}
   325  	exp := [][]byte{
   326  		[]byte(`{"0meta":{"failed":[],"skipped":[],"succeeded":["bar","baz","foo"]},"bar_result":"nested","baz_result":{"bar_result":"nested","foo_result":{"outter":"value","root":{"path":"nested"}},"outter":"value","root":{"path":"nested"}},"foo_result":{"outter":"value","root":{"path":"nested"}},"outter":"value","root":{"path":"nested"}}`),
   327  	}
   328  
   329  	msg, res := c.ProcessMessage(message.New([][]byte{
   330  		[]byte(`{"outter":"value","root":{"path":"nested"}}`),
   331  	}))
   332  	if res != nil {
   333  		t.Error(res.Error())
   334  	}
   335  	if act := message.GetAllBytes(msg[0]); !reflect.DeepEqual(act, exp) {
   336  		t.Errorf("Wrong result: %s != %s", act, exp)
   337  	}
   338  }