github.com/Jeffail/benthos/v3@v3.65.0/lib/processor/workflow_deprecated_test.go (about) 1 package processor 2 3 import ( 4 "reflect" 5 "testing" 6 7 "github.com/Jeffail/benthos/v3/lib/condition" 8 "github.com/Jeffail/benthos/v3/lib/log" 9 "github.com/Jeffail/benthos/v3/lib/message" 10 "github.com/Jeffail/benthos/v3/lib/metrics" 11 ) 12 13 func TestWorkflowCircular(t *testing.T) { 14 conf := NewConfig() 15 conf.Type = "workflow" 16 conf.Workflow.Stages["foo"] = createProcMapConf("tmp.baz", "tmp.foo") 17 conf.Workflow.Stages["bar"] = createProcMapConf("tmp.foo", "tmp.bar") 18 conf.Workflow.Stages["baz"] = createProcMapConf("tmp.bar", "tmp.baz") 19 20 _, err := New(conf, nil, log.Noop(), metrics.Noop()) 21 if err == nil { 22 t.Error("expected error from circular deps") 23 } 24 } 25 26 func TestWorkflowBadNames(t *testing.T) { 27 conf := NewConfig() 28 conf.Type = "workflow" 29 conf.Workflow.Stages["foo,bar"] = createProcMapConf("tmp.baz", "tmp.foo") 30 31 _, err := New(conf, nil, log.Noop(), metrics.Noop()) 32 if err == nil { 33 t.Error("expected error from bad name") 34 } 35 36 conf = NewConfig() 37 conf.Type = "workflow" 38 conf.Workflow.Stages["foo$"] = createProcMapConf("tmp.baz", "tmp.foo") 39 40 if _, err = New(conf, nil, log.Noop(), metrics.Noop()); err == nil { 41 t.Error("expected error from bad name") 42 } 43 } 44 45 func TestWorkflowGoodNames(t *testing.T) { 46 conf := NewConfig() 47 conf.Type = "workflow" 48 conf.Workflow.Stages["foo_bar"] = createProcMapConf("tmp.baz", "tmp.foo") 49 conf.Workflow.Stages["FOO-BAR"] = createProcMapConf("tmp.baz", "tmp.foo") 50 conf.Workflow.Stages["FOO-9"] = createProcMapConf("tmp.baz", "tmp.foo") 51 conf.Workflow.Stages["FOO-10"] = createProcMapConf("tmp.baz", "tmp.foo") 52 53 if _, err := New(conf, nil, log.Noop(), metrics.Noop()); err != nil { 54 t.Error(err) 55 } 56 } 57 58 func TestWorkflowSimple(t *testing.T) { 59 conf := NewConfig() 60 conf.Type = "workflow" 61 conf.Workflow.MetaPath = "0meta" 62 conf.Workflow.Stages["foo"] = createProcMapConf("root", "tmp.foo") 63 conf.Workflow.Stages["bar"] = createProcMapConf("tmp.foo", "tmp.bar") 64 conf.Workflow.Stages["baz"] = createProcMapConf("tmp.bar", "tmp.baz") 65 66 c, err := New(conf, nil, log.Noop(), metrics.Noop()) 67 if err != nil { 68 t.Fatal(err) 69 } 70 exp := [][]byte{ 71 []byte(`{"0meta":{"failed":["bar","baz","foo"],"skipped":[],"succeeded":[]},"oops":"no root"}`), 72 []byte(`{"0meta":{"failed":[],"skipped":[],"succeeded":["bar","baz","foo"]},"root":"foobarbaz","tmp":{"bar":"foobarbaz","baz":"foobarbaz","foo":"foobarbaz"}}`), 73 []byte(`{"0meta":{"failed":[],"skipped":[],"succeeded":["bar","baz","foo"]},"root":"foobarbaz","tmp":{"also":"here","bar":"foobarbaz","baz":"foobarbaz","foo":"foobarbaz"}}`), 74 } 75 76 msg, res := c.ProcessMessage(message.New([][]byte{ 77 []byte(`{"oops":"no root"}`), 78 []byte(`{"root":"foobarbaz"}`), 79 []byte(`{"root":"foobarbaz","tmp":{"also":"here"}}`), 80 })) 81 if res != nil { 82 t.Error(res.Error()) 83 } 84 if act := message.GetAllBytes(msg[0]); !reflect.DeepEqual(act, exp) { 85 t.Errorf("Wrong result: %s != %s", act, exp) 86 } 87 } 88 89 func TestWorkflowNoMeta(t *testing.T) { 90 conf := NewConfig() 91 conf.Type = "workflow" 92 conf.Workflow.MetaPath = "" 93 conf.Workflow.Stages["foo"] = createProcMapConf("root", "tmp.foo") 94 conf.Workflow.Stages["bar"] = createProcMapConf("tmp.foo", "tmp.bar") 95 conf.Workflow.Stages["baz"] = createProcMapConf("tmp.bar", "tmp.baz") 96 97 c, err := New(conf, nil, log.Noop(), metrics.Noop()) 98 if err != nil { 99 t.Fatal(err) 100 } 101 exp := [][]byte{ 102 []byte(`{"oops":"no root"}`), 103 []byte(`{"root":"foobarbaz","tmp":{"bar":"foobarbaz","baz":"foobarbaz","foo":"foobarbaz"}}`), 104 []byte(`{"root":"foobarbaz","tmp":{"also":"here","bar":"foobarbaz","baz":"foobarbaz","foo":"foobarbaz"}}`), 105 } 106 107 msg, res := c.ProcessMessage(message.New([][]byte{ 108 []byte(`{"oops":"no root"}`), 109 []byte(`{"root":"foobarbaz"}`), 110 []byte(`{"root":"foobarbaz","tmp":{"also":"here"}}`), 111 })) 112 if res != nil { 113 t.Error(res.Error()) 114 } 115 if act := message.GetAllBytes(msg[0]); !reflect.DeepEqual(act, exp) { 116 t.Errorf("Wrong result: %s != %s", act, exp) 117 } 118 } 119 120 func TestWorkflowSimpleFromPrevious(t *testing.T) { 121 conf := NewConfig() 122 conf.Type = "workflow" 123 conf.Workflow.MetaPath = "0meta" 124 conf.Workflow.Stages["foo"] = createProcMapConf("root", "tmp.foo") 125 conf.Workflow.Stages["bar"] = createProcMapConf("tmp.foo", "tmp.bar") 126 conf.Workflow.Stages["baz"] = createProcMapConf("tmp.bar", "tmp.baz") 127 128 c, err := New(conf, nil, log.Noop(), metrics.Noop()) 129 if err != nil { 130 t.Fatal(err) 131 } 132 exp := [][]byte{ 133 []byte(`{"0meta":{"failed":["baz","foo"],"previous":{"failed":["baz","foo"],"skipped":["bar"],"succeeded":[]},"skipped":["bar"],"succeeded":[]},"oops":"no root"}`), 134 []byte(`{"0meta":{"failed":[],"previous":{"failed":[],"skipped":["baz"],"succeeded":[]},"skipped":["baz"],"succeeded":["bar","foo"]},"root":"foobarbaz","tmp":{"bar":"foobarbaz","foo":"foobarbaz"}}`), 135 []byte(`{"0meta":{"failed":[],"previous":{"failed":[],"skipped":[],"succeeded":["baz"]},"skipped":["baz"],"succeeded":["bar","foo"]},"root":"foobarbaz","tmp":{"also":"here","bar":"foobarbaz","foo":"foobarbaz"}}`), 136 } 137 138 msg, res := c.ProcessMessage(message.New([][]byte{ 139 []byte(`{"0meta":{"failed":["baz","foo"],"skipped":["bar"],"succeeded":[]},"oops":"no root"}`), 140 []byte(`{"0meta":{"failed":[],"skipped":["baz"],"succeeded":[]},"root":"foobarbaz"}`), 141 []byte(`{"0meta":{"failed":[],"skipped":[],"succeeded":["baz"]},"root":"foobarbaz","tmp":{"also":"here"}}`), 142 })) 143 if res != nil { 144 t.Error(res.Error()) 145 } 146 if act := message.GetAllBytes(msg[0]); !reflect.DeepEqual(act, exp) { 147 t.Errorf("Wrong result: %s != %s", act, exp) 148 } 149 } 150 151 func TestWorkflowParallel(t *testing.T) { 152 condConf := condition.NewConfig() 153 condConf.Type = condition.TypeText 154 condConf.Text.Operator = "contains" 155 condConf.Text.Arg = "foo" 156 157 procConf := NewConfig() 158 procConf.Type = TypeMetadata 159 procConf.Metadata.Operator = "set" 160 procConf.Metadata.Key = "A" 161 procConf.Metadata.Value = "foo: ${!json(\"foo\")}" 162 163 fooConf := NewProcessMapConfig() 164 fooConf.Conditions = []condition.Config{condConf} 165 fooConf.Premap["."] = "." 166 fooConf.Postmap["tmp.A"] = "." 167 fooConf.Processors = []Config{procConf} 168 169 condConf = condition.NewConfig() 170 condConf.Type = condition.TypeText 171 condConf.Text.Operator = "contains" 172 condConf.Text.Arg = "bar" 173 174 procConf = NewConfig() 175 procConf.Type = TypeMetadata 176 procConf.Metadata.Operator = "set" 177 procConf.Metadata.Key = "A" 178 procConf.Metadata.Value = "bar: ${!json(\"bar\")}" 179 180 barConf := NewProcessMapConfig() 181 barConf.Conditions = []condition.Config{condConf} 182 barConf.Premap["."] = "." 183 barConf.Postmap["tmp.A"] = "." 184 barConf.Processors = []Config{procConf} 185 186 condConf = condition.NewConfig() 187 condConf.Type = condition.TypeText 188 condConf.Text.Operator = "contains" 189 condConf.Text.Arg = "baz" 190 191 procConf = NewConfig() 192 procConf.Type = TypeMetadata 193 procConf.Metadata.Operator = "set" 194 procConf.Metadata.Key = "B" 195 procConf.Metadata.Value = "${!meta(\"A\")}" 196 197 bazConf := NewProcessMapConfig() 198 bazConf.Conditions = []condition.Config{condConf} 199 bazConf.Premap["."] = "tmp.A" 200 bazConf.Postmap["tmp.B"] = "." 201 bazConf.Processors = []Config{procConf} 202 203 condConf = condition.NewConfig() 204 condConf.Type = condition.TypeText 205 condConf.Text.Operator = "contains" 206 condConf.Text.Arg = "qux" 207 208 procConf = NewConfig() 209 procConf.Type = TypeMetadata 210 procConf.Metadata.Operator = "set" 211 procConf.Metadata.Key = "B" 212 procConf.Metadata.Value = "${!meta(\"A\")}" 213 214 quxConf := NewProcessMapConfig() 215 quxConf.Conditions = []condition.Config{condConf} 216 quxConf.Premap["."] = "tmp.A" 217 quxConf.Postmap["tmp.B"] = "." 218 quxConf.Processors = []Config{procConf} 219 220 conf := NewConfig() 221 conf.Type = TypeWorkflow 222 conf.Workflow.MetaPath = "0meta" 223 conf.Workflow.Stages["foo"] = DepProcessMapConfig{ 224 ProcessMapConfig: fooConf, 225 } 226 conf.Workflow.Stages["bar"] = DepProcessMapConfig{ 227 ProcessMapConfig: barConf, 228 } 229 conf.Workflow.Stages["baz"] = DepProcessMapConfig{ 230 ProcessMapConfig: bazConf, 231 } 232 conf.Workflow.Stages["qux"] = DepProcessMapConfig{ 233 ProcessMapConfig: quxConf, 234 } 235 236 c, err := New(conf, nil, log.Noop(), metrics.Noop()) 237 if err != nil { 238 t.Fatal(err) 239 } 240 expParts := [][]byte{ 241 []byte(`{"0meta":{"failed":[],"skipped":["bar","baz"],"succeeded":["foo","qux"]},"foo":"1","qux":"2","tmp":{"A":{"foo":"1","qux":"2"},"B":{"foo":"1","qux":"2"}}}`), 242 []byte(`{"0meta":{"failed":[],"skipped":["baz","foo"],"succeeded":["bar","qux"]},"bar":"3","qux":"4","tmp":{"A":{"bar":"3","qux":"4"},"B":{"bar":"3","qux":"4"}}}`), 243 []byte(`{"0meta":{"failed":[],"skipped":["bar","qux"],"succeeded":["baz","foo"]},"baz":"6","foo":"5","tmp":{"A":{"baz":"6","foo":"5"},"B":{"baz":"6","foo":"5"}}}`), 244 []byte(`{"0meta":{"failed":[],"skipped":["foo","qux"],"succeeded":["bar","baz"]},"bar":"7","baz":"8","tmp":{"A":{"bar":"7","baz":"8"},"B":{"bar":"7","baz":"8"}}}`), 245 []byte(`{"0meta":{"failed":[],"skipped":["bar","baz","qux"],"succeeded":["foo"]},"foo":"9","tmp":{"A":{"foo":"9"}}}`), 246 []byte(`{"0meta":{"failed":[],"skipped":["baz","foo","qux"],"succeeded":["bar"]},"bar":"10","tmp":{"A":{"bar":"10"}}}`), 247 []byte(`{"0meta":{"failed":[],"skipped":["bar","baz"],"succeeded":["foo","qux"]},"foo":"11","qux":"12","tmp":{"A":{"foo":"11","qux":"12"},"B":{"foo":"11","qux":"12"}}}`), 248 []byte(`{"0meta":{"failed":[],"skipped":["baz","foo"],"succeeded":["bar","qux"]},"bar":"13","qux":"14","tmp":{"A":{"bar":"13","qux":"14"},"B":{"bar":"13","qux":"14"}}}`), 249 []byte(`{"0meta":{"failed":[],"skipped":["bar","qux"],"succeeded":["baz","foo"]},"baz":"16","foo":"15","tmp":{"A":{"baz":"16","foo":"15"},"B":{"baz":"16","foo":"15"}}}`), 250 []byte(`{"0meta":{"failed":[],"skipped":["foo","qux"],"succeeded":["bar","baz"]},"bar":"17","baz":"18","tmp":{"A":{"bar":"17","baz":"18"},"B":{"bar":"17","baz":"18"}}}`), 251 } 252 253 msg, res := c.ProcessMessage(message.New([][]byte{ 254 []byte(`{"foo":"1","qux":"2"}`), 255 []byte(`{"bar":"3","qux":"4"}`), 256 []byte(`{"foo":"5","baz":"6"}`), 257 []byte(`{"bar":"7","baz":"8"}`), 258 []byte(`{"foo":"9"}`), 259 []byte(`{"bar":"10"}`), 260 []byte(`{"foo":"11","qux":"12"}`), 261 []byte(`{"bar":"13","qux":"14"}`), 262 []byte(`{"foo":"15","baz":"16"}`), 263 []byte(`{"bar":"17","baz":"18"}`), 264 []byte(`{"baz":"19"}`), 265 []byte(`{"qux":"20"}`), 266 })) 267 if res != nil { 268 t.Error(res.Error()) 269 } 270 271 actParts := message.GetAllBytes(msg[0]) 272 for i, exp := range expParts { 273 if len(actParts) <= i { 274 t.Errorf("Missing result part index '%v': %s", i, exp) 275 } 276 if actStr, expStr := string(actParts[i]), string(exp); actStr != expStr { 277 t.Errorf("Wrong part result: %v != %v", actStr, expStr) 278 } 279 } 280 } 281 282 func TestWorkflowRoot(t *testing.T) { 283 conf := NewConfig() 284 conf.Type = "workflow" 285 conf.Workflow.MetaPath = "0meta" 286 conf.Workflow.Stages["foo"] = createProcMapConf("root", "tmp.foo") 287 conf.Workflow.Stages["bar"] = createProcMapConf("", "tmp.bar") 288 conf.Workflow.Stages["baz"] = createProcMapConf("tmp.bar", "tmp.baz") 289 290 c, err := New(conf, nil, log.Noop(), metrics.Noop()) 291 if err != nil { 292 t.Fatal(err) 293 } 294 exp := [][]byte{ 295 []byte(`{"0meta":{"failed":["foo"],"skipped":[],"succeeded":["bar","baz"]},"oops":"no root","tmp":{"bar":{"oops":"no root"},"baz":{"oops":"no root"}}}`), 296 []byte(`{"0meta":{"failed":[],"skipped":[],"succeeded":["bar","baz","foo"]},"root":"foobarbaz","tmp":{"bar":{"root":"foobarbaz"},"baz":{"root":"foobarbaz"},"foo":"foobarbaz"}}`), 297 []byte(`{"0meta":{"failed":[],"skipped":[],"succeeded":["bar","baz","foo"]},"root":"foobarbaz","tmp":{"also":"here","bar":{"root":"foobarbaz","tmp":{"also":"here"}},"baz":{"root":"foobarbaz","tmp":{"also":"here"}},"foo":"foobarbaz"}}`), 298 } 299 300 msg, res := c.ProcessMessage(message.New([][]byte{ 301 []byte(`{"oops":"no root"}`), 302 []byte(`{"root":"foobarbaz"}`), 303 []byte(`{"root":"foobarbaz","tmp":{"also":"here"}}`), 304 })) 305 if res != nil { 306 t.Error(res.Error()) 307 } 308 if act := message.GetAllBytes(msg[0]); !reflect.DeepEqual(act, exp) { 309 t.Errorf("Wrong result: %s != %s", act, exp) 310 } 311 } 312 313 func TestWorkflowDiamond(t *testing.T) { 314 conf := NewConfig() 315 conf.Type = "workflow" 316 conf.Workflow.MetaPath = "0meta" 317 conf.Workflow.Stages["foo"] = createProcMapConf(".", "foo_result") 318 conf.Workflow.Stages["bar"] = createProcMapConf("root.path", "bar_result") 319 conf.Workflow.Stages["baz"] = createProcMapConf(".", "baz_result", "foo_result", "bar_result") 320 321 c, err := New(conf, nil, log.Noop(), metrics.Noop()) 322 if err != nil { 323 t.Fatal(err) 324 } 325 exp := [][]byte{ 326 []byte(`{"0meta":{"failed":[],"skipped":[],"succeeded":["bar","baz","foo"]},"bar_result":"nested","baz_result":{"bar_result":"nested","foo_result":{"outter":"value","root":{"path":"nested"}},"outter":"value","root":{"path":"nested"}},"foo_result":{"outter":"value","root":{"path":"nested"}},"outter":"value","root":{"path":"nested"}}`), 327 } 328 329 msg, res := c.ProcessMessage(message.New([][]byte{ 330 []byte(`{"outter":"value","root":{"path":"nested"}}`), 331 })) 332 if res != nil { 333 t.Error(res.Error()) 334 } 335 if act := message.GetAllBytes(msg[0]); !reflect.DeepEqual(act, exp) { 336 t.Errorf("Wrong result: %s != %s", act, exp) 337 } 338 }