github.com/Jeffail/benthos/v3@v3.65.0/lib/processor/parallel_test.go (about) 1 package processor 2 3 import ( 4 "io" 5 "net/http" 6 "net/http/httptest" 7 "sync" 8 "sync/atomic" 9 "testing" 10 "time" 11 12 "github.com/Jeffail/benthos/v3/lib/log" 13 "github.com/Jeffail/benthos/v3/lib/message" 14 "github.com/Jeffail/benthos/v3/lib/metrics" 15 ) 16 17 func TestParallelBasic(t *testing.T) { 18 wg := sync.WaitGroup{} 19 wg.Add(5) 20 ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 21 wg.Done() 22 wg.Wait() 23 w.Write([]byte("foobar")) 24 })) 25 defer ts.Close() 26 27 httpConf := NewConfig() 28 httpConf.Type = TypeHTTP 29 httpConf.HTTP.Client.URL = ts.URL + "/testpost" 30 conf := NewConfig() 31 conf.Parallel.Processors = []Config{httpConf} 32 33 h, err := NewParallel(conf, nil, log.Noop(), metrics.Noop()) 34 if err != nil { 35 t.Fatal(err) 36 } 37 38 msgs, res := h.ProcessMessage(message.New([][]byte{ 39 []byte("foo"), 40 []byte("bar"), 41 []byte("baz"), 42 []byte("qux"), 43 []byte("quz"), 44 })) 45 if res != nil { 46 t.Error(res.Error()) 47 } else if expC, actC := 5, msgs[0].Len(); actC != expC { 48 t.Errorf("Wrong result count: %v != %v", actC, expC) 49 } else if exp, act := "foobar", string(message.GetAllBytes(msgs[0])[0]); act != exp { 50 t.Errorf("Wrong result: %v != %v", act, exp) 51 } 52 } 53 54 func TestParallelError(t *testing.T) { 55 wg := sync.WaitGroup{} 56 wg.Add(5) 57 ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 58 wg.Done() 59 wg.Wait() 60 reqBytes, err := io.ReadAll(r.Body) 61 if err != nil { 62 t.Fatal(err) 63 } 64 if string(reqBytes) == "baz" { 65 http.Error(w, "test error", http.StatusForbidden) 66 return 67 } 68 w.Write([]byte("foobar")) 69 })) 70 defer ts.Close() 71 72 httpConf := NewConfig() 73 httpConf.Type = TypeHTTP 74 httpConf.HTTP.Client.URL = ts.URL + "/testpost" 75 httpConf.HTTP.Client.NumRetries = 0 76 77 conf := NewConfig() 78 conf.Parallel.Processors = []Config{httpConf} 79 80 h, err := NewParallel(conf, nil, log.Noop(), metrics.Noop()) 81 if err != nil { 82 t.Fatal(err) 83 } 84 85 msgs, res := h.ProcessMessage(message.New([][]byte{ 86 []byte("foo"), 87 []byte("bar"), 88 []byte("baz"), 89 []byte("qux"), 90 []byte("quz"), 91 })) 92 if res != nil { 93 t.Error(res.Error()) 94 } 95 if expC, actC := 5, msgs[0].Len(); actC != expC { 96 t.Fatalf("Wrong result count: %v != %v", actC, expC) 97 } 98 if exp, act := "baz", string(msgs[0].Get(2).Get()); act != exp { 99 t.Errorf("Wrong result: %v != %v", act, exp) 100 } 101 if !HasFailed(msgs[0].Get(2)) { 102 t.Error("Expected failed flag") 103 } 104 for _, i := range []int{0, 1, 3, 4} { 105 if exp, act := "foobar", string(msgs[0].Get(i).Get()); act != exp { 106 t.Errorf("Wrong result: %v != %v", act, exp) 107 } 108 if HasFailed(msgs[0].Get(i)) { 109 t.Error("Did not expect failed flag") 110 } 111 } 112 } 113 114 func TestParallelUnack(t *testing.T) { 115 batchConf := NewConfig() 116 batchConf.Type = TypeBatch 117 batchConf.Batch.Count = 1000 118 119 conf := NewConfig() 120 conf.Parallel.Processors = []Config{batchConf} 121 122 h, err := NewParallel(conf, nil, log.Noop(), metrics.Noop()) 123 if err != nil { 124 t.Fatal(err) 125 } 126 127 msgs, res := h.ProcessMessage(message.New([][]byte{ 128 []byte("foo"), 129 []byte("bar"), 130 []byte("baz"), 131 []byte("qux"), 132 []byte("quz"), 133 })) 134 if res == nil { 135 t.Error("Expected non-nil response") 136 } 137 if len(msgs) != 0 { 138 t.Error("Expected empty msgs response") 139 } 140 if !res.SkipAck() { 141 t.Errorf("Expected unack response, received: %v", res) 142 } 143 } 144 145 func TestParallelCapped(t *testing.T) { 146 var reqs int64 147 ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 148 if req := atomic.AddInt64(&reqs, 1); req > 5 { 149 t.Errorf("Beyond parallelism cap: %v", req) 150 } 151 <-time.After(time.Millisecond * 10) 152 w.Write([]byte("foobar")) 153 atomic.AddInt64(&reqs, -1) 154 })) 155 defer ts.Close() 156 157 httpConf := NewConfig() 158 httpConf.Type = TypeHTTP 159 httpConf.HTTP.Client.URL = ts.URL + "/testpost" 160 161 conf := NewConfig() 162 conf.Parallel.Processors = []Config{httpConf} 163 conf.Parallel.Cap = 5 164 165 h, err := NewParallel(conf, nil, log.Noop(), metrics.Noop()) 166 if err != nil { 167 t.Fatal(err) 168 } 169 170 msgs, res := h.ProcessMessage(message.New([][]byte{ 171 []byte("foo"), 172 []byte("bar"), 173 []byte("baz"), 174 []byte("qux"), 175 []byte("quz"), 176 []byte("foo2"), 177 []byte("bar2"), 178 []byte("baz2"), 179 []byte("qux2"), 180 []byte("quz2"), 181 })) 182 if res != nil { 183 t.Error(res.Error()) 184 } else if expC, actC := 10, msgs[0].Len(); actC != expC { 185 t.Errorf("Wrong result count: %v != %v", actC, expC) 186 } else if exp, act := "foobar", string(message.GetAllBytes(msgs[0])[0]); act != exp { 187 t.Errorf("Wrong result: %v != %v", act, exp) 188 } 189 }