github.com/Jeffail/benthos/v3@v3.65.0/lib/processor/parallel_test.go (about)

     1  package processor
     2  
     3  import (
     4  	"io"
     5  	"net/http"
     6  	"net/http/httptest"
     7  	"sync"
     8  	"sync/atomic"
     9  	"testing"
    10  	"time"
    11  
    12  	"github.com/Jeffail/benthos/v3/lib/log"
    13  	"github.com/Jeffail/benthos/v3/lib/message"
    14  	"github.com/Jeffail/benthos/v3/lib/metrics"
    15  )
    16  
    17  func TestParallelBasic(t *testing.T) {
    18  	wg := sync.WaitGroup{}
    19  	wg.Add(5)
    20  	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
    21  		wg.Done()
    22  		wg.Wait()
    23  		w.Write([]byte("foobar"))
    24  	}))
    25  	defer ts.Close()
    26  
    27  	httpConf := NewConfig()
    28  	httpConf.Type = TypeHTTP
    29  	httpConf.HTTP.Client.URL = ts.URL + "/testpost"
    30  	conf := NewConfig()
    31  	conf.Parallel.Processors = []Config{httpConf}
    32  
    33  	h, err := NewParallel(conf, nil, log.Noop(), metrics.Noop())
    34  	if err != nil {
    35  		t.Fatal(err)
    36  	}
    37  
    38  	msgs, res := h.ProcessMessage(message.New([][]byte{
    39  		[]byte("foo"),
    40  		[]byte("bar"),
    41  		[]byte("baz"),
    42  		[]byte("qux"),
    43  		[]byte("quz"),
    44  	}))
    45  	if res != nil {
    46  		t.Error(res.Error())
    47  	} else if expC, actC := 5, msgs[0].Len(); actC != expC {
    48  		t.Errorf("Wrong result count: %v != %v", actC, expC)
    49  	} else if exp, act := "foobar", string(message.GetAllBytes(msgs[0])[0]); act != exp {
    50  		t.Errorf("Wrong result: %v != %v", act, exp)
    51  	}
    52  }
    53  
    54  func TestParallelError(t *testing.T) {
    55  	wg := sync.WaitGroup{}
    56  	wg.Add(5)
    57  	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
    58  		wg.Done()
    59  		wg.Wait()
    60  		reqBytes, err := io.ReadAll(r.Body)
    61  		if err != nil {
    62  			t.Fatal(err)
    63  		}
    64  		if string(reqBytes) == "baz" {
    65  			http.Error(w, "test error", http.StatusForbidden)
    66  			return
    67  		}
    68  		w.Write([]byte("foobar"))
    69  	}))
    70  	defer ts.Close()
    71  
    72  	httpConf := NewConfig()
    73  	httpConf.Type = TypeHTTP
    74  	httpConf.HTTP.Client.URL = ts.URL + "/testpost"
    75  	httpConf.HTTP.Client.NumRetries = 0
    76  
    77  	conf := NewConfig()
    78  	conf.Parallel.Processors = []Config{httpConf}
    79  
    80  	h, err := NewParallel(conf, nil, log.Noop(), metrics.Noop())
    81  	if err != nil {
    82  		t.Fatal(err)
    83  	}
    84  
    85  	msgs, res := h.ProcessMessage(message.New([][]byte{
    86  		[]byte("foo"),
    87  		[]byte("bar"),
    88  		[]byte("baz"),
    89  		[]byte("qux"),
    90  		[]byte("quz"),
    91  	}))
    92  	if res != nil {
    93  		t.Error(res.Error())
    94  	}
    95  	if expC, actC := 5, msgs[0].Len(); actC != expC {
    96  		t.Fatalf("Wrong result count: %v != %v", actC, expC)
    97  	}
    98  	if exp, act := "baz", string(msgs[0].Get(2).Get()); act != exp {
    99  		t.Errorf("Wrong result: %v != %v", act, exp)
   100  	}
   101  	if !HasFailed(msgs[0].Get(2)) {
   102  		t.Error("Expected failed flag")
   103  	}
   104  	for _, i := range []int{0, 1, 3, 4} {
   105  		if exp, act := "foobar", string(msgs[0].Get(i).Get()); act != exp {
   106  			t.Errorf("Wrong result: %v != %v", act, exp)
   107  		}
   108  		if HasFailed(msgs[0].Get(i)) {
   109  			t.Error("Did not expect failed flag")
   110  		}
   111  	}
   112  }
   113  
   114  func TestParallelUnack(t *testing.T) {
   115  	batchConf := NewConfig()
   116  	batchConf.Type = TypeBatch
   117  	batchConf.Batch.Count = 1000
   118  
   119  	conf := NewConfig()
   120  	conf.Parallel.Processors = []Config{batchConf}
   121  
   122  	h, err := NewParallel(conf, nil, log.Noop(), metrics.Noop())
   123  	if err != nil {
   124  		t.Fatal(err)
   125  	}
   126  
   127  	msgs, res := h.ProcessMessage(message.New([][]byte{
   128  		[]byte("foo"),
   129  		[]byte("bar"),
   130  		[]byte("baz"),
   131  		[]byte("qux"),
   132  		[]byte("quz"),
   133  	}))
   134  	if res == nil {
   135  		t.Error("Expected non-nil response")
   136  	}
   137  	if len(msgs) != 0 {
   138  		t.Error("Expected empty msgs response")
   139  	}
   140  	if !res.SkipAck() {
   141  		t.Errorf("Expected unack response, received: %v", res)
   142  	}
   143  }
   144  
   145  func TestParallelCapped(t *testing.T) {
   146  	var reqs int64
   147  	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
   148  		if req := atomic.AddInt64(&reqs, 1); req > 5 {
   149  			t.Errorf("Beyond parallelism cap: %v", req)
   150  		}
   151  		<-time.After(time.Millisecond * 10)
   152  		w.Write([]byte("foobar"))
   153  		atomic.AddInt64(&reqs, -1)
   154  	}))
   155  	defer ts.Close()
   156  
   157  	httpConf := NewConfig()
   158  	httpConf.Type = TypeHTTP
   159  	httpConf.HTTP.Client.URL = ts.URL + "/testpost"
   160  
   161  	conf := NewConfig()
   162  	conf.Parallel.Processors = []Config{httpConf}
   163  	conf.Parallel.Cap = 5
   164  
   165  	h, err := NewParallel(conf, nil, log.Noop(), metrics.Noop())
   166  	if err != nil {
   167  		t.Fatal(err)
   168  	}
   169  
   170  	msgs, res := h.ProcessMessage(message.New([][]byte{
   171  		[]byte("foo"),
   172  		[]byte("bar"),
   173  		[]byte("baz"),
   174  		[]byte("qux"),
   175  		[]byte("quz"),
   176  		[]byte("foo2"),
   177  		[]byte("bar2"),
   178  		[]byte("baz2"),
   179  		[]byte("qux2"),
   180  		[]byte("quz2"),
   181  	}))
   182  	if res != nil {
   183  		t.Error(res.Error())
   184  	} else if expC, actC := 10, msgs[0].Len(); actC != expC {
   185  		t.Errorf("Wrong result count: %v != %v", actC, expC)
   186  	} else if exp, act := "foobar", string(message.GetAllBytes(msgs[0])[0]); act != exp {
   187  		t.Errorf("Wrong result: %v != %v", act, exp)
   188  	}
   189  }