github.com/Jeffail/benthos/v3@v3.65.0/lib/stream/example_split_to_messages_test.go (about)

     1  package stream
     2  
     3  import (
     4  	"bytes"
     5  	"log"
     6  	"os"
     7  	"os/signal"
     8  	"syscall"
     9  	"time"
    10  
    11  	"github.com/Jeffail/benthos/v3/lib/input"
    12  	"github.com/Jeffail/benthos/v3/lib/message"
    13  	"github.com/Jeffail/benthos/v3/lib/output"
    14  	"github.com/Jeffail/benthos/v3/lib/types"
    15  )
    16  
    17  // SplitToMessages is a types.Processor implementation that reads a single
    18  // message containing line delimited payloads and splits the payloads into a
    19  // single message per line.
    20  type SplitToMessages struct{}
    21  
    22  // ProcessMessage splits messages of a batch by lines and sends them onwards as
    23  // an individual message per payload.
    24  func (p SplitToMessages) ProcessMessage(m types.Message) ([]types.Message, types.Response) {
    25  	var splitParts [][]byte
    26  	m.Iter(func(i int, b types.Part) error {
    27  		splitParts = append(splitParts, bytes.Split(b.Get(), []byte("\n"))...)
    28  		return nil
    29  	})
    30  
    31  	messages := make([]types.Message, len(splitParts))
    32  	for i, part := range splitParts {
    33  		messages[i] = message.New([][]byte{part})
    34  	}
    35  	return messages, nil
    36  }
    37  
    38  // CloseAsync shuts down the processor and stops processing requests.
    39  func (p SplitToMessages) CloseAsync() {
    40  	// Do nothing as our processor doesn't require resource cleanup.
    41  }
    42  
    43  // WaitForClose blocks until the processor has closed down.
    44  func (p SplitToMessages) WaitForClose(timeout time.Duration) error {
    45  	// Do nothing as our processor doesn't require resource cleanup.
    46  	return nil
    47  }
    48  
    49  // ExampleSplitToMessages demonstrates running a Kafka to Kafka stream where
    50  // each incoming message is parsed as a line delimited blob of payloads and the
    51  // payloads are sent on as a single message per payload.
    52  func Example_splitToMessages() {
    53  	conf := NewConfig()
    54  
    55  	conf.Input.Type = input.TypeKafka
    56  	conf.Input.Kafka.Addresses = []string{
    57  		"localhost:9092",
    58  	}
    59  	conf.Input.Kafka.Topic = "example_topic_one"
    60  
    61  	conf.Output.Type = output.TypeKafka
    62  	conf.Output.Kafka.Addresses = []string{
    63  		"localhost:9092",
    64  	}
    65  	conf.Output.Kafka.Topic = "example_topic_two"
    66  
    67  	s, err := New(conf, OptAddProcessors(func() (types.Processor, error) {
    68  		return SplitToMessages{}, nil
    69  	}))
    70  	if err != nil {
    71  		panic(err)
    72  	}
    73  
    74  	defer s.Stop(time.Second)
    75  
    76  	sigChan := make(chan os.Signal, 1)
    77  	signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
    78  
    79  	// Wait for termination signal
    80  	<-sigChan
    81  
    82  	log.Println("Received SIGTERM, the service is closing.")
    83  }