github.com/Jeffail/benthos/v3@v3.65.0/lib/stream/example_split_to_messages_test.go (about) 1 package stream 2 3 import ( 4 "bytes" 5 "log" 6 "os" 7 "os/signal" 8 "syscall" 9 "time" 10 11 "github.com/Jeffail/benthos/v3/lib/input" 12 "github.com/Jeffail/benthos/v3/lib/message" 13 "github.com/Jeffail/benthos/v3/lib/output" 14 "github.com/Jeffail/benthos/v3/lib/types" 15 ) 16 17 // SplitToMessages is a types.Processor implementation that reads a single 18 // message containing line delimited payloads and splits the payloads into a 19 // single message per line. 20 type SplitToMessages struct{} 21 22 // ProcessMessage splits messages of a batch by lines and sends them onwards as 23 // an individual message per payload. 24 func (p SplitToMessages) ProcessMessage(m types.Message) ([]types.Message, types.Response) { 25 var splitParts [][]byte 26 m.Iter(func(i int, b types.Part) error { 27 splitParts = append(splitParts, bytes.Split(b.Get(), []byte("\n"))...) 28 return nil 29 }) 30 31 messages := make([]types.Message, len(splitParts)) 32 for i, part := range splitParts { 33 messages[i] = message.New([][]byte{part}) 34 } 35 return messages, nil 36 } 37 38 // CloseAsync shuts down the processor and stops processing requests. 39 func (p SplitToMessages) CloseAsync() { 40 // Do nothing as our processor doesn't require resource cleanup. 41 } 42 43 // WaitForClose blocks until the processor has closed down. 44 func (p SplitToMessages) WaitForClose(timeout time.Duration) error { 45 // Do nothing as our processor doesn't require resource cleanup. 46 return nil 47 } 48 49 // ExampleSplitToMessages demonstrates running a Kafka to Kafka stream where 50 // each incoming message is parsed as a line delimited blob of payloads and the 51 // payloads are sent on as a single message per payload. 52 func Example_splitToMessages() { 53 conf := NewConfig() 54 55 conf.Input.Type = input.TypeKafka 56 conf.Input.Kafka.Addresses = []string{ 57 "localhost:9092", 58 } 59 conf.Input.Kafka.Topic = "example_topic_one" 60 61 conf.Output.Type = output.TypeKafka 62 conf.Output.Kafka.Addresses = []string{ 63 "localhost:9092", 64 } 65 conf.Output.Kafka.Topic = "example_topic_two" 66 67 s, err := New(conf, OptAddProcessors(func() (types.Processor, error) { 68 return SplitToMessages{}, nil 69 })) 70 if err != nil { 71 panic(err) 72 } 73 74 defer s.Stop(time.Second) 75 76 sigChan := make(chan os.Signal, 1) 77 signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) 78 79 // Wait for termination signal 80 <-sigChan 81 82 log.Println("Received SIGTERM, the service is closing.") 83 }