github.com/fibonacci-chain/fbc@v0.0.0-20231124064014-c7636198c1e9/app/logevents/kafka.go (about)

     1  package logevents
     2  
     3  import (
     4  	"context"
     5  	"encoding/json"
     6  	"github.com/segmentio/kafka-go"
     7  	"strings"
     8  	"sync"
     9  	"time"
    10  )
    11  
    12  const (
    13  	FBCLogTopic      = "fbclog"
    14  	LogConsumerGroup = "fbclog-consumer-group"
    15  
    16  	HeartbeatTopic = "fbclog-subscriber-heartbeat"
    17  
    18  	HeartbeatInterval = 5 * time.Second
    19  	ExpiredInterval   = 6 * HeartbeatInterval
    20  )
    21  
    22  type logClient struct {
    23  	wt      string
    24  	rt      string
    25  	groupID string
    26  	*kafka.Writer
    27  	*kafka.Reader
    28  }
    29  
    30  func newLogClient(kafkaAddrs string, wt, rt string, groupID string) *logClient {
    31  	addrs := strings.Split(kafkaAddrs, ",")
    32  	return &logClient{
    33  		wt:      wt,
    34  		rt:      rt,
    35  		groupID: groupID,
    36  		Writer: kafka.NewWriter(kafka.WriterConfig{
    37  			Brokers:  addrs,
    38  			Topic:    wt,
    39  			Balancer: &kafka.LeastBytes{},
    40  		}),
    41  		Reader: getKafkaReader(kafkaAddrs, rt, groupID),
    42  	}
    43  }
    44  
    45  type KafkaMsg struct {
    46  	Topic string `json:"topic"`
    47  	Data  string `json:"data"`
    48  }
    49  
    50  var KafkaMsgPool = sync.Pool{
    51  	New: func() interface{} {
    52  		return &KafkaMsg{}
    53  	},
    54  }
    55  
    56  func (kc *logClient) recv() (string, *KafkaMsg, error) {
    57  	const empty = ""
    58  	rawMsg, err := kc.ReadMessage(context.Background())
    59  	if err != nil {
    60  		return empty, nil, err
    61  	}
    62  
    63  	var msg KafkaMsg
    64  	err = json.Unmarshal(rawMsg.Value, &msg)
    65  	if err != nil {
    66  		return empty, nil, err
    67  	}
    68  
    69  	return string(rawMsg.Key), &msg, err
    70  }
    71  
    72  func (kc *logClient) send(key string, rawMsg *KafkaMsg) error {
    73  	rawMsg.Topic = kc.wt
    74  
    75  	msg, err := json.Marshal(*rawMsg)
    76  	if err != nil {
    77  		return err
    78  	}
    79  
    80  	// Automatic retries and reconnections on errors.
    81  	return kc.WriteMessages(context.Background(),
    82  		kafka.Message{
    83  			Key:   []byte(key),
    84  			Value: msg,
    85  		},
    86  	)
    87  }
    88  
    89  func getKafkaReader(kafkaURL, topic, groupID string) *kafka.Reader {
    90  	brokers := strings.Split(kafkaURL, ",")
    91  	return kafka.NewReader(kafka.ReaderConfig{
    92  		Brokers: brokers,
    93  		GroupID: groupID,
    94  		Topic:   topic,
    95  		//MinBytes: 10e3, // 10KB
    96  		MaxBytes: 10e6, // 10MB
    97  	})
    98  }