github.com/teng231/kafclient@v1.2.9/README.Md (about)

     1  # Kafclient
     2  > move by teng231/kafclient
     3  helper for create system using kafka to streaming and events driven base.
     4  
     5  [![Go](https://github.com/teng231/kafclient/actions/workflows/go.yml/badge.svg?branch=master)](https://github.com/teng231/kafclient/actions/workflows/go.yml)
     6  [![Go Reference](https://pkg.go.dev/badge/github.com/princjef/gomarkdoc.svg)](https://pkg.go.dev/github.com/teng231/kafclient)
     7  
     8  ### Install
     9  
    10  ``` bash
    11   go get github.com/teng231/kafclient
    12  ```
    13  have 3 env variables for config:
    14  * KAFKA_NUM_PARTITION: config number of partition( default: 3)
    15  * KAFKA_REPLICATION_FACTOR: config replication factor.(default:  -1)
    16  * KAFKA_VERSION: for set kafka version for sarama config
    17  
    18  you can overide 3 variables for cluster configs
    19  
    20  ### Usage
    21  
    22  Library have 2 modules publisher and consumer, or use can use both.
    23  For example to setup:
    24  
    25  * Both setup
    26  
    27  ``` go
    28  func NewKafClient() error {
    29  	ps := &kafclient.Client{}
    30  	brokers := strings.Split("localhost:9002,localhost:9003", ",")
    31  	ps.InitPublisher(brokers...)
    32  	err := ps.InitConsumerGroup(cf.PubsubSubscription, brokers...)
    33  	if err != nil {
    34  		return err
    35  	}
    36  	return nil
    37  }
    38  
    39  ```
    40  
    41  
    42  * Publisher setup
    43  
    44  
    45  ``` go
    46  
    47  func NewKafClient() error {
    48  	ps := &kafclient.Client{}
    49  	brokers := strings.Split("localhost:9002,localhost:9003", ",")
    50  	ps.InitPublisher(brokers...)
    51  	if err != nil {
    52  		return err
    53  	}
    54  	return nil
    55  }
    56  
    57  
    58  // using for publish event
    59  func SendMessage(msg string) {
    60      sms := map[string]interface{}{
    61  		Body: "hello everyone",
    62          App:  "company.vn",
    63  		To:   []string{"0374140511"},
    64  	}
    65      if err := ps.Publish("topic_send_example", sms); err != nil {
    66  		log.Print(err)
    67  	}
    68  }
    69  
    70  
    71  ```
    72  
    73  
    74  * Consumer setup
    75  ``` go
    76  
    77  func main() {
    78  
    79      // if this task running will block main thread
    80      onStreamRunning(...)
    81  
    82  }
    83  
    84  
    85  func NewKafClient() error {
    86  	ps := &kafclient.Client{}
    87  	brokers := strings.Split("localhost:9002,localhost:9003", ",")
    88  	err := ps.InitConsumerGroup(cf.KafClientSubscription, brokers...)
    89  	if err != nil {
    90  		return err
    91  	}
    92  	return nil
    93  }
    94  
    95  // using for publish event
    96  func SendMessage() error {
    97      sms := map[string]interface{}{
    98  		Body: "hello everyone",
    99          App:  "company.vn",
   100  		To:   []string{"0374140511"},
   101  	}
   102      if err := ps.Publish("topic_send_example", sms); err != nil {
   103  		log.Print(err)
   104          return err
   105  	}
   106      return nil
   107  }
   108  
   109  
   110  func SendMessageExtraConfig() error {
   111      sms := map[string]interface{}{
   112  		Body: "hello everyone",
   113          App:  "company.vn",
   114  		To:   []string{"0374140511"},
   115  	}
   116      if err := ps.PublishWithConfig(Topic{Name:"topic_send_example", AutoCommit: false}, &SenderConfig{
   117  		Header: map[string]string{"sender_by": "containerX"},
   118  		Metadata: "any_if_you_want",
   119  	}, sms); err != nil {
   120  		log.Print(err)
   121          return err
   122  	}
   123      return nil
   124  }
   125  
   126  // send message help sync task
   127  func SendMessageToSinglePartition() error {
   128      sms := map[string]interface{}{
   129  		Body: "hello everyone",
   130          App:  "company.vn",
   131  		To:   []string{"0374140511"},
   132  	}
   133      if err := ps.PublishWithConfig(Topic{Name:"topic_send_example", AutoCommit: false, Partition: ToPInt32(1)}, &SenderConfig{
   134  		Header: map[string]string{"sender_by": "containerX"},
   135  		Metadata: "any_if_you_want",
   136  	}, sms); err != nil {
   137  		log.Print(err)
   138          return err
   139  	}
   140      return nil
   141  }
   142  
   143  
   144  func onListenUserInsert(payload []byte, h *Account, fnCommit func()) {
   145  	defer fnCommit()
   146  	user := &User{}
   147      onCreateUser(user)
   148      ...
   149  }
   150  
   151  
   152  type MsgHandler struct {
   153  	f     func([]byte, *Account, func())
   154  	topic *KafClient.Topic
   155  }
   156  
   157  var (
   158  	backend_user_insert_user = "backend_user_insert_user"
   159      numberOfConcurrents = 1
   160  	subsList                   = map[string]*MsgHandler{
   161  		backend_user_insert_user: {onListenUserInsert, &KafClient.Topic{
   162  			Name:       backend_user_insert_user,
   163  			AutoCommit: false,
   164  		}},
   165  	}
   166  )
   167  
   168  func onStreamRunning(ps *KafClient.KafClient, h *Account) {
   169  	log.Print("Stream start")
   170  	msgBuf := make(chan KafClient.Message, 1000)
   171  
   172  	numberOfConcurrents := 1
   173  	topics := make([]*KafClient.Topic, 0, 100)
   174  
   175  	for _, msghandler := range kafkaTopics {
   176  		topics = append(topics, msghandler.topic)
   177  	}
   178  
   179  	go func() {
   180  		for {
   181  			psMessage := <-msgBuf
   182  			msghandler := subsList[psMessage.Topic]
   183  			if msghandler.f != nil {
   184  				msghandler.f(psMessage.Body, h, psMessage.Commit)
   185  			}
   186  		}
   187  	}()
   188  	err := ps.OnAsyncSubscribe(topics, numberOfConcurrents, msgBuf)
   189  	if err != nil {
   190  		log.Print(err)
   191  	}
   192  }
   193  ```