gitee.com/woood2/luca@v1.0.4/cmd/consumer/main.go

package main

import (
	"context"
	"gitee.com/woood2/luca/cmd/consumer/internal/alarm"
	"gitee.com/woood2/luca/cmd/consumer/internal/handler"
	_ "gitee.com/woood2/luca/cmd/consumer/internal/handler"
	"gitee.com/woood2/luca/cmd/consumer/internal/subscriber"
	lucaSDK "gitee.com/woood2/luca/cmd/micro/pkg"
	"gitee.com/woood2/luca/internal/conf"
	"gitee.com/woood2/luca/internal/discovery"
	myLog "gitee.com/woood2/luca/internal/log"
	"gitee.com/woood2/luca/internal/producer"
	"gitee.com/woood2/luca/internal/status"
	"gitee.com/woood2/luca/internal/trace"
	"github.com/Shopify/sarama"
	consulsd "github.com/go-kit/kit/sd/consul"
	"go.uber.org/zap"
	"log"
	"os"
	"os/signal"
	"strings"
	"sync"
	"syscall"
	"time"
)

const entrance = "consumer"

func main() {
	//config
	attr := conf.Load("application.yml", "configs/application.yml")
	//consul
	client, consulClient := discovery.Client(attr.Consul.Host, attr.Consul.Port)
	conf.MergeConsul(attr, consulClient)
	//zap logger
	logger := myLog.Build(attr.Env, attr.Project, entrance, attr.Host, attr.ConsoleLog)
	defer logger.Sync()
	//zipkin
	trace.Open(attr.Zipkin)
	defer trace.Close()
	//sdk
	setupSDK(client, logger)
	//Pprof & prometheus & hystrix
	go status.Pprof(attr.Pprof, attr.Env, attr.Consumer.PprofAddr)
	go status.Prometheus(attr.Consumer.MetricsAddr)
	go status.Hystrix(attr.Consumer.HystrixPort)
	//alarm
	alarm.WatchFunc(func(queue chan *alarm.Alarm) {
		for {
			a := <-queue
			log.Printf("alarm watch: groupID=%s, topic=%s, partition=%d, retry=%d, value=%s\n",
				a.GroupID, a.Msg.Topic, a.Msg.Partition, a.Retry, string(a.Msg.Value))
		}
	})
	//sarama
	time.Sleep(500 * time.Millisecond)
	log.Println("Starting a new Sarama consumer")
	wg := &sync.WaitGroup{}
	ctx, cancel := context.WithCancel(context.Background())

	handler.RegisterHandlers(attr.Project, entrance, logger)
	subs := subscriber.List()
	for _, sub := range subs {
		wg.Add(1)
		go func(s *subscriber.Subscriber) {
			defer wg.Done()
			log.Printf("enter consume loop, topicName=%s, groupID=%s\n", s.TopicName, s.GroupID)
			config := saramaConfig(attr.Kafka, s)
			client, err := sarama.NewConsumerGroup(attr.Kafka.Brokers, s.GroupID, config)
			if err != nil {
				log.Panicf("Error creating consumer group client: %v", err)
			}
			gen := 0
			delay := 3 * time.Second
			for {
				log.Printf("start consume session, topicName=%s, groupID=%s\n", s.TopicName, s.GroupID)

				if err := client.Consume(ctx, strings.Split(s.TopicName, ","), s.ConsumerGroupHandler); err != nil {
					if gen == 0 {
						log.Panicf("Error from consumer: %v\n", err)
					} else {
						log.Printf("Error from consumer: %v, retry again in 3 seconds, gen=%d\n", err, gen)
						time.Sleep(delay)
					}
				}
				log.Printf("end consume session, topicName=%s, groupID=%s\n", s.TopicName, s.GroupID)
				// check if context was cancelled, signaling that the consumer should stop
				if ctx.Err() != nil {
					if err = client.Close(); err != nil {
						log.Panicf("Error closing client: %v\n", err)
					}
					log.Printf("quit consume loop, topicName=%s, groupID=%s\n", s.TopicName, s.GroupID)
					return
				}
				gen++
			}
		}(sub)
	}
	sigterm := make(chan os.Signal, 1)
	signal.Notify(sigterm, syscall.SIGINT, syscall.SIGTERM)
	select {
	case <-ctx.Done():
		log.Println("terminating: context cancelled")
	case <-sigterm:
		log.Println("terminating: via signal")
	}
	cancel()
	stopped := make(chan struct{})
	go func() {
		wg.Wait()
		close(stopped)
	}()
	t := time.NewTimer(5 * time.Second)
	select {
	case <-t.C:
		log.Println("Server exiting in 5 sec...")
	case <-stopped:
		log.Println("Server exiting...")
	}
}

func saramaConfig(c *conf.Kafka, g *subscriber.Subscriber) *sarama.Config {
	verbose := false
	if verbose {
		sarama.Logger = log.New(os.Stdout, "[sarama] ", log.LstdFlags)
	}
	version, err := sarama.ParseKafkaVersion(c.Version)
	if err != nil {
		log.Panicf("Error parsing Kafka version: %v\n", err)
	}
	config := sarama.NewConfig()
	config.Version = version
	if g.OffsetOldest {
		config.Consumer.Offsets.Initial = sarama.OffsetOldest
	}
	if c.EnableSASL {
		config.Net.SASL.Enable = c.EnableSASL
		config.Net.SASL.User = c.User
		config.Net.SASL.Password = c.Password
		config.Net.SASL.Handshake = true
		if c.Algorithm == "sha512" {
			config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient { return &producer.XDGSCRAMClient{HashGeneratorFcn: producer.SHA512} }
			config.Net.SASL.Mechanism = sarama.SASLTypeSCRAMSHA512
		} else if c.Algorithm == "sha256" {
			config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient { return &producer.XDGSCRAMClient{HashGeneratorFcn: producer.SHA256} }
			config.Net.SASL.Mechanism = sarama.SASLTypeSCRAMSHA256
		} else {
			log.Panicf("invalid SHA algorithm \"%s\": can be either \"sha256\" or \"sha512\"\n", c.Algorithm)
		}
	}
	return config
}

func setupSDK(client consulsd.Client, zapLogger *zap.Logger) {
	lucaClient := lucaSDK.NewSD(client, "weibo2", "todo", zapLogger, trace.ClientTrace())
	lucaSDK.SetGlobal(lucaClient)
	//More...
}