github.com/hechain20/hechain@v0.0.0-20220316014945-b544036ba106/integration/kafka/health_test.go (about)

     1  /*
     2  Copyright hechain All Rights Reserved.
     3  
     4  SPDX-License-Identifier: Apache-2.0
     5  */
     6  
     7  package kafka
     8  
     9  import (
    10  	"encoding/json"
    11  	"fmt"
    12  	"io/ioutil"
    13  	"net/http"
    14  	"os"
    15  	"syscall"
    16  
    17  	docker "github.com/fsouza/go-dockerclient"
    18  	"github.com/hechain20/hechain/integration/nwo"
    19  	"github.com/hyperledger/fabric-lib-go/healthz"
    20  	. "github.com/onsi/ginkgo"
    21  	. "github.com/onsi/gomega"
    22  	"github.com/tedsuo/ifrit"
    23  )
    24  
    25  var _ = Describe("Kafka Health", func() {
    26  	var (
    27  		testDir string
    28  		client  *docker.Client
    29  		network *nwo.Network
    30  		process ifrit.Process
    31  	)
    32  
    33  	BeforeEach(func() {
    34  		var err error
    35  		testDir, err = ioutil.TempDir("", "kafka-health")
    36  		Expect(err).NotTo(HaveOccurred())
    37  
    38  		client, err = docker.NewClientFromEnv()
    39  		Expect(err).NotTo(HaveOccurred())
    40  
    41  		config := nwo.BasicKafka()
    42  		config.Consensus.Brokers = 3
    43  
    44  		network = nwo.New(config, testDir, client, StartPort(), components)
    45  		network.GenerateConfigTree()
    46  		network.Bootstrap()
    47  	})
    48  
    49  	AfterEach(func() {
    50  		if process != nil {
    51  			process.Signal(syscall.SIGTERM)
    52  			Eventually(process.Wait(), network.EventuallyTimeout).Should(Receive())
    53  		}
    54  		if network != nil {
    55  			network.Cleanup()
    56  		}
    57  		os.RemoveAll(testDir)
    58  	})
    59  
    60  	Describe("Kafka health checks", func() {
    61  		var (
    62  			oProcess ifrit.Process
    63  			zProcess ifrit.Process
    64  			kProcess []ifrit.Process
    65  
    66  			authClient *http.Client
    67  			healthURL  string
    68  		)
    69  
    70  		BeforeEach(func() {
    71  			// Start Zookeeper
    72  			zookeepers := []string{}
    73  			zk := network.ZooKeeperRunner(0)
    74  			zookeepers = append(zookeepers, fmt.Sprintf("%s:2181", zk.Name))
    75  			zProcess = ifrit.Invoke(zk)
    76  			Eventually(zProcess.Ready(), network.EventuallyTimeout).Should(BeClosed())
    77  
    78  			// Start Kafka Brokers
    79  			for i := 0; i < network.Consensus.Brokers; i++ {
    80  				kafkaRunner := network.BrokerRunner(i, zookeepers)
    81  				kp := ifrit.Invoke(kafkaRunner)
    82  				Eventually(kp.Ready(), network.EventuallyTimeout).Should(BeClosed())
    83  				kProcess = append(kProcess, kp)
    84  			}
    85  
    86  			// Start Orderer
    87  			ordererRunner := network.OrdererGroupRunner()
    88  			oProcess = ifrit.Invoke(ordererRunner)
    89  			Eventually(oProcess.Ready(), network.EventuallyTimeout).Should(BeClosed())
    90  
    91  			orderer := network.Orderers[0]
    92  			authClient, _ = nwo.OrdererOperationalClients(network, orderer)
    93  			healthURL = fmt.Sprintf("https://127.0.0.1:%d/healthz", network.OrdererPort(orderer, nwo.OperationsPort))
    94  		})
    95  
    96  		AfterEach(func() {
    97  			if zProcess != nil {
    98  				zProcess.Signal(syscall.SIGTERM)
    99  				Eventually(zProcess.Wait(), network.EventuallyTimeout).Should(Receive())
   100  			}
   101  			if oProcess != nil {
   102  				oProcess.Signal(syscall.SIGTERM)
   103  				Eventually(oProcess.Wait(), network.EventuallyTimeout).Should(Receive())
   104  			}
   105  			for _, k := range kProcess {
   106  				if k != nil {
   107  					k.Signal(syscall.SIGTERM)
   108  					Eventually(k.Wait(), network.EventuallyTimeout).Should(Receive())
   109  				}
   110  			}
   111  		})
   112  
   113  		Context("when running health checks on orderer the kafka health check", func() {
   114  			It("returns appropriate response code", func() {
   115  				By("returning 200 when all brokers are online", func() {
   116  					statusCode, status := doHealthCheck(authClient, healthURL)
   117  					Expect(statusCode).To(Equal(http.StatusOK))
   118  					Expect(status.Status).To(Equal("OK"))
   119  				})
   120  
   121  				By("returning a 200 when one of the three brokers goes offline", func() {
   122  					k := kProcess[1]
   123  					k.Signal(syscall.SIGTERM)
   124  					Eventually(k.Wait(), network.EventuallyTimeout).Should(Receive())
   125  
   126  					var statusCode int
   127  					var status *healthz.HealthStatus
   128  					Consistently(func() int {
   129  						statusCode, status = doHealthCheck(authClient, healthURL)
   130  						return statusCode
   131  					}).Should(Equal(http.StatusOK))
   132  					Expect(status.Status).To(Equal("OK"))
   133  				})
   134  
   135  				By("returning a 503 when two of the three brokers go offline", func() {
   136  					k := kProcess[0]
   137  					k.Signal(syscall.SIGTERM)
   138  					Eventually(k.Wait(), network.EventuallyTimeout).Should(Receive())
   139  
   140  					var statusCode int
   141  					var status *healthz.HealthStatus
   142  					Eventually(func() int {
   143  						statusCode, status = doHealthCheck(authClient, healthURL)
   144  						return statusCode
   145  					}, network.EventuallyTimeout).Should(Equal(http.StatusServiceUnavailable))
   146  					Expect(status.Status).To(Equal("Service Unavailable"))
   147  					Expect(status.FailedChecks[0].Component).To(Equal("systemchannel/0"))
   148  					Expect(status.FailedChecks[0].Reason).To(MatchRegexp(`\[replica ids: \[\d \d \d\]\]: kafka server: Messages are rejected since there are fewer in-sync replicas than required\.`))
   149  				})
   150  			})
   151  		})
   152  	})
   153  })
   154  
   155  func doHealthCheck(client *http.Client, url string) (int, *healthz.HealthStatus) {
   156  	resp, err := client.Get(url)
   157  	Expect(err).NotTo(HaveOccurred())
   158  
   159  	bodyBytes, err := ioutil.ReadAll(resp.Body)
   160  	Expect(err).NotTo(HaveOccurred())
   161  	resp.Body.Close()
   162  
   163  	// This occurs when a request to the health check server times out, no body is
   164  	// returned when a timeout occurs
   165  	if len(bodyBytes) == 0 {
   166  		return resp.StatusCode, nil
   167  	}
   168  
   169  	healthStatus := &healthz.HealthStatus{}
   170  	err = json.Unmarshal(bodyBytes, &healthStatus)
   171  	Expect(err).NotTo(HaveOccurred())
   172  
   173  	return resp.StatusCode, healthStatus
   174  }