go.temporal.io/server@v1.23.0/common/membership/ringpop/monitor_test.go (about)

     1  // The MIT License
     2  //
     3  // Copyright (c) 2020 Temporal Technologies Inc.  All rights reserved.
     4  //
     5  // Copyright (c) 2020 Uber Technologies, Inc.
     6  //
     7  // Permission is hereby granted, free of charge, to any person obtaining a copy
     8  // of this software and associated documentation files (the "Software"), to deal
     9  // in the Software without restriction, including without limitation the rights
    10  // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    11  // copies of the Software, and to permit persons to whom the Software is
    12  // furnished to do so, subject to the following conditions:
    13  //
    14  // The above copyright notice and this permission notice shall be included in
    15  // all copies or substantial portions of the Software.
    16  //
    17  // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    18  // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    19  // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    20  // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    21  // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    22  // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    23  // THE SOFTWARE.
    24  
    25  package ringpop
    26  
    27  import (
    28  	"testing"
    29  	"time"
    30  
    31  	"golang.org/x/exp/maps"
    32  
    33  	"go.temporal.io/server/common/membership"
    34  	"go.temporal.io/server/common/primitives"
    35  
    36  	"github.com/stretchr/testify/require"
    37  	"github.com/stretchr/testify/suite"
    38  )
    39  
    40  type RpoSuite struct {
    41  	*require.Assertions // override suite.Suite.Assertions with require.Assertions; this means that s.NotNil(nil) will stop the test, not merely log an error
    42  	suite.Suite
    43  }
    44  
    45  func TestRpoSuite(t *testing.T) {
    46  	suite.Run(t, new(RpoSuite))
    47  }
    48  
    49  func (s *RpoSuite) SetupTest() {
    50  	s.Assertions = require.New(s.T()) // Have to define our overridden assertions in the test setup. If we did it earlier, s.T() will return nil
    51  }
    52  
    53  func (s *RpoSuite) TestMonitor() {
    54  	serviceName := primitives.HistoryService
    55  	testService := newTestCluster(s.T(), "rpm-test", 3, "127.0.0.1", "", serviceName, "127.0.0.1")
    56  	s.NotNil(testService, "Failed to create test service")
    57  
    58  	rpm := testService.rings[0]
    59  
    60  	time.Sleep(time.Second)
    61  
    62  	listenCh := make(chan *membership.ChangedEvent, 5)
    63  	r, err := rpm.GetResolver(serviceName)
    64  	s.NoError(err)
    65  	err = r.AddListener("test-listener", listenCh)
    66  	s.Nil(err, "AddListener failed")
    67  
    68  	host, err := r.Lookup("key")
    69  	s.Nil(err, "Ringpop monitor failed to find host for key")
    70  	s.NotNil(host, "Ringpop monitor returned a nil host")
    71  
    72  	// Force refresh now and drain the notification channel
    73  	resolver, _ := rpm.GetResolver(serviceName)
    74  	s.NoError(resolver.(*serviceResolver).refresh())
    75  	drainChannel(listenCh)
    76  
    77  	s.T().Log("Killing host 1")
    78  	testService.KillHost(testService.hostUUIDs[1])
    79  
    80  	timer := time.NewTimer(time.Minute)
    81  	select {
    82  	case e := <-listenCh:
    83  		timer.Stop()
    84  		s.Equal(1, len(e.HostsRemoved), "ringpop monitor event does not report the removed host")
    85  		s.Equal(testService.hostAddrs[1], e.HostsRemoved[0].GetAddress(), "ringpop monitor reported that a wrong host was removed")
    86  		s.Nil(e.HostsAdded, "Unexpected host reported to be added by ringpop monitor")
    87  	case <-timer.C:
    88  		s.Fail("Timed out waiting for failure to be detected by ringpop")
    89  	}
    90  
    91  	host, err = r.Lookup("key")
    92  	s.Nil(err, "Ringpop monitor failed to find host for key")
    93  	s.NotEqual(testService.hostAddrs[1], host.GetAddress(), "Ringpop monitor assigned key to dead host")
    94  
    95  	err = r.RemoveListener("test-listener")
    96  	s.Nil(err, "RemoveListener() failed")
    97  
    98  	rpm.Stop()
    99  	testService.Stop()
   100  }
   101  
   102  func (s *RpoSuite) TestCompareMembers() {
   103  	s.verifyMemberDiff([]string{}, []string{"a"}, []string{"+a"})
   104  	s.verifyMemberDiff([]string{}, []string{"a", "b"}, []string{"+a", "+b"})
   105  	s.verifyMemberDiff([]string{"a"}, []string{"a", "b"}, []string{"+b"})
   106  	s.verifyMemberDiff([]string{}, []string{}, nil)
   107  	s.verifyMemberDiff([]string{"a"}, []string{"a"}, nil)
   108  	s.verifyMemberDiff([]string{"a"}, []string{}, []string{"-a"})
   109  	s.verifyMemberDiff([]string{"a", "b"}, []string{}, []string{"-a", "-b"})
   110  	s.verifyMemberDiff([]string{"a", "b"}, []string{"a", "b"}, nil)
   111  	s.verifyMemberDiff([]string{"a", "b", "c"}, []string{"b", "d", "e"}, []string{"+d", "+e", "-a", "-c"})
   112  }
   113  
   114  func (s *RpoSuite) verifyMemberDiff(curr []string, new []string, expectedDiff []string) {
   115  	resolver := &serviceResolver{}
   116  	currMembers := make(map[string]struct{}, len(curr))
   117  	for _, m := range curr {
   118  		currMembers[m] = struct{}{}
   119  	}
   120  	resolver.membersMap = currMembers
   121  	newMembers, event := resolver.compareMembers(new)
   122  	s.ElementsMatch(new, maps.Keys(newMembers))
   123  	s.Equal(expectedDiff != nil, event != nil)
   124  	if event != nil {
   125  		var diff []string
   126  		for _, a := range event.HostsAdded {
   127  			diff = append(diff, "+"+a.GetAddress())
   128  		}
   129  		for _, a := range event.HostsRemoved {
   130  			diff = append(diff, "-"+a.GetAddress())
   131  		}
   132  		s.ElementsMatch(expectedDiff, diff)
   133  	}
   134  }
   135  
   136  func drainChannel[T any](ch <-chan T) {
   137  	for {
   138  		select {
   139  		case <-ch:
   140  		default:
   141  			return
   142  		}
   143  	}
   144  }