github.com/m3db/m3@v1.5.0/src/cluster/services/leader/client_test.go (about) 1 // Copyright (c) 2017 Uber Technologies, Inc. 2 // 3 // Permission is hereby granted, free of charge, to any person obtaining a copy 4 // of this software and associated documentation files (the "Software"), to deal 5 // in the Software without restriction, including without limitation the rights 6 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 // copies of the Software, and to permit persons to whom the Software is 8 // furnished to do so, subject to the following conditions: 9 // 10 // The above copyright notice and this permission notice shall be included in 11 // all copies or substantial portions of the Software. 12 // 13 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 // THE SOFTWARE. 20 21 package leader 22 23 import ( 24 "fmt" 25 "reflect" 26 "testing" 27 "time" 28 29 "github.com/m3db/m3/src/cluster/services" 30 "github.com/m3db/m3/src/cluster/services/leader/campaign" 31 "github.com/m3db/m3/src/cluster/services/leader/election" 32 33 "github.com/stretchr/testify/assert" 34 "github.com/stretchr/testify/require" 35 clientv3 "go.etcd.io/etcd/client/v3" 36 "go.etcd.io/etcd/tests/v3/framework/integration" 37 "golang.org/x/net/context" 38 ) 39 40 var ( 41 newStatus = campaign.NewStatus 42 newErr = campaign.NewErrorStatus 43 followerS = newStatus(campaign.Follower) 44 leaderS = newStatus(campaign.Leader) 45 ) 46 47 func waitForStates(ch <-chan campaign.Status, early bool, states ...campaign.Status) error { 48 var seen []campaign.Status 49 for s := range ch { 50 seen = append(seen, s) 51 // terminate early (before channel closes) 52 if early && reflect.DeepEqual(seen, states) { 53 return nil 54 } 55 } 56 57 if !reflect.DeepEqual(seen, states) { 58 return fmt.Errorf("states did not match: %v != %v", seen, states) 59 } 60 61 return nil 62 } 63 64 type testCluster struct { 65 t *testing.T 66 cluster *integration.Cluster 67 } 68 69 func newTestCluster(t *testing.T) *testCluster { 70 integration.BeforeTestExternal(t) 71 return &testCluster{ 72 t: t, 73 cluster: integration.NewCluster(t, &integration.ClusterConfig{ 74 Size: 1, 75 }), 76 } 77 } 78 79 func (tc *testCluster) close() { 80 tc.cluster.Terminate(tc.t) 81 } 82 83 func (tc *testCluster) etcdClient() *clientv3.Client { 84 return tc.cluster.RandClient() 85 } 86 87 func (tc *testCluster) options() Options { 88 sid := services.NewServiceID(). 89 SetEnvironment("e1"). 90 SetName("s1"). 91 SetZone("z1") 92 93 eopts := services.NewElectionOptions(). 94 SetTTLSecs(5) 95 96 return NewOptions(). 97 SetServiceID(sid). 98 SetElectionOpts(eopts) 99 } 100 101 func (tc *testCluster) client() *client { 102 svc, err := newClient(tc.etcdClient(), tc.options(), "") 103 require.NoError(tc.t, err) 104 105 return svc 106 } 107 108 func (tc *testCluster) service() services.LeaderService { 109 svc, err := NewService(tc.etcdClient(), tc.options()) 110 require.NoError(tc.t, err) 111 112 return svc 113 } 114 115 func (tc *testCluster) opts(val string) services.CampaignOptions { 116 opts, err := services.NewCampaignOptions() 117 require.NoError(tc.t, err) 118 return opts.SetLeaderValue(val) 119 } 120 121 func TestNewClient(t *testing.T) { 122 tc := newTestCluster(t) 123 defer tc.close() 124 125 svc, err := newClient(tc.etcdClient(), tc.options(), "") 126 assert.NoError(t, err) 127 assert.NotNil(t, svc) 128 } 129 130 func TestNewClient_BadCluster(t *testing.T) { 131 tc := newTestCluster(t) 132 cl := tc.etcdClient() 133 tc.close() 134 135 _, err := newClient(cl, tc.options(), "") 136 assert.Error(t, err) 137 } 138 139 func TestCampaign(t *testing.T) { 140 tc := newTestCluster(t) 141 defer tc.close() 142 143 svc := tc.client() 144 145 sc, err := svc.campaign(tc.opts("foo")) 146 assert.NoError(t, err) 147 148 waitForStates(sc, true, followerS, leaderS) 149 150 _, err = svc.campaign(tc.opts("foo2")) 151 assert.Equal(t, ErrCampaignInProgress, err) 152 153 err = svc.resign() 154 assert.NoError(t, err) 155 156 errC := make(chan error) 157 go func() { 158 errC <- waitForStates(sc, false, followerS) 159 }() 160 161 err = <-errC 162 assert.NoError(t, err) 163 164 sc, err = svc.campaign(tc.opts("foo3")) 165 assert.NoError(t, err) 166 167 waitForStates(sc, true, followerS, leaderS) 168 169 err = svc.resign() 170 assert.NoError(t, err) 171 assert.NoError(t, waitForStates(sc, false, followerS)) 172 } 173 174 func TestCampaign_Override(t *testing.T) { 175 tc := newTestCluster(t) 176 defer tc.close() 177 178 svc := tc.client() 179 180 sc, err := svc.campaign(tc.opts("foo")) 181 assert.NoError(t, err) 182 assert.NoError(t, waitForStates(sc, true, followerS, leaderS)) 183 184 ld, err := svc.leader() 185 assert.NoError(t, err) 186 assert.Equal(t, "foo", ld) 187 } 188 189 func TestCampaign_Renew(t *testing.T) { 190 tc := newTestCluster(t) 191 defer tc.close() 192 193 svc := tc.client() 194 sc, err := svc.campaign(tc.opts("")) 195 assert.NoError(t, err) 196 assert.NoError(t, waitForStates(sc, true, followerS, leaderS)) 197 198 err = svc.resign() 199 assert.NoError(t, err) 200 assert.NoError(t, waitForStates(sc, false, followerS)) 201 202 _, err = svc.leader() 203 assert.Equal(t, ErrNoLeader, err) 204 205 sc2, err := svc.campaign(tc.opts("")) 206 assert.NoError(t, err) 207 assert.NoError(t, waitForStates(sc2, true, followerS, leaderS)) 208 } 209 210 func TestResign(t *testing.T) { 211 tc := newTestCluster(t) 212 defer tc.close() 213 214 svc := tc.client() 215 216 sc, err := svc.campaign(tc.opts("i1")) 217 assert.NoError(t, err) 218 219 assert.NoError(t, waitForStates(sc, true, followerS, leaderS)) 220 221 ld, err := svc.leader() 222 assert.NoError(t, err) 223 assert.Equal(t, "i1", ld) 224 225 err = svc.resign() 226 assert.NoError(t, err) 227 228 assert.NoError(t, waitForStates(sc, false, followerS)) 229 230 ld, err = svc.leader() 231 assert.Equal(t, ErrNoLeader, err) 232 assert.Equal(t, "", ld) 233 } 234 235 func TestResign_BlockingCampaign(t *testing.T) { 236 tc := newTestCluster(t) 237 defer tc.close() 238 239 svc1, svc2 := tc.client(), tc.client() 240 241 sc1, err := svc1.campaign(tc.opts("i1")) 242 assert.NoError(t, err) 243 assert.NoError(t, waitForStates(sc1, true, followerS, leaderS)) 244 245 sc2, err := svc2.campaign(tc.opts("i2")) 246 assert.NoError(t, err) 247 assert.NoError(t, waitForStates(sc2, true, followerS)) 248 249 err = svc2.resign() 250 assert.NoError(t, err) 251 assert.NoError(t, waitForStates(sc2, false, newErr(context.Canceled))) 252 } 253 254 func TestResign_Early(t *testing.T) { 255 tc := newTestCluster(t) 256 defer tc.close() 257 258 svc := tc.client() 259 260 err := svc.resign() 261 assert.NoError(t, err) 262 } 263 264 func TestObserve(t *testing.T) { 265 tc := newTestCluster(t) 266 defer tc.close() 267 268 svc1 := tc.client() 269 270 obsC, err := svc1.observe() 271 assert.NoError(t, err) 272 273 sc1, err := svc1.campaign(tc.opts("i1")) 274 assert.NoError(t, err) 275 assert.NoError(t, waitForStates(sc1, true, followerS, leaderS)) 276 277 select { 278 case <-time.After(time.Second): 279 t.Error("expected to receive leader update") 280 case v := <-obsC: 281 assert.Equal(t, "i1", v) 282 } 283 284 assert.NoError(t, svc1.close()) 285 select { 286 case <-time.After(5 * time.Second): 287 t.Error("expected client channel to be closed") 288 case _, ok := <-obsC: 289 assert.False(t, ok) 290 } 291 292 _, err = svc1.observe() 293 assert.Equal(t, errClientClosed, err) 294 } 295 296 func testHandoff(t *testing.T, resign bool) { 297 tc := newTestCluster(t) 298 defer tc.close() 299 300 svc1, svc2 := tc.client(), tc.client() 301 302 sc1, err := svc1.campaign(tc.opts("i1")) 303 assert.NoError(t, err) 304 305 assert.NoError(t, waitForStates(sc1, true, followerS, leaderS)) 306 307 sc2, err := svc2.campaign(tc.opts("i2")) 308 assert.NoError(t, err) 309 310 assert.NoError(t, waitForStates(sc2, true, followerS)) 311 312 ld, err := svc1.leader() 313 assert.NoError(t, err) 314 assert.Equal(t, ld, "i1") 315 316 if resign { 317 err = svc1.resign() 318 assert.NoError(t, waitForStates(sc1, false, followerS)) 319 } else { 320 err = svc1.close() 321 assert.NoError(t, waitForStates(sc1, false, newErr(election.ErrSessionExpired))) 322 } 323 assert.NoError(t, err) 324 325 assert.NoError(t, waitForStates(sc2, true, leaderS)) 326 327 ld, err = svc2.leader() 328 assert.NoError(t, err) 329 assert.Equal(t, ld, "i2") 330 } 331 332 func TestCampaign_Cancel_Resign(t *testing.T) { 333 testHandoff(t, true) 334 } 335 336 func TestCampaign_Cancel_Close(t *testing.T) { 337 testHandoff(t, false) 338 } 339 340 func TestCampaign_Close_NonLeader(t *testing.T) { 341 tc := newTestCluster(t) 342 defer tc.close() 343 344 svc1, svc2 := tc.client(), tc.client() 345 346 sc1, err := svc1.campaign(tc.opts("i1")) 347 assert.NoError(t, err) 348 assert.NoError(t, waitForStates(sc1, true, followerS, leaderS)) 349 350 sc2, err := svc2.campaign(tc.opts("i2")) 351 assert.NoError(t, err) 352 assert.NoError(t, waitForStates(sc2, true, followerS)) 353 354 ld, err := svc1.leader() 355 assert.NoError(t, err) 356 assert.Equal(t, ld, "i1") 357 358 err = svc2.close() 359 assert.NoError(t, err) 360 assert.NoError(t, waitForStates(sc2, false, newErr(context.Canceled))) 361 362 err = svc1.resign() 363 assert.NoError(t, err) 364 assert.NoError(t, waitForStates(sc1, false, followerS)) 365 366 _, err = svc2.leader() 367 assert.Equal(t, errClientClosed, err) 368 } 369 370 func TestClose(t *testing.T) { 371 tc := newTestCluster(t) 372 defer tc.close() 373 374 svc := tc.client() 375 376 sc, err := svc.campaign(tc.opts("i1")) 377 assert.NoError(t, err) 378 assert.NoError(t, waitForStates(sc, true, followerS, leaderS)) 379 380 ld, err := svc.leader() 381 assert.NoError(t, err) 382 assert.Equal(t, "i1", ld) 383 384 err = svc.close() 385 assert.NoError(t, err) 386 assert.True(t, svc.isClosed()) 387 assert.NoError(t, waitForStates(sc, false, newErr(election.ErrSessionExpired))) 388 389 err = svc.resign() 390 assert.Equal(t, errClientClosed, err) 391 392 _, err = svc.campaign(tc.opts("")) 393 assert.Equal(t, errClientClosed, err) 394 } 395 396 func TestLeader(t *testing.T) { 397 tc := newTestCluster(t) 398 defer tc.close() 399 400 svc1, svc2 := tc.client(), tc.client() 401 sc, err := svc1.campaign(tc.opts("i1")) 402 assert.NoError(t, err) 403 assert.NoError(t, waitForStates(sc, true, followerS, leaderS)) 404 405 ld, err := svc2.leader() 406 assert.NoError(t, err) 407 408 assert.Equal(t, "i1", ld) 409 } 410 411 func TestElectionPrefix(t *testing.T) { 412 for args, exp := range map[*struct { 413 env, name, eid string 414 }]string{ 415 {"", "svc", ""}: "_ld/svc/default", 416 {"env", "svc", ""}: "_ld/env/svc/default", 417 {"", "svc", "foo"}: "_ld/svc/foo", 418 {"env", "svc", "foo"}: "_ld/env/svc/foo", 419 } { 420 sid := services.NewServiceID(). 421 SetEnvironment(args.env). 422 SetName(args.name) 423 424 pfx := electionPrefix(sid, args.eid) 425 426 assert.Equal(t, exp, pfx) 427 } 428 }