github.com/manicqin/nomad@v0.9.5/nomad/serf_test.go (about) 1 package nomad 2 3 import ( 4 "errors" 5 "fmt" 6 "os" 7 "path" 8 "strings" 9 "testing" 10 "time" 11 12 "github.com/hashicorp/nomad/testutil" 13 "github.com/hashicorp/serf/serf" 14 ) 15 16 func TestNomad_JoinPeer(t *testing.T) { 17 t.Parallel() 18 19 s1, cleanupS1 := TestServer(t, nil) 20 defer cleanupS1() 21 s2, cleanupS2 := TestServer(t, func(c *Config) { 22 c.Region = "region2" 23 }) 24 defer cleanupS2() 25 TestJoin(t, s1, s2) 26 27 testutil.WaitForResult(func() (bool, error) { 28 if members := s1.Members(); len(members) != 2 { 29 return false, fmt.Errorf("bad: %#v", members) 30 } 31 if members := s2.Members(); len(members) != 2 { 32 return false, fmt.Errorf("bad: %#v", members) 33 } 34 return true, nil 35 }, func(err error) { 36 t.Fatalf("err: %v", err) 37 }) 38 39 testutil.WaitForResult(func() (bool, error) { 40 if len(s1.peers) != 2 { 41 return false, fmt.Errorf("bad: %#v", s1.peers) 42 } 43 if len(s2.peers) != 2 { 44 return false, fmt.Errorf("bad: %#v", s2.peers) 45 } 46 if len(s1.localPeers) != 1 { 47 return false, fmt.Errorf("bad: %#v", s1.localPeers) 48 } 49 if len(s2.localPeers) != 1 { 50 return false, fmt.Errorf("bad: %#v", s2.localPeers) 51 } 52 return true, nil 53 }, func(err error) { 54 t.Fatalf("err: %v", err) 55 }) 56 } 57 58 func TestNomad_RemovePeer(t *testing.T) { 59 t.Parallel() 60 61 s1, cleanupS1 := TestServer(t, nil) 62 defer cleanupS1() 63 s2, cleanupS2 := TestServer(t, func(c *Config) { 64 c.Region = "global" 65 }) 66 defer cleanupS2() 67 TestJoin(t, s1, s2) 68 69 testutil.WaitForResult(func() (bool, error) { 70 if members := s1.Members(); len(members) != 2 { 71 return false, fmt.Errorf("bad: %#v", members) 72 } 73 if members := s2.Members(); len(members) != 2 { 74 return false, fmt.Errorf("bad: %#v", members) 75 } 76 return true, nil 77 }, func(err error) { 78 t.Fatalf("err: %v", err) 79 }) 80 81 // Leave immediately 82 s2.Leave() 83 s2.Shutdown() 84 85 testutil.WaitForResult(func() (bool, error) { 86 if len(s1.peers) != 1 { 87 return false, fmt.Errorf("bad: %#v", s1.peers) 88 } 89 if len(s2.peers) != 1 { 90 return false, fmt.Errorf("bad: %#v", s2.peers) 91 } 92 return true, nil 93 }, func(err error) { 94 t.Fatalf("err: %v", err) 95 }) 96 } 97 98 func TestNomad_ReapPeer(t *testing.T) { 99 t.Parallel() 100 101 dir := tmpDir(t) 102 defer os.RemoveAll(dir) 103 104 s1, cleanupS1 := TestServer(t, func(c *Config) { 105 c.NodeName = "node1" 106 c.BootstrapExpect = 3 107 c.DevMode = false 108 c.DevDisableBootstrap = true 109 c.DataDir = path.Join(dir, "node1") 110 }) 111 defer cleanupS1() 112 s2, cleanupS2 := TestServer(t, func(c *Config) { 113 c.NodeName = "node2" 114 c.BootstrapExpect = 3 115 c.DevMode = false 116 c.DevDisableBootstrap = true 117 c.DataDir = path.Join(dir, "node2") 118 }) 119 defer cleanupS2() 120 s3, cleanupS3 := TestServer(t, func(c *Config) { 121 c.NodeName = "node3" 122 c.BootstrapExpect = 3 123 c.DevMode = false 124 c.DevDisableBootstrap = true 125 c.DataDir = path.Join(dir, "node3") 126 }) 127 defer cleanupS3() 128 TestJoin(t, s1, s2, s3) 129 130 testutil.WaitForResult(func() (bool, error) { 131 // Retry the join to decrease flakiness 132 TestJoin(t, s1, s2, s3) 133 if members := s1.Members(); len(members) != 3 { 134 return false, fmt.Errorf("bad s1: %#v", members) 135 } 136 if members := s2.Members(); len(members) != 3 { 137 return false, fmt.Errorf("bad s2: %#v", members) 138 } 139 if members := s3.Members(); len(members) != 3 { 140 return false, fmt.Errorf("bad s3: %#v", members) 141 } 142 return true, nil 143 }, func(err error) { 144 t.Fatalf("err: %v", err) 145 }) 146 147 testutil.WaitForLeader(t, s1.RPC) 148 149 // Simulate a reap 150 mems := s1.Members() 151 var s2mem serf.Member 152 for _, m := range mems { 153 if strings.Contains(m.Name, s2.config.NodeName) { 154 s2mem = m 155 s2mem.Status = StatusReap 156 break 157 } 158 } 159 160 // Shutdown and then send the reap 161 s2.Shutdown() 162 s1.reconcileCh <- s2mem 163 s2.reconcileCh <- s2mem 164 s3.reconcileCh <- s2mem 165 166 testutil.WaitForResult(func() (bool, error) { 167 if len(s1.peers["global"]) != 2 { 168 return false, fmt.Errorf("bad: %#v", s1.peers["global"]) 169 } 170 peers, err := s1.numPeers() 171 if err != nil { 172 return false, fmt.Errorf("numPeers() failed: %v", err) 173 } 174 if peers != 2 { 175 return false, fmt.Errorf("bad: %#v", peers) 176 } 177 178 if len(s3.peers["global"]) != 2 { 179 return false, fmt.Errorf("bad: %#v", s1.peers["global"]) 180 } 181 peers, err = s3.numPeers() 182 if err != nil { 183 return false, fmt.Errorf("numPeers() failed: %v", err) 184 } 185 if peers != 2 { 186 return false, fmt.Errorf("bad: %#v", peers) 187 } 188 return true, nil 189 }, func(err error) { 190 t.Fatalf("err: %v", err) 191 }) 192 } 193 194 func TestNomad_BootstrapExpect(t *testing.T) { 195 t.Parallel() 196 197 dir := tmpDir(t) 198 defer os.RemoveAll(dir) 199 200 s1, cleanupS1 := TestServer(t, func(c *Config) { 201 c.BootstrapExpect = 3 202 c.DevMode = false 203 c.DevDisableBootstrap = true 204 c.DataDir = path.Join(dir, "node1") 205 }) 206 defer cleanupS1() 207 s2, cleanupS2 := TestServer(t, func(c *Config) { 208 c.BootstrapExpect = 3 209 c.DevMode = false 210 c.DevDisableBootstrap = true 211 c.DataDir = path.Join(dir, "node2") 212 }) 213 defer cleanupS2() 214 s3, cleanupS3 := TestServer(t, func(c *Config) { 215 c.BootstrapExpect = 3 216 c.DevMode = false 217 c.DevDisableBootstrap = true 218 c.DataDir = path.Join(dir, "node3") 219 }) 220 defer cleanupS3() 221 TestJoin(t, s1, s2, s3) 222 223 testutil.WaitForResult(func() (bool, error) { 224 // Retry the join to decrease flakiness 225 TestJoin(t, s1, s2, s3) 226 peers, err := s1.numPeers() 227 if err != nil { 228 return false, err 229 } 230 if peers != 3 { 231 return false, fmt.Errorf("bad: %#v", peers) 232 } 233 peers, err = s2.numPeers() 234 if err != nil { 235 return false, err 236 } 237 if peers != 3 { 238 return false, fmt.Errorf("bad: %#v", peers) 239 } 240 peers, err = s3.numPeers() 241 if err != nil { 242 return false, err 243 } 244 if peers != 3 { 245 return false, fmt.Errorf("bad: %#v", peers) 246 } 247 if len(s1.localPeers) != 3 { 248 return false, fmt.Errorf("bad: %#v", s1.localPeers) 249 } 250 if len(s2.localPeers) != 3 { 251 return false, fmt.Errorf("bad: %#v", s2.localPeers) 252 } 253 if len(s3.localPeers) != 3 { 254 return false, fmt.Errorf("bad: %#v", s3.localPeers) 255 } 256 return true, nil 257 }, func(err error) { 258 t.Fatalf("err: %v", err) 259 }) 260 261 // Join a fourth server after quorum has already been formed and ensure 262 // there is no election 263 s4, cleanupS4 := TestServer(t, func(c *Config) { 264 c.BootstrapExpect = 3 265 c.DevMode = false 266 c.DevDisableBootstrap = true 267 c.DataDir = path.Join(dir, "node4") 268 }) 269 defer cleanupS4() 270 271 // Make sure a leader is elected, grab the current term and then add in 272 // the fourth server. 273 testutil.WaitForLeader(t, s1.RPC) 274 termBefore := s1.raft.Stats()["last_log_term"] 275 276 var addresses []string 277 for _, s := range []*Server{s1, s2, s3} { 278 addr := fmt.Sprintf("127.0.0.1:%d", s.config.SerfConfig.MemberlistConfig.BindPort) 279 addresses = append(addresses, addr) 280 } 281 if _, err := s4.Join(addresses); err != nil { 282 t.Fatalf("err: %v", err) 283 } 284 285 // Wait for the new server to see itself added to the cluster. 286 var p4 int 287 testutil.WaitForResult(func() (bool, error) { 288 // Retry join to reduce flakiness 289 if _, err := s4.Join(addresses); err != nil { 290 t.Fatalf("err: %v", err) 291 } 292 p4, _ = s4.numPeers() 293 return p4 == 4, errors.New(fmt.Sprintf("%d", p4)) 294 }, func(err error) { 295 t.Fatalf("should have 4 peers: %v", err) 296 }) 297 298 // Make sure there's still a leader and that the term didn't change, 299 // so we know an election didn't occur. 300 testutil.WaitForLeader(t, s1.RPC) 301 termAfter := s1.raft.Stats()["last_log_term"] 302 if termAfter != termBefore { 303 t.Fatalf("looks like an election took place") 304 } 305 } 306 307 func TestNomad_BootstrapExpect_NonVoter(t *testing.T) { 308 t.Parallel() 309 310 dir := tmpDir(t) 311 defer os.RemoveAll(dir) 312 313 s1, cleanupS1 := TestServer(t, func(c *Config) { 314 c.BootstrapExpect = 2 315 c.DevMode = false 316 c.DevDisableBootstrap = true 317 c.DataDir = path.Join(dir, "node1") 318 c.NonVoter = true 319 }) 320 defer cleanupS1() 321 s2, cleanupS2 := TestServer(t, func(c *Config) { 322 c.BootstrapExpect = 2 323 c.DevMode = false 324 c.DevDisableBootstrap = true 325 c.DataDir = path.Join(dir, "node2") 326 c.NonVoter = true 327 }) 328 defer cleanupS2() 329 s3, cleanupS3 := TestServer(t, func(c *Config) { 330 c.BootstrapExpect = 2 331 c.DevMode = false 332 c.DevDisableBootstrap = true 333 c.DataDir = path.Join(dir, "node3") 334 }) 335 defer cleanupS3() 336 TestJoin(t, s1, s2, s3) 337 338 // Assert that we do not bootstrap 339 testutil.AssertUntil(testutil.Timeout(time.Second), func() (bool, error) { 340 _, p := s1.getLeader() 341 if p != nil { 342 return false, fmt.Errorf("leader %v", p) 343 } 344 345 return true, nil 346 }, func(err error) { 347 t.Fatalf("should not have leader: %v", err) 348 }) 349 350 // Add the fourth server that is a voter 351 s4, cleanupS4 := TestServer(t, func(c *Config) { 352 c.BootstrapExpect = 2 353 c.DevMode = false 354 c.DevDisableBootstrap = true 355 c.DataDir = path.Join(dir, "node4") 356 }) 357 defer cleanupS4() 358 TestJoin(t, s1, s2, s3, s4) 359 360 testutil.WaitForResult(func() (bool, error) { 361 // Retry the join to decrease flakiness 362 TestJoin(t, s1, s2, s3, s4) 363 peers, err := s1.numPeers() 364 if err != nil { 365 return false, err 366 } 367 if peers != 4 { 368 return false, fmt.Errorf("bad: %#v", peers) 369 } 370 peers, err = s2.numPeers() 371 if err != nil { 372 return false, err 373 } 374 if peers != 4 { 375 return false, fmt.Errorf("bad: %#v", peers) 376 } 377 peers, err = s3.numPeers() 378 if err != nil { 379 return false, err 380 } 381 if peers != 4 { 382 return false, fmt.Errorf("bad: %#v", peers) 383 } 384 peers, err = s4.numPeers() 385 if err != nil { 386 return false, err 387 } 388 if peers != 4 { 389 return false, fmt.Errorf("bad: %#v", peers) 390 } 391 392 if len(s1.localPeers) != 4 { 393 return false, fmt.Errorf("bad: %#v", s1.localPeers) 394 } 395 if len(s2.localPeers) != 4 { 396 return false, fmt.Errorf("bad: %#v", s2.localPeers) 397 } 398 if len(s3.localPeers) != 4 { 399 return false, fmt.Errorf("bad: %#v", s3.localPeers) 400 } 401 if len(s4.localPeers) != 4 { 402 return false, fmt.Errorf("bad: %#v", s3.localPeers) 403 } 404 405 _, p := s1.getLeader() 406 if p == nil { 407 return false, fmt.Errorf("no leader") 408 } 409 return true, nil 410 }, func(err error) { 411 t.Fatalf("err: %v", err) 412 }) 413 414 } 415 416 func TestNomad_BadExpect(t *testing.T) { 417 t.Parallel() 418 419 s1, cleanupS1 := TestServer(t, func(c *Config) { 420 c.BootstrapExpect = 2 421 c.DevDisableBootstrap = true 422 }) 423 defer cleanupS1() 424 s2, cleanupS2 := TestServer(t, func(c *Config) { 425 c.BootstrapExpect = 3 426 c.DevDisableBootstrap = true 427 }) 428 defer cleanupS2() 429 servers := []*Server{s1, s2} 430 TestJoin(t, s1, s2) 431 432 // Serf members should update 433 testutil.WaitForResult(func() (bool, error) { 434 for _, s := range servers { 435 members := s.Members() 436 if len(members) != 2 { 437 return false, fmt.Errorf("%d", len(members)) 438 } 439 } 440 return true, nil 441 }, func(err error) { 442 t.Fatalf("should have 2 peers: %v", err) 443 }) 444 445 // should still have no peers (because s2 is in expect=2 mode) 446 testutil.WaitForResult(func() (bool, error) { 447 for _, s := range servers { 448 p, _ := s.numPeers() 449 if p != 1 { 450 return false, fmt.Errorf("%d", p) 451 } 452 } 453 return true, nil 454 }, func(err error) { 455 t.Fatalf("should have 0 peers: %v", err) 456 }) 457 }