gopkg.in/hashicorp/nomad.v0@v0.11.8/nomad/serf_test.go (about) 1 package nomad 2 3 import ( 4 "errors" 5 "fmt" 6 "os" 7 "path" 8 "strings" 9 "testing" 10 "time" 11 12 "github.com/hashicorp/nomad/testutil" 13 "github.com/hashicorp/serf/serf" 14 ) 15 16 func TestNomad_JoinPeer(t *testing.T) { 17 t.Parallel() 18 19 s1, cleanupS1 := TestServer(t, nil) 20 defer cleanupS1() 21 s2, cleanupS2 := TestServer(t, func(c *Config) { 22 c.Region = "region2" 23 }) 24 defer cleanupS2() 25 TestJoin(t, s1, s2) 26 27 testutil.WaitForResult(func() (bool, error) { 28 if members := s1.Members(); len(members) != 2 { 29 return false, fmt.Errorf("bad: %#v", members) 30 } 31 if members := s2.Members(); len(members) != 2 { 32 return false, fmt.Errorf("bad: %#v", members) 33 } 34 return true, nil 35 }, func(err error) { 36 t.Fatalf("err: %v", err) 37 }) 38 39 testutil.WaitForResult(func() (bool, error) { 40 if len(s1.peers) != 2 { 41 return false, fmt.Errorf("bad: %#v", s1.peers) 42 } 43 if len(s2.peers) != 2 { 44 return false, fmt.Errorf("bad: %#v", s2.peers) 45 } 46 if len(s1.localPeers) != 1 { 47 return false, fmt.Errorf("bad: %#v", s1.localPeers) 48 } 49 if len(s2.localPeers) != 1 { 50 return false, fmt.Errorf("bad: %#v", s2.localPeers) 51 } 52 return true, nil 53 }, func(err error) { 54 t.Fatalf("err: %v", err) 55 }) 56 } 57 58 func TestNomad_RemovePeer(t *testing.T) { 59 t.Parallel() 60 61 s1, cleanupS1 := TestServer(t, nil) 62 defer cleanupS1() 63 s2, cleanupS2 := TestServer(t, func(c *Config) { 64 c.Region = "global" 65 }) 66 defer cleanupS2() 67 TestJoin(t, s1, s2) 68 69 testutil.WaitForResult(func() (bool, error) { 70 if members := s1.Members(); len(members) != 2 { 71 return false, fmt.Errorf("bad: %#v", members) 72 } 73 if members := s2.Members(); len(members) != 2 { 74 return false, fmt.Errorf("bad: %#v", members) 75 } 76 return true, nil 77 }, func(err error) { 78 t.Fatalf("err: %v", err) 79 }) 80 81 // Leave immediately 82 s2.Leave() 83 s2.Shutdown() 84 85 testutil.WaitForResult(func() (bool, error) { 86 if len(s1.peers) != 1 { 87 return false, fmt.Errorf("bad: %#v", s1.peers) 88 } 89 if len(s2.peers) != 1 { 90 return false, fmt.Errorf("bad: %#v", s2.peers) 91 } 92 return true, nil 93 }, func(err error) { 94 t.Fatalf("err: %v", err) 95 }) 96 } 97 98 func TestNomad_ReapPeer(t *testing.T) { 99 t.Parallel() 100 101 dir := tmpDir(t) 102 defer os.RemoveAll(dir) 103 104 s1, cleanupS1 := TestServer(t, func(c *Config) { 105 c.NodeName = "node1" 106 c.BootstrapExpect = 3 107 c.DevMode = false 108 c.DataDir = path.Join(dir, "node1") 109 }) 110 defer cleanupS1() 111 s2, cleanupS2 := TestServer(t, func(c *Config) { 112 c.NodeName = "node2" 113 c.BootstrapExpect = 3 114 c.DevMode = false 115 c.DataDir = path.Join(dir, "node2") 116 }) 117 defer cleanupS2() 118 s3, cleanupS3 := TestServer(t, func(c *Config) { 119 c.NodeName = "node3" 120 c.BootstrapExpect = 3 121 c.DevMode = false 122 c.DataDir = path.Join(dir, "node3") 123 }) 124 defer cleanupS3() 125 TestJoin(t, s1, s2, s3) 126 127 testutil.WaitForResult(func() (bool, error) { 128 // Retry the join to decrease flakiness 129 TestJoin(t, s1, s2, s3) 130 if members := s1.Members(); len(members) != 3 { 131 return false, fmt.Errorf("bad s1: %#v", members) 132 } 133 if members := s2.Members(); len(members) != 3 { 134 return false, fmt.Errorf("bad s2: %#v", members) 135 } 136 if members := s3.Members(); len(members) != 3 { 137 return false, fmt.Errorf("bad s3: %#v", members) 138 } 139 return true, nil 140 }, func(err error) { 141 t.Fatalf("err: %v", err) 142 }) 143 144 testutil.WaitForLeader(t, s1.RPC) 145 146 // Simulate a reap 147 mems := s1.Members() 148 var s2mem serf.Member 149 for _, m := range mems { 150 if strings.Contains(m.Name, s2.config.NodeName) { 151 s2mem = m 152 s2mem.Status = StatusReap 153 break 154 } 155 } 156 157 // Shutdown and then send the reap 158 s2.Shutdown() 159 s1.reconcileCh <- s2mem 160 s2.reconcileCh <- s2mem 161 s3.reconcileCh <- s2mem 162 163 testutil.WaitForResult(func() (bool, error) { 164 if len(s1.peers["global"]) != 2 { 165 return false, fmt.Errorf("bad: %#v", s1.peers["global"]) 166 } 167 peers, err := s1.numPeers() 168 if err != nil { 169 return false, fmt.Errorf("numPeers() failed: %v", err) 170 } 171 if peers != 2 { 172 return false, fmt.Errorf("bad: %#v", peers) 173 } 174 175 if len(s3.peers["global"]) != 2 { 176 return false, fmt.Errorf("bad: %#v", s1.peers["global"]) 177 } 178 peers, err = s3.numPeers() 179 if err != nil { 180 return false, fmt.Errorf("numPeers() failed: %v", err) 181 } 182 if peers != 2 { 183 return false, fmt.Errorf("bad: %#v", peers) 184 } 185 return true, nil 186 }, func(err error) { 187 t.Fatalf("err: %v", err) 188 }) 189 } 190 191 func TestNomad_BootstrapExpect(t *testing.T) { 192 t.Parallel() 193 194 dir := tmpDir(t) 195 defer os.RemoveAll(dir) 196 197 s1, cleanupS1 := TestServer(t, func(c *Config) { 198 c.BootstrapExpect = 3 199 c.DevMode = false 200 c.DataDir = path.Join(dir, "node1") 201 }) 202 defer cleanupS1() 203 s2, cleanupS2 := TestServer(t, func(c *Config) { 204 c.BootstrapExpect = 3 205 c.DevMode = false 206 c.DataDir = path.Join(dir, "node2") 207 }) 208 defer cleanupS2() 209 s3, cleanupS3 := TestServer(t, func(c *Config) { 210 c.BootstrapExpect = 3 211 c.DevMode = false 212 c.DataDir = path.Join(dir, "node3") 213 }) 214 defer cleanupS3() 215 TestJoin(t, s1, s2, s3) 216 217 testutil.WaitForResult(func() (bool, error) { 218 // Retry the join to decrease flakiness 219 TestJoin(t, s1, s2, s3) 220 peers, err := s1.numPeers() 221 if err != nil { 222 return false, err 223 } 224 if peers != 3 { 225 return false, fmt.Errorf("bad: %#v", peers) 226 } 227 peers, err = s2.numPeers() 228 if err != nil { 229 return false, err 230 } 231 if peers != 3 { 232 return false, fmt.Errorf("bad: %#v", peers) 233 } 234 peers, err = s3.numPeers() 235 if err != nil { 236 return false, err 237 } 238 if peers != 3 { 239 return false, fmt.Errorf("bad: %#v", peers) 240 } 241 if len(s1.localPeers) != 3 { 242 return false, fmt.Errorf("bad: %#v", s1.localPeers) 243 } 244 if len(s2.localPeers) != 3 { 245 return false, fmt.Errorf("bad: %#v", s2.localPeers) 246 } 247 if len(s3.localPeers) != 3 { 248 return false, fmt.Errorf("bad: %#v", s3.localPeers) 249 } 250 return true, nil 251 }, func(err error) { 252 t.Fatalf("err: %v", err) 253 }) 254 255 // Join a fourth server after quorum has already been formed and ensure 256 // there is no election 257 s4, cleanupS4 := TestServer(t, func(c *Config) { 258 c.BootstrapExpect = 3 259 c.DevMode = false 260 c.DataDir = path.Join(dir, "node4") 261 }) 262 defer cleanupS4() 263 264 // Make sure a leader is elected, grab the current term and then add in 265 // the fourth server. 266 testutil.WaitForLeader(t, s1.RPC) 267 termBefore := s1.raft.Stats()["last_log_term"] 268 269 var addresses []string 270 for _, s := range []*Server{s1, s2, s3} { 271 addr := fmt.Sprintf("127.0.0.1:%d", s.config.SerfConfig.MemberlistConfig.BindPort) 272 addresses = append(addresses, addr) 273 } 274 if _, err := s4.Join(addresses); err != nil { 275 t.Fatalf("err: %v", err) 276 } 277 278 // Wait for the new server to see itself added to the cluster. 279 var p4 int 280 testutil.WaitForResult(func() (bool, error) { 281 // Retry join to reduce flakiness 282 if _, err := s4.Join(addresses); err != nil { 283 t.Fatalf("err: %v", err) 284 } 285 p4, _ = s4.numPeers() 286 return p4 == 4, errors.New(fmt.Sprintf("%d", p4)) 287 }, func(err error) { 288 t.Fatalf("should have 4 peers: %v", err) 289 }) 290 291 // Make sure there's still a leader and that the term didn't change, 292 // so we know an election didn't occur. 293 testutil.WaitForLeader(t, s1.RPC) 294 termAfter := s1.raft.Stats()["last_log_term"] 295 if termAfter != termBefore { 296 t.Fatalf("looks like an election took place") 297 } 298 } 299 300 func TestNomad_BootstrapExpect_NonVoter(t *testing.T) { 301 t.Parallel() 302 303 dir := tmpDir(t) 304 defer os.RemoveAll(dir) 305 306 s1, cleanupS1 := TestServer(t, func(c *Config) { 307 c.BootstrapExpect = 2 308 c.DevMode = false 309 c.DataDir = path.Join(dir, "node1") 310 c.NonVoter = true 311 }) 312 defer cleanupS1() 313 s2, cleanupS2 := TestServer(t, func(c *Config) { 314 c.BootstrapExpect = 2 315 c.DevMode = false 316 c.DataDir = path.Join(dir, "node2") 317 c.NonVoter = true 318 }) 319 defer cleanupS2() 320 s3, cleanupS3 := TestServer(t, func(c *Config) { 321 c.BootstrapExpect = 2 322 c.DevMode = false 323 c.DataDir = path.Join(dir, "node3") 324 }) 325 defer cleanupS3() 326 TestJoin(t, s1, s2, s3) 327 328 // Assert that we do not bootstrap 329 testutil.AssertUntil(testutil.Timeout(time.Second), func() (bool, error) { 330 _, p := s1.getLeader() 331 if p != nil { 332 return false, fmt.Errorf("leader %v", p) 333 } 334 335 return true, nil 336 }, func(err error) { 337 t.Fatalf("should not have leader: %v", err) 338 }) 339 340 // Add the fourth server that is a voter 341 s4, cleanupS4 := TestServer(t, func(c *Config) { 342 c.BootstrapExpect = 2 343 c.DevMode = false 344 c.DataDir = path.Join(dir, "node4") 345 }) 346 defer cleanupS4() 347 TestJoin(t, s1, s2, s3, s4) 348 349 testutil.WaitForResult(func() (bool, error) { 350 // Retry the join to decrease flakiness 351 TestJoin(t, s1, s2, s3, s4) 352 peers, err := s1.numPeers() 353 if err != nil { 354 return false, err 355 } 356 if peers != 4 { 357 return false, fmt.Errorf("bad: %#v", peers) 358 } 359 peers, err = s2.numPeers() 360 if err != nil { 361 return false, err 362 } 363 if peers != 4 { 364 return false, fmt.Errorf("bad: %#v", peers) 365 } 366 peers, err = s3.numPeers() 367 if err != nil { 368 return false, err 369 } 370 if peers != 4 { 371 return false, fmt.Errorf("bad: %#v", peers) 372 } 373 peers, err = s4.numPeers() 374 if err != nil { 375 return false, err 376 } 377 if peers != 4 { 378 return false, fmt.Errorf("bad: %#v", peers) 379 } 380 381 if len(s1.localPeers) != 4 { 382 return false, fmt.Errorf("bad: %#v", s1.localPeers) 383 } 384 if len(s2.localPeers) != 4 { 385 return false, fmt.Errorf("bad: %#v", s2.localPeers) 386 } 387 if len(s3.localPeers) != 4 { 388 return false, fmt.Errorf("bad: %#v", s3.localPeers) 389 } 390 if len(s4.localPeers) != 4 { 391 return false, fmt.Errorf("bad: %#v", s3.localPeers) 392 } 393 394 _, p := s1.getLeader() 395 if p == nil { 396 return false, fmt.Errorf("no leader") 397 } 398 return true, nil 399 }, func(err error) { 400 t.Fatalf("err: %v", err) 401 }) 402 403 } 404 405 func TestNomad_BadExpect(t *testing.T) { 406 t.Parallel() 407 408 s1, cleanupS1 := TestServer(t, func(c *Config) { 409 c.BootstrapExpect = 2 410 }) 411 defer cleanupS1() 412 s2, cleanupS2 := TestServer(t, func(c *Config) { 413 c.BootstrapExpect = 3 414 }) 415 defer cleanupS2() 416 servers := []*Server{s1, s2} 417 TestJoin(t, s1, s2) 418 419 // Serf members should update 420 testutil.WaitForResult(func() (bool, error) { 421 for _, s := range servers { 422 members := s.Members() 423 if len(members) != 2 { 424 return false, fmt.Errorf("%d", len(members)) 425 } 426 } 427 return true, nil 428 }, func(err error) { 429 t.Fatalf("should have 2 peers: %v", err) 430 }) 431 432 // should still have no peers (because s2 is in expect=2 mode) 433 testutil.WaitForResult(func() (bool, error) { 434 for _, s := range servers { 435 p, _ := s.numPeers() 436 if p != 0 { 437 return false, fmt.Errorf("%d", p) 438 } 439 } 440 return true, nil 441 }, func(err error) { 442 t.Fatalf("should have 0 peers: %v", err) 443 }) 444 }