github.com/hhrutter/nomad@v0.6.0-rc2.0.20170723054333-80c4b03f0705/nomad/serf_test.go (about) 1 package nomad 2 3 import ( 4 "errors" 5 "fmt" 6 "os" 7 "path" 8 "strings" 9 "testing" 10 11 "github.com/hashicorp/nomad/testutil" 12 "github.com/hashicorp/serf/serf" 13 ) 14 15 func TestNomad_JoinPeer(t *testing.T) { 16 s1 := testServer(t, nil) 17 defer s1.Shutdown() 18 s2 := testServer(t, func(c *Config) { 19 c.Region = "region2" 20 }) 21 defer s2.Shutdown() 22 testJoin(t, s1, s2) 23 24 testutil.WaitForResult(func() (bool, error) { 25 if members := s1.Members(); len(members) != 2 { 26 return false, fmt.Errorf("bad: %#v", members) 27 } 28 if members := s2.Members(); len(members) != 2 { 29 return false, fmt.Errorf("bad: %#v", members) 30 } 31 return true, nil 32 }, func(err error) { 33 t.Fatalf("err: %v", err) 34 }) 35 36 testutil.WaitForResult(func() (bool, error) { 37 if len(s1.peers) != 2 { 38 return false, fmt.Errorf("bad: %#v", s1.peers) 39 } 40 if len(s2.peers) != 2 { 41 return false, fmt.Errorf("bad: %#v", s2.peers) 42 } 43 if len(s1.localPeers) != 1 { 44 return false, fmt.Errorf("bad: %#v", s1.localPeers) 45 } 46 if len(s2.localPeers) != 1 { 47 return false, fmt.Errorf("bad: %#v", s2.localPeers) 48 } 49 return true, nil 50 }, func(err error) { 51 t.Fatalf("err: %v", err) 52 }) 53 } 54 55 func TestNomad_RemovePeer(t *testing.T) { 56 s1 := testServer(t, nil) 57 defer s1.Shutdown() 58 s2 := testServer(t, func(c *Config) { 59 c.Region = "region2" 60 }) 61 defer s2.Shutdown() 62 testJoin(t, s1, s2) 63 64 testutil.WaitForResult(func() (bool, error) { 65 if members := s1.Members(); len(members) != 2 { 66 return false, fmt.Errorf("bad: %#v", members) 67 } 68 if members := s2.Members(); len(members) != 2 { 69 return false, fmt.Errorf("bad: %#v", members) 70 } 71 return true, nil 72 }, func(err error) { 73 t.Fatalf("err: %v", err) 74 }) 75 76 // Leave immediately 77 s2.Leave() 78 s2.Shutdown() 79 80 testutil.WaitForResult(func() (bool, error) { 81 if len(s1.peers) != 1 { 82 return false, fmt.Errorf("bad: %#v", s1.peers) 83 } 84 if len(s2.peers) != 1 { 85 return false, fmt.Errorf("bad: %#v", s2.peers) 86 } 87 return true, nil 88 }, func(err error) { 89 t.Fatalf("err: %v", err) 90 }) 91 } 92 93 func TestNomad_ReapPeer(t *testing.T) { 94 dir := tmpDir(t) 95 defer os.RemoveAll(dir) 96 s1 := testServer(t, func(c *Config) { 97 c.BootstrapExpect = 3 98 c.DevMode = false 99 c.DevDisableBootstrap = true 100 c.DataDir = path.Join(dir, "node1") 101 }) 102 defer s1.Shutdown() 103 s2 := testServer(t, func(c *Config) { 104 c.BootstrapExpect = 3 105 c.DevMode = false 106 c.DevDisableBootstrap = true 107 c.DataDir = path.Join(dir, "node2") 108 }) 109 defer s2.Shutdown() 110 s3 := testServer(t, func(c *Config) { 111 c.BootstrapExpect = 3 112 c.DevMode = false 113 c.DevDisableBootstrap = true 114 c.DataDir = path.Join(dir, "node3") 115 }) 116 defer s3.Shutdown() 117 testJoin(t, s1, s2, s3) 118 119 testutil.WaitForResult(func() (bool, error) { 120 if members := s1.Members(); len(members) != 3 { 121 return false, fmt.Errorf("bad: %#v", members) 122 } 123 if members := s2.Members(); len(members) != 3 { 124 return false, fmt.Errorf("bad: %#v", members) 125 } 126 if members := s3.Members(); len(members) != 3 { 127 return false, fmt.Errorf("bad: %#v", members) 128 } 129 return true, nil 130 }, func(err error) { 131 t.Fatalf("err: %v", err) 132 }) 133 134 testutil.WaitForLeader(t, s1.RPC) 135 136 // Simulate a reap 137 mems := s1.Members() 138 var s2mem serf.Member 139 for _, m := range mems { 140 if strings.Contains(m.Name, s2.config.NodeName) { 141 s2mem = m 142 s2mem.Status = StatusReap 143 break 144 } 145 } 146 147 // Shutdown and then send the reap 148 s2.Shutdown() 149 s1.reconcileCh <- s2mem 150 s2.reconcileCh <- s2mem 151 s3.reconcileCh <- s2mem 152 153 testutil.WaitForResult(func() (bool, error) { 154 if len(s1.peers["global"]) != 2 { 155 return false, fmt.Errorf("bad: %#v", s1.peers["global"]) 156 } 157 peers, err := s1.numPeers() 158 if err != nil { 159 return false, fmt.Errorf("numPeers() failed: %v", err) 160 } 161 if peers != 2 { 162 return false, fmt.Errorf("bad: %#v", peers) 163 } 164 165 if len(s3.peers["global"]) != 2 { 166 return false, fmt.Errorf("bad: %#v", s1.peers["global"]) 167 } 168 peers, err = s3.numPeers() 169 if err != nil { 170 return false, fmt.Errorf("numPeers() failed: %v", err) 171 } 172 if peers != 2 { 173 return false, fmt.Errorf("bad: %#v", peers) 174 } 175 return true, nil 176 }, func(err error) { 177 t.Fatalf("err: %v", err) 178 }) 179 } 180 181 func TestNomad_BootstrapExpect(t *testing.T) { 182 dir := tmpDir(t) 183 defer os.RemoveAll(dir) 184 185 s1 := testServer(t, func(c *Config) { 186 c.BootstrapExpect = 3 187 c.DevMode = false 188 c.DevDisableBootstrap = true 189 c.DataDir = path.Join(dir, "node1") 190 }) 191 defer s1.Shutdown() 192 s2 := testServer(t, func(c *Config) { 193 c.BootstrapExpect = 3 194 c.DevMode = false 195 c.DevDisableBootstrap = true 196 c.DataDir = path.Join(dir, "node2") 197 }) 198 defer s2.Shutdown() 199 s3 := testServer(t, func(c *Config) { 200 c.BootstrapExpect = 3 201 c.DevMode = false 202 c.DevDisableBootstrap = true 203 c.DataDir = path.Join(dir, "node3") 204 }) 205 defer s3.Shutdown() 206 testJoin(t, s1, s2, s3) 207 208 testutil.WaitForResult(func() (bool, error) { 209 peers, err := s1.numPeers() 210 if err != nil { 211 return false, err 212 } 213 if peers != 3 { 214 return false, fmt.Errorf("bad: %#v", peers) 215 } 216 peers, err = s2.numPeers() 217 if err != nil { 218 return false, err 219 } 220 if peers != 3 { 221 return false, fmt.Errorf("bad: %#v", peers) 222 } 223 peers, err = s3.numPeers() 224 if err != nil { 225 return false, err 226 } 227 if peers != 3 { 228 return false, fmt.Errorf("bad: %#v", peers) 229 } 230 if len(s1.localPeers) != 3 { 231 return false, fmt.Errorf("bad: %#v", s1.localPeers) 232 } 233 if len(s2.localPeers) != 3 { 234 return false, fmt.Errorf("bad: %#v", s2.localPeers) 235 } 236 if len(s3.localPeers) != 3 { 237 return false, fmt.Errorf("bad: %#v", s3.localPeers) 238 } 239 return true, nil 240 }, func(err error) { 241 t.Fatalf("err: %v", err) 242 }) 243 244 // Join a fourth server after quorum has already been formed and ensure 245 // there is no election 246 s4 := testServer(t, func(c *Config) { 247 c.BootstrapExpect = 3 248 c.DevMode = false 249 c.DevDisableBootstrap = true 250 c.DataDir = path.Join(dir, "node4") 251 }) 252 defer s4.Shutdown() 253 254 // Make sure a leader is elected, grab the current term and then add in 255 // the fourth server. 256 testutil.WaitForLeader(t, s1.RPC) 257 termBefore := s1.raft.Stats()["last_log_term"] 258 addr := fmt.Sprintf("127.0.0.1:%d", s1.config.SerfConfig.MemberlistConfig.BindPort) 259 if _, err := s4.Join([]string{addr}); err != nil { 260 t.Fatalf("err: %v", err) 261 } 262 263 // Wait for the new server to see itself added to the cluster. 264 var p4 int 265 testutil.WaitForResult(func() (bool, error) { 266 p4, _ = s4.numPeers() 267 return p4 == 4, errors.New(fmt.Sprintf("%d", p4)) 268 }, func(err error) { 269 t.Fatalf("should have 4 peers: %v", err) 270 }) 271 272 // Make sure there's still a leader and that the term didn't change, 273 // so we know an election didn't occur. 274 testutil.WaitForLeader(t, s1.RPC) 275 termAfter := s1.raft.Stats()["last_log_term"] 276 if termAfter != termBefore { 277 t.Fatalf("looks like an election took place") 278 } 279 } 280 281 func TestNomad_BadExpect(t *testing.T) { 282 s1 := testServer(t, func(c *Config) { 283 c.BootstrapExpect = 2 284 c.DevDisableBootstrap = true 285 }) 286 defer s1.Shutdown() 287 s2 := testServer(t, func(c *Config) { 288 c.BootstrapExpect = 3 289 c.DevDisableBootstrap = true 290 }) 291 defer s2.Shutdown() 292 servers := []*Server{s1, s2} 293 testJoin(t, s1, s2) 294 295 // Serf members should update 296 testutil.WaitForResult(func() (bool, error) { 297 for _, s := range servers { 298 members := s.Members() 299 if len(members) != 2 { 300 return false, fmt.Errorf("%d", len(members)) 301 } 302 } 303 return true, nil 304 }, func(err error) { 305 t.Fatalf("should have 2 peers: %v", err) 306 }) 307 308 // should still have no peers (because s2 is in expect=2 mode) 309 testutil.WaitForResult(func() (bool, error) { 310 for _, s := range servers { 311 p, _ := s.numPeers() 312 if p != 1 { 313 return false, fmt.Errorf("%d", p) 314 } 315 } 316 return true, nil 317 }, func(err error) { 318 t.Fatalf("should have 0 peers: %v", err) 319 }) 320 }