github.com/huiliang/nomad@v0.2.1-0.20151124023127-7a8b664699ff/nomad/heartbeat_test.go (about) 1 package nomad 2 3 import ( 4 "fmt" 5 "testing" 6 "time" 7 8 "github.com/hashicorp/net-rpc-msgpackrpc" 9 "github.com/hashicorp/nomad/nomad/mock" 10 "github.com/hashicorp/nomad/nomad/structs" 11 "github.com/hashicorp/nomad/testutil" 12 ) 13 14 func TestInitializeHeartbeatTimers(t *testing.T) { 15 s1 := testServer(t, nil) 16 defer s1.Shutdown() 17 testutil.WaitForLeader(t, s1.RPC) 18 19 node := mock.Node() 20 state := s1.fsm.State() 21 err := state.UpsertNode(1, node) 22 if err != nil { 23 t.Fatalf("err: %v", err) 24 } 25 26 // Reset the heartbeat timers 27 err = s1.initializeHeartbeatTimers() 28 if err != nil { 29 t.Fatalf("err: %v", err) 30 } 31 32 // Check that we have a timer 33 _, ok := s1.heartbeatTimers[node.ID] 34 if !ok { 35 t.Fatalf("missing heartbeat timer") 36 } 37 } 38 39 func TestResetHeartbeatTimer(t *testing.T) { 40 s1 := testServer(t, nil) 41 defer s1.Shutdown() 42 testutil.WaitForLeader(t, s1.RPC) 43 44 // Create a new timer 45 ttl, err := s1.resetHeartbeatTimer("test") 46 if err != nil { 47 t.Fatalf("err: %v", err) 48 } 49 if ttl < s1.config.MinHeartbeatTTL || ttl > 2*s1.config.MinHeartbeatTTL { 50 t.Fatalf("bad: %#v", ttl) 51 } 52 53 // Check that we have a timer 54 _, ok := s1.heartbeatTimers["test"] 55 if !ok { 56 t.Fatalf("missing heartbeat timer") 57 } 58 } 59 60 func TestResetHeartbeatTimerLocked(t *testing.T) { 61 s1 := testServer(t, nil) 62 defer s1.Shutdown() 63 testutil.WaitForLeader(t, s1.RPC) 64 65 s1.heartbeatTimersLock.Lock() 66 s1.resetHeartbeatTimerLocked("foo", 5*time.Millisecond) 67 s1.heartbeatTimersLock.Unlock() 68 69 if _, ok := s1.heartbeatTimers["foo"]; !ok { 70 t.Fatalf("missing timer") 71 } 72 73 time.Sleep(10 * time.Millisecond) 74 75 if _, ok := s1.heartbeatTimers["foo"]; ok { 76 t.Fatalf("timer should be gone") 77 } 78 } 79 80 func TestResetHeartbeatTimerLocked_Renew(t *testing.T) { 81 s1 := testServer(t, nil) 82 defer s1.Shutdown() 83 testutil.WaitForLeader(t, s1.RPC) 84 85 s1.heartbeatTimersLock.Lock() 86 s1.resetHeartbeatTimerLocked("foo", 5*time.Millisecond) 87 s1.heartbeatTimersLock.Unlock() 88 89 if _, ok := s1.heartbeatTimers["foo"]; !ok { 90 t.Fatalf("missing timer") 91 } 92 93 time.Sleep(2 * time.Millisecond) 94 95 // Renew the heartbeat 96 s1.heartbeatTimersLock.Lock() 97 s1.resetHeartbeatTimerLocked("foo", 5*time.Millisecond) 98 s1.heartbeatTimersLock.Unlock() 99 renew := time.Now() 100 101 // Watch for invalidation 102 for time.Now().Sub(renew) < 20*time.Millisecond { 103 s1.heartbeatTimersLock.Lock() 104 _, ok := s1.heartbeatTimers["foo"] 105 s1.heartbeatTimersLock.Unlock() 106 if !ok { 107 end := time.Now() 108 if diff := end.Sub(renew); diff < 5*time.Millisecond { 109 t.Fatalf("early invalidate %v", diff) 110 } 111 return 112 } 113 time.Sleep(time.Millisecond) 114 } 115 t.Fatalf("should have expired") 116 } 117 118 func TestInvalidateHeartbeat(t *testing.T) { 119 s1 := testServer(t, nil) 120 defer s1.Shutdown() 121 testutil.WaitForLeader(t, s1.RPC) 122 123 // Create a node 124 node := mock.Node() 125 state := s1.fsm.State() 126 err := state.UpsertNode(1, node) 127 if err != nil { 128 t.Fatalf("err: %v", err) 129 } 130 131 // This should cause a status update 132 s1.invalidateHeartbeat(node.ID) 133 134 // Check it is updated 135 out, err := state.NodeByID(node.ID) 136 if err != nil { 137 t.Fatalf("err: %v", err) 138 } 139 if !out.TerminalStatus() { 140 t.Fatalf("should update node: %#v", out) 141 } 142 } 143 144 func TestClearHeartbeatTimer(t *testing.T) { 145 s1 := testServer(t, nil) 146 defer s1.Shutdown() 147 testutil.WaitForLeader(t, s1.RPC) 148 149 s1.heartbeatTimersLock.Lock() 150 s1.resetHeartbeatTimerLocked("foo", 5*time.Millisecond) 151 s1.heartbeatTimersLock.Unlock() 152 153 err := s1.clearHeartbeatTimer("foo") 154 if err != nil { 155 t.Fatalf("err: %v", err) 156 } 157 158 if _, ok := s1.heartbeatTimers["foo"]; ok { 159 t.Fatalf("timer should be gone") 160 } 161 } 162 163 func TestClearAllHeartbeatTimers(t *testing.T) { 164 s1 := testServer(t, nil) 165 defer s1.Shutdown() 166 testutil.WaitForLeader(t, s1.RPC) 167 168 s1.heartbeatTimersLock.Lock() 169 s1.resetHeartbeatTimerLocked("foo", 10*time.Millisecond) 170 s1.resetHeartbeatTimerLocked("bar", 10*time.Millisecond) 171 s1.resetHeartbeatTimerLocked("baz", 10*time.Millisecond) 172 s1.heartbeatTimersLock.Unlock() 173 174 err := s1.clearAllHeartbeatTimers() 175 if err != nil { 176 t.Fatalf("err: %v", err) 177 } 178 179 if len(s1.heartbeatTimers) != 0 { 180 t.Fatalf("timers should be gone") 181 } 182 } 183 184 func TestServer_HeartbeatTTL_Failover(t *testing.T) { 185 s1 := testServer(t, nil) 186 defer s1.Shutdown() 187 188 s2 := testServer(t, func(c *Config) { 189 c.DevDisableBootstrap = true 190 }) 191 defer s2.Shutdown() 192 193 s3 := testServer(t, func(c *Config) { 194 c.DevDisableBootstrap = true 195 }) 196 defer s3.Shutdown() 197 servers := []*Server{s1, s2, s3} 198 testJoin(t, s1, s2, s3) 199 200 testutil.WaitForResult(func() (bool, error) { 201 peers, _ := s1.raftPeers.Peers() 202 return len(peers) == 3, nil 203 }, func(err error) { 204 t.Fatalf("should have 3 peers") 205 }) 206 207 // Find the leader 208 var leader *Server 209 for _, s := range servers { 210 // Check that s.heartbeatTimers is empty 211 if len(s.heartbeatTimers) != 0 { 212 t.Fatalf("should have no heartbeatTimers") 213 } 214 // Find the leader too 215 if s.IsLeader() { 216 leader = s 217 } 218 } 219 if leader == nil { 220 t.Fatalf("Should have a leader") 221 } 222 codec := rpcClient(t, leader) 223 224 // Create the register request 225 node := mock.Node() 226 req := &structs.NodeRegisterRequest{ 227 Node: node, 228 WriteRequest: structs.WriteRequest{Region: "global"}, 229 } 230 231 // Fetch the response 232 var resp structs.GenericResponse 233 if err := msgpackrpc.CallWithCodec(codec, "Node.Register", req, &resp); err != nil { 234 t.Fatalf("err: %v", err) 235 } 236 237 // Check that heartbeatTimers has the heartbeat ID 238 if _, ok := leader.heartbeatTimers[node.ID]; !ok { 239 t.Fatalf("missing heartbeat timer") 240 } 241 242 // Shutdown the leader! 243 leader.Shutdown() 244 245 // heartbeatTimers should be cleared on leader shutdown 246 if len(leader.heartbeatTimers) != 0 { 247 t.Fatalf("heartbeat timers should be empty on the shutdown leader") 248 } 249 250 // Find the new leader 251 testutil.WaitForResult(func() (bool, error) { 252 leader = nil 253 for _, s := range servers { 254 if s.IsLeader() { 255 leader = s 256 } 257 } 258 if leader == nil { 259 return false, fmt.Errorf("Should have a new leader") 260 } 261 262 // Ensure heartbeat timer is restored 263 if _, ok := leader.heartbeatTimers[node.ID]; !ok { 264 return false, fmt.Errorf("missing heartbeat timer") 265 } 266 267 return true, nil 268 }, func(err error) { 269 t.Fatalf("err: %s", err) 270 }) 271 }