github.com/anuvu/nomad@v0.8.7-atom1/command/node_drain_test.go (about) 1 package command 2 3 import ( 4 "bytes" 5 "fmt" 6 "strings" 7 "testing" 8 "time" 9 10 "github.com/hashicorp/nomad/api" 11 "github.com/hashicorp/nomad/command/agent" 12 "github.com/hashicorp/nomad/helper" 13 "github.com/hashicorp/nomad/testutil" 14 "github.com/mitchellh/cli" 15 "github.com/posener/complete" 16 "github.com/stretchr/testify/assert" 17 "github.com/stretchr/testify/require" 18 ) 19 20 func TestNodeDrainCommand_Implements(t *testing.T) { 21 t.Parallel() 22 var _ cli.Command = &NodeDrainCommand{} 23 } 24 25 func TestNodeDrainCommand_Detach(t *testing.T) { 26 t.Parallel() 27 require := require.New(t) 28 server, client, url := testServer(t, true, func(c *agent.Config) { 29 c.NodeName = "drain_detach_node" 30 }) 31 defer server.Shutdown() 32 33 // Wait for a node to appear 34 var nodeID string 35 testutil.WaitForResult(func() (bool, error) { 36 nodes, _, err := client.Nodes().List(nil) 37 if err != nil { 38 return false, err 39 } 40 if len(nodes) == 0 { 41 return false, fmt.Errorf("missing node") 42 } 43 nodeID = nodes[0].ID 44 return true, nil 45 }, func(err error) { 46 t.Fatalf("err: %s", err) 47 }) 48 49 // Register a job to create an alloc to drain that will block draining 50 job := &api.Job{ 51 ID: helper.StringToPtr("mock_service"), 52 Name: helper.StringToPtr("mock_service"), 53 Datacenters: []string{"dc1"}, 54 TaskGroups: []*api.TaskGroup{ 55 { 56 Name: helper.StringToPtr("mock_group"), 57 Tasks: []*api.Task{ 58 { 59 Name: "mock_task", 60 Driver: "mock_driver", 61 Config: map[string]interface{}{ 62 "run_for": "10m", 63 "exit_after": "10m", 64 }, 65 }, 66 }, 67 }, 68 }, 69 } 70 71 _, _, err := client.Jobs().Register(job, nil) 72 require.Nil(err) 73 74 testutil.WaitForResult(func() (bool, error) { 75 allocs, _, err := client.Nodes().Allocations(nodeID, nil) 76 if err != nil { 77 return false, err 78 } 79 return len(allocs) > 0, fmt.Errorf("no allocs") 80 }, func(err error) { 81 t.Fatalf("err: %v", err) 82 }) 83 84 ui := new(cli.MockUi) 85 cmd := &NodeDrainCommand{Meta: Meta{Ui: ui}} 86 if code := cmd.Run([]string{"-address=" + url, "-self", "-enable", "-detach"}); code != 0 { 87 t.Fatalf("expected exit 0, got: %d", code) 88 } 89 90 out := ui.OutputWriter.String() 91 expected := "drain strategy set" 92 require.Contains(out, expected) 93 94 node, _, err := client.Nodes().Info(nodeID, nil) 95 require.Nil(err) 96 require.NotNil(node.DrainStrategy) 97 } 98 99 func TestNodeDrainCommand_Monitor(t *testing.T) { 100 t.Parallel() 101 require := require.New(t) 102 server, client, url := testServer(t, true, func(c *agent.Config) { 103 c.NodeName = "drain_monitor_node" 104 }) 105 defer server.Shutdown() 106 107 // Wait for a node to appear 108 var nodeID string 109 testutil.WaitForResult(func() (bool, error) { 110 nodes, _, err := client.Nodes().List(nil) 111 if err != nil { 112 return false, err 113 } 114 if len(nodes) == 0 { 115 return false, fmt.Errorf("missing node") 116 } 117 nodeID = nodes[0].ID 118 return true, nil 119 }, func(err error) { 120 t.Fatalf("err: %s", err) 121 }) 122 123 // Register a service job to create allocs to drain 124 serviceCount := 3 125 job := &api.Job{ 126 ID: helper.StringToPtr("mock_service"), 127 Name: helper.StringToPtr("mock_service"), 128 Datacenters: []string{"dc1"}, 129 Type: helper.StringToPtr("service"), 130 TaskGroups: []*api.TaskGroup{ 131 { 132 Name: helper.StringToPtr("mock_group"), 133 Count: &serviceCount, 134 Migrate: &api.MigrateStrategy{ 135 MaxParallel: helper.IntToPtr(1), 136 HealthCheck: helper.StringToPtr("task_states"), 137 MinHealthyTime: helper.TimeToPtr(10 * time.Millisecond), 138 HealthyDeadline: helper.TimeToPtr(5 * time.Minute), 139 }, 140 Tasks: []*api.Task{ 141 { 142 Name: "mock_task", 143 Driver: "mock_driver", 144 Config: map[string]interface{}{ 145 "run_for": "10m", 146 }, 147 Resources: &api.Resources{ 148 CPU: helper.IntToPtr(50), 149 MemoryMB: helper.IntToPtr(50), 150 }, 151 }, 152 }, 153 }, 154 }, 155 } 156 157 _, _, err := client.Jobs().Register(job, nil) 158 require.Nil(err) 159 160 // Register a system job to ensure it is ignored during draining 161 sysjob := &api.Job{ 162 ID: helper.StringToPtr("mock_system"), 163 Name: helper.StringToPtr("mock_system"), 164 Datacenters: []string{"dc1"}, 165 Type: helper.StringToPtr("system"), 166 TaskGroups: []*api.TaskGroup{ 167 { 168 Name: helper.StringToPtr("mock_sysgroup"), 169 Count: helper.IntToPtr(1), 170 Tasks: []*api.Task{ 171 { 172 Name: "mock_systask", 173 Driver: "mock_driver", 174 Config: map[string]interface{}{ 175 "run_for": "10m", 176 }, 177 Resources: &api.Resources{ 178 CPU: helper.IntToPtr(50), 179 MemoryMB: helper.IntToPtr(50), 180 }, 181 }, 182 }, 183 }, 184 }, 185 } 186 187 _, _, err = client.Jobs().Register(sysjob, nil) 188 require.Nil(err) 189 190 var allocs []*api.Allocation 191 testutil.WaitForResult(func() (bool, error) { 192 allocs, _, err = client.Nodes().Allocations(nodeID, nil) 193 if err != nil { 194 return false, err 195 } 196 if len(allocs) != serviceCount+1 { 197 return false, fmt.Errorf("number of allocs %d != count (%d)", len(allocs), serviceCount+1) 198 } 199 for _, a := range allocs { 200 if a.ClientStatus != "running" { 201 return false, fmt.Errorf("alloc %q still not running: %s", a.ID, a.ClientStatus) 202 } 203 } 204 return true, nil 205 }, func(err error) { 206 t.Fatalf("err: %v", err) 207 }) 208 209 outBuf := bytes.NewBuffer(nil) 210 ui := &cli.BasicUi{ 211 Reader: bytes.NewReader(nil), 212 Writer: outBuf, 213 ErrorWriter: outBuf, 214 } 215 cmd := &NodeDrainCommand{Meta: Meta{Ui: ui}} 216 args := []string{"-address=" + url, "-self", "-enable", "-deadline", "1s", "-ignore-system"} 217 t.Logf("Running: %v", args) 218 if code := cmd.Run(args); code != 0 { 219 t.Fatalf("expected exit 0, got: %d\n%s", code, outBuf.String()) 220 } 221 222 out := outBuf.String() 223 t.Logf("Output:\n%s", out) 224 225 // Unfortunately travis is too slow to reliably see the expected output. The 226 // monitor goroutines may start only after some or all the allocs have been 227 // migrated. 228 if !testutil.IsTravis() { 229 require.Contains(out, "marked all allocations for migration") 230 for _, a := range allocs { 231 if *a.Job.Type == "system" { 232 if strings.Contains(out, a.ID) { 233 t.Fatalf("output should not contain system alloc %q", a.ID) 234 } 235 continue 236 } 237 require.Contains(out, fmt.Sprintf("Alloc %q marked for migration", a.ID)) 238 require.Contains(out, fmt.Sprintf("Alloc %q draining", a.ID)) 239 } 240 241 expected := fmt.Sprintf("All allocations on node %q have stopped.\n", nodeID) 242 if !strings.HasSuffix(out, expected) { 243 t.Fatalf("expected output to end with:\n%s", expected) 244 } 245 } 246 247 // Test -monitor flag 248 outBuf.Reset() 249 args = []string{"-address=" + url, "-self", "-monitor", "-ignore-system"} 250 t.Logf("Running: %v", args) 251 if code := cmd.Run(args); code != 0 { 252 t.Fatalf("expected exit 0, got: %d\n%s", code, outBuf.String()) 253 } 254 255 out = outBuf.String() 256 t.Logf("Output:\n%s", out) 257 require.Contains(out, "No drain strategy set") 258 } 259 260 func TestNodeDrainCommand_Monitor_NoDrainStrategy(t *testing.T) { 261 t.Parallel() 262 require := require.New(t) 263 server, client, url := testServer(t, true, func(c *agent.Config) { 264 c.NodeName = "drain_monitor_node2" 265 }) 266 defer server.Shutdown() 267 268 // Wait for a node to appear 269 testutil.WaitForResult(func() (bool, error) { 270 nodes, _, err := client.Nodes().List(nil) 271 if err != nil { 272 return false, err 273 } 274 if len(nodes) == 0 { 275 return false, fmt.Errorf("missing node") 276 } 277 return true, nil 278 }, func(err error) { 279 t.Fatalf("err: %s", err) 280 }) 281 282 // Test -monitor flag 283 outBuf := bytes.NewBuffer(nil) 284 ui := &cli.BasicUi{ 285 Reader: bytes.NewReader(nil), 286 Writer: outBuf, 287 ErrorWriter: outBuf, 288 } 289 cmd := &NodeDrainCommand{Meta: Meta{Ui: ui}} 290 args := []string{"-address=" + url, "-self", "-monitor", "-ignore-system"} 291 t.Logf("Running: %v", args) 292 if code := cmd.Run(args); code != 0 { 293 t.Fatalf("expected exit 0, got: %d\n%s", code, outBuf.String()) 294 } 295 296 out := outBuf.String() 297 t.Logf("Output:\n%s", out) 298 299 require.Contains(out, "Monitoring node") 300 require.Contains(out, "No drain strategy set") 301 } 302 303 func TestNodeDrainCommand_Fails(t *testing.T) { 304 t.Parallel() 305 srv, _, url := testServer(t, false, nil) 306 defer srv.Shutdown() 307 308 ui := new(cli.MockUi) 309 cmd := &NodeDrainCommand{Meta: Meta{Ui: ui}} 310 311 // Fails on misuse 312 if code := cmd.Run([]string{"some", "bad", "args"}); code != 1 { 313 t.Fatalf("expected exit code 1, got: %d", code) 314 } 315 if out := ui.ErrorWriter.String(); !strings.Contains(out, commandErrorText(cmd)) { 316 t.Fatalf("expected help output, got: %s", out) 317 } 318 ui.ErrorWriter.Reset() 319 320 // Fails on connection failure 321 if code := cmd.Run([]string{"-address=nope", "-enable", "12345678-abcd-efab-cdef-123456789abc"}); code != 1 { 322 t.Fatalf("expected exit code 1, got: %d", code) 323 } 324 if out := ui.ErrorWriter.String(); !strings.Contains(out, "Error toggling") { 325 t.Fatalf("expected failed toggle error, got: %s", out) 326 } 327 ui.ErrorWriter.Reset() 328 329 // Fails on nonexistent node 330 if code := cmd.Run([]string{"-address=" + url, "-enable", "12345678-abcd-efab-cdef-123456789abc"}); code != 1 { 331 t.Fatalf("expected exit 1, got: %d", code) 332 } 333 if out := ui.ErrorWriter.String(); !strings.Contains(out, "No node(s) with prefix or id") { 334 t.Fatalf("expected not exist error, got: %s", out) 335 } 336 ui.ErrorWriter.Reset() 337 338 // Fails if both enable and disable specified 339 if code := cmd.Run([]string{"-enable", "-disable", "12345678-abcd-efab-cdef-123456789abc"}); code != 1 { 340 t.Fatalf("expected exit 1, got: %d", code) 341 } 342 if out := ui.ErrorWriter.String(); !strings.Contains(out, commandErrorText(cmd)) { 343 t.Fatalf("expected help output, got: %s", out) 344 } 345 ui.ErrorWriter.Reset() 346 347 // Fails if neither enable or disable specified 348 if code := cmd.Run([]string{"12345678-abcd-efab-cdef-123456789abc"}); code != 1 { 349 t.Fatalf("expected exit 1, got: %d", code) 350 } 351 if out := ui.ErrorWriter.String(); !strings.Contains(out, commandErrorText(cmd)) { 352 t.Fatalf("expected help output, got: %s", out) 353 } 354 ui.ErrorWriter.Reset() 355 356 // Fail on identifier with too few characters 357 if code := cmd.Run([]string{"-address=" + url, "-enable", "1"}); code != 1 { 358 t.Fatalf("expected exit 1, got: %d", code) 359 } 360 if out := ui.ErrorWriter.String(); !strings.Contains(out, "must contain at least two characters.") { 361 t.Fatalf("expected too few characters error, got: %s", out) 362 } 363 ui.ErrorWriter.Reset() 364 365 // Identifiers with uneven length should produce a query result 366 if code := cmd.Run([]string{"-address=" + url, "-enable", "123"}); code != 1 { 367 t.Fatalf("expected exit 1, got: %d", code) 368 } 369 if out := ui.ErrorWriter.String(); !strings.Contains(out, "No node(s) with prefix or id") { 370 t.Fatalf("expected not exist error, got: %s", out) 371 } 372 ui.ErrorWriter.Reset() 373 374 // Fail on disable being used with drain strategy flags 375 for _, flag := range []string{"-force", "-no-deadline", "-ignore-system"} { 376 if code := cmd.Run([]string{"-address=" + url, "-disable", flag, "12345678-abcd-efab-cdef-123456789abc"}); code != 1 { 377 t.Fatalf("expected exit 1, got: %d", code) 378 } 379 if out := ui.ErrorWriter.String(); !strings.Contains(out, "combined with flags configuring drain strategy") { 380 t.Fatalf("got: %s", out) 381 } 382 ui.ErrorWriter.Reset() 383 } 384 385 // Fail on setting a deadline plus deadline modifying flags 386 for _, flag := range []string{"-force", "-no-deadline"} { 387 if code := cmd.Run([]string{"-address=" + url, "-enable", "-deadline=10s", flag, "12345678-abcd-efab-cdef-123456789abc"}); code != 1 { 388 t.Fatalf("expected exit 1, got: %d", code) 389 } 390 if out := ui.ErrorWriter.String(); !strings.Contains(out, "deadline can't be combined with") { 391 t.Fatalf("got: %s", out) 392 } 393 ui.ErrorWriter.Reset() 394 } 395 396 // Fail on setting a force and no deadline 397 if code := cmd.Run([]string{"-address=" + url, "-enable", "-force", "-no-deadline", "12345678-abcd-efab-cdef-123456789abc"}); code != 1 { 398 t.Fatalf("expected exit 1, got: %d", code) 399 } 400 if out := ui.ErrorWriter.String(); !strings.Contains(out, "mutually exclusive") { 401 t.Fatalf("got: %s", out) 402 } 403 ui.ErrorWriter.Reset() 404 405 // Fail on setting a bad deadline 406 for _, flag := range []string{"-deadline=0s", "-deadline=-1s"} { 407 if code := cmd.Run([]string{"-address=" + url, "-enable", flag, "12345678-abcd-efab-cdef-123456789abc"}); code != 1 { 408 t.Fatalf("expected exit 1, got: %d", code) 409 } 410 if out := ui.ErrorWriter.String(); !strings.Contains(out, "positive") { 411 t.Fatalf("got: %s", out) 412 } 413 ui.ErrorWriter.Reset() 414 } 415 } 416 417 func TestNodeDrainCommand_AutocompleteArgs(t *testing.T) { 418 assert := assert.New(t) 419 t.Parallel() 420 421 srv, client, url := testServer(t, true, nil) 422 defer srv.Shutdown() 423 424 // Wait for a node to appear 425 var nodeID string 426 testutil.WaitForResult(func() (bool, error) { 427 nodes, _, err := client.Nodes().List(nil) 428 if err != nil { 429 return false, err 430 } 431 if len(nodes) == 0 { 432 return false, fmt.Errorf("missing node") 433 } 434 nodeID = nodes[0].ID 435 return true, nil 436 }, func(err error) { 437 t.Fatalf("err: %s", err) 438 }) 439 440 ui := new(cli.MockUi) 441 cmd := &NodeDrainCommand{Meta: Meta{Ui: ui, flagAddress: url}} 442 443 prefix := nodeID[:len(nodeID)-5] 444 args := complete.Args{Last: prefix} 445 predictor := cmd.AutocompleteArgs() 446 447 res := predictor.Predict(args) 448 assert.Equal(1, len(res)) 449 assert.Equal(nodeID, res[0]) 450 }