github.com/hernad/nomad@v1.6.112/command/node_drain_test.go (about) 1 // Copyright (c) HashiCorp, Inc. 2 // SPDX-License-Identifier: MPL-2.0 3 4 package command 5 6 import ( 7 "bytes" 8 "fmt" 9 "strings" 10 "testing" 11 "time" 12 13 "github.com/hernad/nomad/api" 14 "github.com/hernad/nomad/ci" 15 "github.com/hernad/nomad/command/agent" 16 "github.com/hernad/nomad/helper/pointer" 17 "github.com/hernad/nomad/testutil" 18 "github.com/mitchellh/cli" 19 "github.com/posener/complete" 20 "github.com/stretchr/testify/assert" 21 "github.com/stretchr/testify/require" 22 ) 23 24 func TestNodeDrainCommand_Implements(t *testing.T) { 25 ci.Parallel(t) 26 var _ cli.Command = &NodeDrainCommand{} 27 } 28 29 func TestNodeDrainCommand_Detach(t *testing.T) { 30 ci.Parallel(t) 31 require := require.New(t) 32 server, client, url := testServer(t, true, func(c *agent.Config) { 33 c.NodeName = "drain_detach_node" 34 }) 35 defer server.Shutdown() 36 37 // Wait for a node to appear 38 var nodeID string 39 testutil.WaitForResult(func() (bool, error) { 40 nodes, _, err := client.Nodes().List(nil) 41 if err != nil { 42 return false, err 43 } 44 if len(nodes) == 0 { 45 return false, fmt.Errorf("missing node") 46 } 47 nodeID = nodes[0].ID 48 return true, nil 49 }, func(err error) { 50 t.Fatalf("err: %s", err) 51 }) 52 53 // Register a job to create an alloc to drain that will block draining 54 job := &api.Job{ 55 ID: pointer.Of("mock_service"), 56 Name: pointer.Of("mock_service"), 57 Datacenters: []string{"dc1"}, 58 TaskGroups: []*api.TaskGroup{ 59 { 60 Name: pointer.Of("mock_group"), 61 Tasks: []*api.Task{ 62 { 63 Name: "mock_task", 64 Driver: "mock_driver", 65 Config: map[string]interface{}{ 66 "run_for": "10m", 67 }, 68 }, 69 }, 70 }, 71 }, 72 } 73 74 _, _, err := client.Jobs().Register(job, nil) 75 require.Nil(err) 76 77 testutil.WaitForResult(func() (bool, error) { 78 allocs, _, err := client.Nodes().Allocations(nodeID, nil) 79 if err != nil { 80 return false, err 81 } 82 return len(allocs) > 0, fmt.Errorf("no allocs") 83 }, func(err error) { 84 t.Fatalf("err: %v", err) 85 }) 86 87 ui := cli.NewMockUi() 88 cmd := &NodeDrainCommand{Meta: Meta{Ui: ui}} 89 if code := cmd.Run([]string{"-address=" + url, "-self", "-enable", "-detach"}); code != 0 { 90 t.Fatalf("expected exit 0, got: %d", code) 91 } 92 93 out := ui.OutputWriter.String() 94 expected := "drain strategy set" 95 require.Contains(out, expected) 96 97 node, _, err := client.Nodes().Info(nodeID, nil) 98 require.Nil(err) 99 require.NotNil(node.DrainStrategy) 100 } 101 102 func TestNodeDrainCommand_Monitor(t *testing.T) { 103 ci.Parallel(t) 104 require := require.New(t) 105 server, client, url := testServer(t, true, func(c *agent.Config) { 106 c.NodeName = "drain_monitor_node" 107 }) 108 defer server.Shutdown() 109 110 // Wait for a node to appear 111 var nodeID string 112 testutil.WaitForResult(func() (bool, error) { 113 nodes, _, err := client.Nodes().List(nil) 114 if err != nil { 115 return false, err 116 } 117 if len(nodes) == 0 { 118 return false, fmt.Errorf("missing node") 119 } 120 if _, ok := nodes[0].Drivers["mock_driver"]; !ok { 121 return false, fmt.Errorf("mock_driver not ready") 122 } 123 nodeID = nodes[0].ID 124 return true, nil 125 }, func(err error) { 126 t.Fatalf("err: %s", err) 127 }) 128 129 // Register a service job to create allocs to drain 130 serviceCount := 3 131 job := &api.Job{ 132 ID: pointer.Of("mock_service"), 133 Name: pointer.Of("mock_service"), 134 Datacenters: []string{"dc1"}, 135 Type: pointer.Of("service"), 136 TaskGroups: []*api.TaskGroup{ 137 { 138 Name: pointer.Of("mock_group"), 139 Count: &serviceCount, 140 Migrate: &api.MigrateStrategy{ 141 MaxParallel: pointer.Of(1), 142 HealthCheck: pointer.Of("task_states"), 143 MinHealthyTime: pointer.Of(10 * time.Millisecond), 144 HealthyDeadline: pointer.Of(5 * time.Minute), 145 }, 146 Tasks: []*api.Task{ 147 { 148 Name: "mock_task", 149 Driver: "mock_driver", 150 Config: map[string]interface{}{ 151 "run_for": "10m", 152 }, 153 Resources: &api.Resources{ 154 CPU: pointer.Of(50), 155 MemoryMB: pointer.Of(50), 156 }, 157 }, 158 }, 159 }, 160 }, 161 } 162 163 _, _, err := client.Jobs().Register(job, nil) 164 require.Nil(err) 165 166 // Register a system job to ensure it is ignored during draining 167 sysjob := &api.Job{ 168 ID: pointer.Of("mock_system"), 169 Name: pointer.Of("mock_system"), 170 Datacenters: []string{"dc1"}, 171 Type: pointer.Of("system"), 172 TaskGroups: []*api.TaskGroup{ 173 { 174 Name: pointer.Of("mock_sysgroup"), 175 Count: pointer.Of(1), 176 Tasks: []*api.Task{ 177 { 178 Name: "mock_systask", 179 Driver: "mock_driver", 180 Config: map[string]interface{}{ 181 "run_for": "10m", 182 }, 183 Resources: &api.Resources{ 184 CPU: pointer.Of(50), 185 MemoryMB: pointer.Of(50), 186 }, 187 }, 188 }, 189 }, 190 }, 191 } 192 193 _, _, err = client.Jobs().Register(sysjob, nil) 194 require.Nil(err) 195 196 var allocs []*api.Allocation 197 testutil.WaitForResult(func() (bool, error) { 198 allocs, _, err = client.Nodes().Allocations(nodeID, nil) 199 if err != nil { 200 return false, err 201 } 202 if len(allocs) != serviceCount+1 { 203 return false, fmt.Errorf("number of allocs %d != count (%d)", len(allocs), serviceCount+1) 204 } 205 for _, a := range allocs { 206 if a.ClientStatus != "running" { 207 return false, fmt.Errorf("alloc %q still not running: %s", a.ID, a.ClientStatus) 208 } 209 } 210 return true, nil 211 }, func(err error) { 212 t.Fatalf("err: %v", err) 213 }) 214 215 outBuf := bytes.NewBuffer(nil) 216 ui := &cli.BasicUi{ 217 Reader: bytes.NewReader(nil), 218 Writer: outBuf, 219 ErrorWriter: outBuf, 220 } 221 cmd := &NodeDrainCommand{Meta: Meta{Ui: ui}} 222 args := []string{"-address=" + url, "-self", "-enable", "-deadline", "1s", "-ignore-system"} 223 t.Logf("Running: %v", args) 224 require.Zero(cmd.Run(args)) 225 226 out := outBuf.String() 227 t.Logf("Output:\n%s", out) 228 229 // Unfortunately travis is too slow to reliably see the expected output. The 230 // monitor goroutines may start only after some or all the allocs have been 231 // migrated. 232 if !testutil.IsTravis() { 233 require.Contains(out, "Drain complete for node") 234 for _, a := range allocs { 235 if *a.Job.Type == "system" { 236 if strings.Contains(out, a.ID) { 237 t.Fatalf("output should not contain system alloc %q", a.ID) 238 } 239 continue 240 } 241 require.Contains(out, fmt.Sprintf("Alloc %q marked for migration", a.ID)) 242 require.Contains(out, fmt.Sprintf("Alloc %q draining", a.ID)) 243 } 244 245 expected := fmt.Sprintf("All allocations on node %q have stopped\n", nodeID) 246 if !strings.HasSuffix(out, expected) { 247 t.Fatalf("expected output to end with:\n%s", expected) 248 } 249 } 250 251 // Test -monitor flag 252 outBuf.Reset() 253 args = []string{"-address=" + url, "-self", "-monitor", "-ignore-system"} 254 t.Logf("Running: %v", args) 255 require.Zero(cmd.Run(args)) 256 257 out = outBuf.String() 258 t.Logf("Output:\n%s", out) 259 require.Contains(out, "No drain strategy set") 260 } 261 262 func TestNodeDrainCommand_Monitor_NoDrainStrategy(t *testing.T) { 263 ci.Parallel(t) 264 require := require.New(t) 265 server, client, url := testServer(t, true, func(c *agent.Config) { 266 c.NodeName = "drain_monitor_node2" 267 }) 268 defer server.Shutdown() 269 270 // Wait for a node to appear 271 testutil.WaitForResult(func() (bool, error) { 272 nodes, _, err := client.Nodes().List(nil) 273 if err != nil { 274 return false, err 275 } 276 if len(nodes) == 0 { 277 return false, fmt.Errorf("missing node") 278 } 279 return true, nil 280 }, func(err error) { 281 t.Fatalf("err: %s", err) 282 }) 283 284 // Test -monitor flag 285 outBuf := bytes.NewBuffer(nil) 286 ui := &cli.BasicUi{ 287 Reader: bytes.NewReader(nil), 288 Writer: outBuf, 289 ErrorWriter: outBuf, 290 } 291 cmd := &NodeDrainCommand{Meta: Meta{Ui: ui}} 292 args := []string{"-address=" + url, "-self", "-monitor", "-ignore-system"} 293 t.Logf("Running: %v", args) 294 if code := cmd.Run(args); code != 0 { 295 t.Fatalf("expected exit 0, got: %d\n%s", code, outBuf.String()) 296 } 297 298 out := outBuf.String() 299 t.Logf("Output:\n%s", out) 300 301 require.Contains(out, "No drain strategy set") 302 } 303 304 func TestNodeDrainCommand_Fails(t *testing.T) { 305 ci.Parallel(t) 306 srv, _, url := testServer(t, false, nil) 307 defer srv.Shutdown() 308 309 ui := cli.NewMockUi() 310 cmd := &NodeDrainCommand{Meta: Meta{Ui: ui}} 311 312 // Fails on misuse 313 if code := cmd.Run([]string{"some", "bad", "args"}); code != 1 { 314 t.Fatalf("expected exit code 1, got: %d", code) 315 } 316 if out := ui.ErrorWriter.String(); !strings.Contains(out, commandErrorText(cmd)) { 317 t.Fatalf("expected help output, got: %s", out) 318 } 319 ui.ErrorWriter.Reset() 320 321 // Fails on connection failure 322 if code := cmd.Run([]string{"-address=nope", "-enable", "12345678-abcd-efab-cdef-123456789abc"}); code != 1 { 323 t.Fatalf("expected exit code 1, got: %d", code) 324 } 325 if out := ui.ErrorWriter.String(); !strings.Contains(out, "Error toggling") { 326 t.Fatalf("expected failed toggle error, got: %s", out) 327 } 328 ui.ErrorWriter.Reset() 329 330 // Fails on nonexistent node 331 if code := cmd.Run([]string{"-address=" + url, "-enable", "12345678-abcd-efab-cdef-123456789abc"}); code != 1 { 332 t.Fatalf("expected exit 1, got: %d", code) 333 } 334 if out := ui.ErrorWriter.String(); !strings.Contains(out, "No node(s) with prefix or id") { 335 t.Fatalf("expected not exist error, got: %s", out) 336 } 337 ui.ErrorWriter.Reset() 338 339 // Fails if both enable and disable specified 340 if code := cmd.Run([]string{"-enable", "-disable", "12345678-abcd-efab-cdef-123456789abc"}); code != 1 { 341 t.Fatalf("expected exit 1, got: %d", code) 342 } 343 if out := ui.ErrorWriter.String(); !strings.Contains(out, commandErrorText(cmd)) { 344 t.Fatalf("expected help output, got: %s", out) 345 } 346 ui.ErrorWriter.Reset() 347 348 // Fails if neither enable or disable specified 349 if code := cmd.Run([]string{"12345678-abcd-efab-cdef-123456789abc"}); code != 1 { 350 t.Fatalf("expected exit 1, got: %d", code) 351 } 352 if out := ui.ErrorWriter.String(); !strings.Contains(out, commandErrorText(cmd)) { 353 t.Fatalf("expected help output, got: %s", out) 354 } 355 ui.ErrorWriter.Reset() 356 357 // Fail on identifier with too few characters 358 if code := cmd.Run([]string{"-address=" + url, "-enable", "1"}); code != 1 { 359 t.Fatalf("expected exit 1, got: %d", code) 360 } 361 if out := ui.ErrorWriter.String(); !strings.Contains(out, "must contain at least two characters.") { 362 t.Fatalf("expected too few characters error, got: %s", out) 363 } 364 ui.ErrorWriter.Reset() 365 366 // Identifiers with uneven length should produce a query result 367 if code := cmd.Run([]string{"-address=" + url, "-enable", "123"}); code != 1 { 368 t.Fatalf("expected exit 1, got: %d", code) 369 } 370 if out := ui.ErrorWriter.String(); !strings.Contains(out, "No node(s) with prefix or id") { 371 t.Fatalf("expected not exist error, got: %s", out) 372 } 373 ui.ErrorWriter.Reset() 374 375 // Fail on disable being used with drain strategy flags 376 for _, flag := range []string{"-force", "-no-deadline", "-ignore-system"} { 377 if code := cmd.Run([]string{"-address=" + url, "-disable", flag, "12345678-abcd-efab-cdef-123456789abc"}); code != 1 { 378 t.Fatalf("expected exit 1, got: %d", code) 379 } 380 if out := ui.ErrorWriter.String(); !strings.Contains(out, "combined with flags configuring drain strategy") { 381 t.Fatalf("got: %s", out) 382 } 383 ui.ErrorWriter.Reset() 384 } 385 386 // Fail on setting a deadline plus deadline modifying flags 387 for _, flag := range []string{"-force", "-no-deadline"} { 388 if code := cmd.Run([]string{"-address=" + url, "-enable", "-deadline=10s", flag, "12345678-abcd-efab-cdef-123456789abc"}); code != 1 { 389 t.Fatalf("expected exit 1, got: %d", code) 390 } 391 if out := ui.ErrorWriter.String(); !strings.Contains(out, "deadline can't be combined with") { 392 t.Fatalf("got: %s", out) 393 } 394 ui.ErrorWriter.Reset() 395 } 396 397 // Fail on setting a force and no deadline 398 if code := cmd.Run([]string{"-address=" + url, "-enable", "-force", "-no-deadline", "12345678-abcd-efab-cdef-123456789abc"}); code != 1 { 399 t.Fatalf("expected exit 1, got: %d", code) 400 } 401 if out := ui.ErrorWriter.String(); !strings.Contains(out, "mutually exclusive") { 402 t.Fatalf("got: %s", out) 403 } 404 ui.ErrorWriter.Reset() 405 406 // Fail on setting a bad deadline 407 for _, flag := range []string{"-deadline=0s", "-deadline=-1s"} { 408 if code := cmd.Run([]string{"-address=" + url, "-enable", flag, "12345678-abcd-efab-cdef-123456789abc"}); code != 1 { 409 t.Fatalf("expected exit 1, got: %d", code) 410 } 411 if out := ui.ErrorWriter.String(); !strings.Contains(out, "positive") { 412 t.Fatalf("got: %s", out) 413 } 414 ui.ErrorWriter.Reset() 415 } 416 } 417 418 func TestNodeDrainCommand_AutocompleteArgs(t *testing.T) { 419 ci.Parallel(t) 420 assert := assert.New(t) 421 422 srv, client, url := testServer(t, true, nil) 423 defer srv.Shutdown() 424 425 // Wait for a node to appear 426 var nodeID string 427 testutil.WaitForResult(func() (bool, error) { 428 nodes, _, err := client.Nodes().List(nil) 429 if err != nil { 430 return false, err 431 } 432 if len(nodes) == 0 { 433 return false, fmt.Errorf("missing node") 434 } 435 nodeID = nodes[0].ID 436 return true, nil 437 }, func(err error) { 438 t.Fatalf("err: %s", err) 439 }) 440 441 ui := cli.NewMockUi() 442 cmd := &NodeDrainCommand{Meta: Meta{Ui: ui, flagAddress: url}} 443 444 prefix := nodeID[:len(nodeID)-5] 445 args := complete.Args{Last: prefix} 446 predictor := cmd.AutocompleteArgs() 447 448 res := predictor.Predict(args) 449 assert.Equal(1, len(res)) 450 assert.Equal(nodeID, res[0]) 451 }