github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/command/node_drain_test.go (about) 1 package command 2 3 import ( 4 "bytes" 5 "fmt" 6 "strings" 7 "testing" 8 "time" 9 10 "github.com/hashicorp/nomad/api" 11 "github.com/hashicorp/nomad/ci" 12 "github.com/hashicorp/nomad/command/agent" 13 "github.com/hashicorp/nomad/helper/pointer" 14 "github.com/hashicorp/nomad/testutil" 15 "github.com/mitchellh/cli" 16 "github.com/posener/complete" 17 "github.com/stretchr/testify/assert" 18 "github.com/stretchr/testify/require" 19 ) 20 21 func TestNodeDrainCommand_Implements(t *testing.T) { 22 ci.Parallel(t) 23 var _ cli.Command = &NodeDrainCommand{} 24 } 25 26 func TestNodeDrainCommand_Detach(t *testing.T) { 27 ci.Parallel(t) 28 require := require.New(t) 29 server, client, url := testServer(t, true, func(c *agent.Config) { 30 c.NodeName = "drain_detach_node" 31 }) 32 defer server.Shutdown() 33 34 // Wait for a node to appear 35 var nodeID string 36 testutil.WaitForResult(func() (bool, error) { 37 nodes, _, err := client.Nodes().List(nil) 38 if err != nil { 39 return false, err 40 } 41 if len(nodes) == 0 { 42 return false, fmt.Errorf("missing node") 43 } 44 nodeID = nodes[0].ID 45 return true, nil 46 }, func(err error) { 47 t.Fatalf("err: %s", err) 48 }) 49 50 // Register a job to create an alloc to drain that will block draining 51 job := &api.Job{ 52 ID: pointer.Of("mock_service"), 53 Name: pointer.Of("mock_service"), 54 Datacenters: []string{"dc1"}, 55 TaskGroups: []*api.TaskGroup{ 56 { 57 Name: pointer.Of("mock_group"), 58 Tasks: []*api.Task{ 59 { 60 Name: "mock_task", 61 Driver: "mock_driver", 62 Config: map[string]interface{}{ 63 "run_for": "10m", 64 }, 65 }, 66 }, 67 }, 68 }, 69 } 70 71 _, _, err := client.Jobs().Register(job, nil) 72 require.Nil(err) 73 74 testutil.WaitForResult(func() (bool, error) { 75 allocs, _, err := client.Nodes().Allocations(nodeID, nil) 76 if err != nil { 77 return false, err 78 } 79 return len(allocs) > 0, fmt.Errorf("no allocs") 80 }, func(err error) { 81 t.Fatalf("err: %v", err) 82 }) 83 84 ui := cli.NewMockUi() 85 cmd := &NodeDrainCommand{Meta: Meta{Ui: ui}} 86 if code := cmd.Run([]string{"-address=" + url, "-self", "-enable", "-detach"}); code != 0 { 87 t.Fatalf("expected exit 0, got: %d", code) 88 } 89 90 out := ui.OutputWriter.String() 91 expected := "drain strategy set" 92 require.Contains(out, expected) 93 94 node, _, err := client.Nodes().Info(nodeID, nil) 95 require.Nil(err) 96 require.NotNil(node.DrainStrategy) 97 } 98 99 func TestNodeDrainCommand_Monitor(t *testing.T) { 100 ci.Parallel(t) 101 require := require.New(t) 102 server, client, url := testServer(t, true, func(c *agent.Config) { 103 c.NodeName = "drain_monitor_node" 104 }) 105 defer server.Shutdown() 106 107 // Wait for a node to appear 108 var nodeID string 109 testutil.WaitForResult(func() (bool, error) { 110 nodes, _, err := client.Nodes().List(nil) 111 if err != nil { 112 return false, err 113 } 114 if len(nodes) == 0 { 115 return false, fmt.Errorf("missing node") 116 } 117 if _, ok := nodes[0].Drivers["mock_driver"]; !ok { 118 return false, fmt.Errorf("mock_driver not ready") 119 } 120 nodeID = nodes[0].ID 121 return true, nil 122 }, func(err error) { 123 t.Fatalf("err: %s", err) 124 }) 125 126 // Register a service job to create allocs to drain 127 serviceCount := 3 128 job := &api.Job{ 129 ID: pointer.Of("mock_service"), 130 Name: pointer.Of("mock_service"), 131 Datacenters: []string{"dc1"}, 132 Type: pointer.Of("service"), 133 TaskGroups: []*api.TaskGroup{ 134 { 135 Name: pointer.Of("mock_group"), 136 Count: &serviceCount, 137 Migrate: &api.MigrateStrategy{ 138 MaxParallel: pointer.Of(1), 139 HealthCheck: pointer.Of("task_states"), 140 MinHealthyTime: pointer.Of(10 * time.Millisecond), 141 HealthyDeadline: pointer.Of(5 * time.Minute), 142 }, 143 Tasks: []*api.Task{ 144 { 145 Name: "mock_task", 146 Driver: "mock_driver", 147 Config: map[string]interface{}{ 148 "run_for": "10m", 149 }, 150 Resources: &api.Resources{ 151 CPU: pointer.Of(50), 152 MemoryMB: pointer.Of(50), 153 }, 154 }, 155 }, 156 }, 157 }, 158 } 159 160 _, _, err := client.Jobs().Register(job, nil) 161 require.Nil(err) 162 163 // Register a system job to ensure it is ignored during draining 164 sysjob := &api.Job{ 165 ID: pointer.Of("mock_system"), 166 Name: pointer.Of("mock_system"), 167 Datacenters: []string{"dc1"}, 168 Type: pointer.Of("system"), 169 TaskGroups: []*api.TaskGroup{ 170 { 171 Name: pointer.Of("mock_sysgroup"), 172 Count: pointer.Of(1), 173 Tasks: []*api.Task{ 174 { 175 Name: "mock_systask", 176 Driver: "mock_driver", 177 Config: map[string]interface{}{ 178 "run_for": "10m", 179 }, 180 Resources: &api.Resources{ 181 CPU: pointer.Of(50), 182 MemoryMB: pointer.Of(50), 183 }, 184 }, 185 }, 186 }, 187 }, 188 } 189 190 _, _, err = client.Jobs().Register(sysjob, nil) 191 require.Nil(err) 192 193 var allocs []*api.Allocation 194 testutil.WaitForResult(func() (bool, error) { 195 allocs, _, err = client.Nodes().Allocations(nodeID, nil) 196 if err != nil { 197 return false, err 198 } 199 if len(allocs) != serviceCount+1 { 200 return false, fmt.Errorf("number of allocs %d != count (%d)", len(allocs), serviceCount+1) 201 } 202 for _, a := range allocs { 203 if a.ClientStatus != "running" { 204 return false, fmt.Errorf("alloc %q still not running: %s", a.ID, a.ClientStatus) 205 } 206 } 207 return true, nil 208 }, func(err error) { 209 t.Fatalf("err: %v", err) 210 }) 211 212 outBuf := bytes.NewBuffer(nil) 213 ui := &cli.BasicUi{ 214 Reader: bytes.NewReader(nil), 215 Writer: outBuf, 216 ErrorWriter: outBuf, 217 } 218 cmd := &NodeDrainCommand{Meta: Meta{Ui: ui}} 219 args := []string{"-address=" + url, "-self", "-enable", "-deadline", "1s", "-ignore-system"} 220 t.Logf("Running: %v", args) 221 require.Zero(cmd.Run(args)) 222 223 out := outBuf.String() 224 t.Logf("Output:\n%s", out) 225 226 // Unfortunately travis is too slow to reliably see the expected output. The 227 // monitor goroutines may start only after some or all the allocs have been 228 // migrated. 229 if !testutil.IsTravis() { 230 require.Contains(out, "Drain complete for node") 231 for _, a := range allocs { 232 if *a.Job.Type == "system" { 233 if strings.Contains(out, a.ID) { 234 t.Fatalf("output should not contain system alloc %q", a.ID) 235 } 236 continue 237 } 238 require.Contains(out, fmt.Sprintf("Alloc %q marked for migration", a.ID)) 239 require.Contains(out, fmt.Sprintf("Alloc %q draining", a.ID)) 240 } 241 242 expected := fmt.Sprintf("All allocations on node %q have stopped\n", nodeID) 243 if !strings.HasSuffix(out, expected) { 244 t.Fatalf("expected output to end with:\n%s", expected) 245 } 246 } 247 248 // Test -monitor flag 249 outBuf.Reset() 250 args = []string{"-address=" + url, "-self", "-monitor", "-ignore-system"} 251 t.Logf("Running: %v", args) 252 require.Zero(cmd.Run(args)) 253 254 out = outBuf.String() 255 t.Logf("Output:\n%s", out) 256 require.Contains(out, "No drain strategy set") 257 } 258 259 func TestNodeDrainCommand_Monitor_NoDrainStrategy(t *testing.T) { 260 ci.Parallel(t) 261 require := require.New(t) 262 server, client, url := testServer(t, true, func(c *agent.Config) { 263 c.NodeName = "drain_monitor_node2" 264 }) 265 defer server.Shutdown() 266 267 // Wait for a node to appear 268 testutil.WaitForResult(func() (bool, error) { 269 nodes, _, err := client.Nodes().List(nil) 270 if err != nil { 271 return false, err 272 } 273 if len(nodes) == 0 { 274 return false, fmt.Errorf("missing node") 275 } 276 return true, nil 277 }, func(err error) { 278 t.Fatalf("err: %s", err) 279 }) 280 281 // Test -monitor flag 282 outBuf := bytes.NewBuffer(nil) 283 ui := &cli.BasicUi{ 284 Reader: bytes.NewReader(nil), 285 Writer: outBuf, 286 ErrorWriter: outBuf, 287 } 288 cmd := &NodeDrainCommand{Meta: Meta{Ui: ui}} 289 args := []string{"-address=" + url, "-self", "-monitor", "-ignore-system"} 290 t.Logf("Running: %v", args) 291 if code := cmd.Run(args); code != 0 { 292 t.Fatalf("expected exit 0, got: %d\n%s", code, outBuf.String()) 293 } 294 295 out := outBuf.String() 296 t.Logf("Output:\n%s", out) 297 298 require.Contains(out, "No drain strategy set") 299 } 300 301 func TestNodeDrainCommand_Fails(t *testing.T) { 302 ci.Parallel(t) 303 srv, _, url := testServer(t, false, nil) 304 defer srv.Shutdown() 305 306 ui := cli.NewMockUi() 307 cmd := &NodeDrainCommand{Meta: Meta{Ui: ui}} 308 309 // Fails on misuse 310 if code := cmd.Run([]string{"some", "bad", "args"}); code != 1 { 311 t.Fatalf("expected exit code 1, got: %d", code) 312 } 313 if out := ui.ErrorWriter.String(); !strings.Contains(out, commandErrorText(cmd)) { 314 t.Fatalf("expected help output, got: %s", out) 315 } 316 ui.ErrorWriter.Reset() 317 318 // Fails on connection failure 319 if code := cmd.Run([]string{"-address=nope", "-enable", "12345678-abcd-efab-cdef-123456789abc"}); code != 1 { 320 t.Fatalf("expected exit code 1, got: %d", code) 321 } 322 if out := ui.ErrorWriter.String(); !strings.Contains(out, "Error toggling") { 323 t.Fatalf("expected failed toggle error, got: %s", out) 324 } 325 ui.ErrorWriter.Reset() 326 327 // Fails on nonexistent node 328 if code := cmd.Run([]string{"-address=" + url, "-enable", "12345678-abcd-efab-cdef-123456789abc"}); code != 1 { 329 t.Fatalf("expected exit 1, got: %d", code) 330 } 331 if out := ui.ErrorWriter.String(); !strings.Contains(out, "No node(s) with prefix or id") { 332 t.Fatalf("expected not exist error, got: %s", out) 333 } 334 ui.ErrorWriter.Reset() 335 336 // Fails if both enable and disable specified 337 if code := cmd.Run([]string{"-enable", "-disable", "12345678-abcd-efab-cdef-123456789abc"}); code != 1 { 338 t.Fatalf("expected exit 1, got: %d", code) 339 } 340 if out := ui.ErrorWriter.String(); !strings.Contains(out, commandErrorText(cmd)) { 341 t.Fatalf("expected help output, got: %s", out) 342 } 343 ui.ErrorWriter.Reset() 344 345 // Fails if neither enable or disable specified 346 if code := cmd.Run([]string{"12345678-abcd-efab-cdef-123456789abc"}); code != 1 { 347 t.Fatalf("expected exit 1, got: %d", code) 348 } 349 if out := ui.ErrorWriter.String(); !strings.Contains(out, commandErrorText(cmd)) { 350 t.Fatalf("expected help output, got: %s", out) 351 } 352 ui.ErrorWriter.Reset() 353 354 // Fail on identifier with too few characters 355 if code := cmd.Run([]string{"-address=" + url, "-enable", "1"}); code != 1 { 356 t.Fatalf("expected exit 1, got: %d", code) 357 } 358 if out := ui.ErrorWriter.String(); !strings.Contains(out, "must contain at least two characters.") { 359 t.Fatalf("expected too few characters error, got: %s", out) 360 } 361 ui.ErrorWriter.Reset() 362 363 // Identifiers with uneven length should produce a query result 364 if code := cmd.Run([]string{"-address=" + url, "-enable", "123"}); code != 1 { 365 t.Fatalf("expected exit 1, got: %d", code) 366 } 367 if out := ui.ErrorWriter.String(); !strings.Contains(out, "No node(s) with prefix or id") { 368 t.Fatalf("expected not exist error, got: %s", out) 369 } 370 ui.ErrorWriter.Reset() 371 372 // Fail on disable being used with drain strategy flags 373 for _, flag := range []string{"-force", "-no-deadline", "-ignore-system"} { 374 if code := cmd.Run([]string{"-address=" + url, "-disable", flag, "12345678-abcd-efab-cdef-123456789abc"}); code != 1 { 375 t.Fatalf("expected exit 1, got: %d", code) 376 } 377 if out := ui.ErrorWriter.String(); !strings.Contains(out, "combined with flags configuring drain strategy") { 378 t.Fatalf("got: %s", out) 379 } 380 ui.ErrorWriter.Reset() 381 } 382 383 // Fail on setting a deadline plus deadline modifying flags 384 for _, flag := range []string{"-force", "-no-deadline"} { 385 if code := cmd.Run([]string{"-address=" + url, "-enable", "-deadline=10s", flag, "12345678-abcd-efab-cdef-123456789abc"}); code != 1 { 386 t.Fatalf("expected exit 1, got: %d", code) 387 } 388 if out := ui.ErrorWriter.String(); !strings.Contains(out, "deadline can't be combined with") { 389 t.Fatalf("got: %s", out) 390 } 391 ui.ErrorWriter.Reset() 392 } 393 394 // Fail on setting a force and no deadline 395 if code := cmd.Run([]string{"-address=" + url, "-enable", "-force", "-no-deadline", "12345678-abcd-efab-cdef-123456789abc"}); code != 1 { 396 t.Fatalf("expected exit 1, got: %d", code) 397 } 398 if out := ui.ErrorWriter.String(); !strings.Contains(out, "mutually exclusive") { 399 t.Fatalf("got: %s", out) 400 } 401 ui.ErrorWriter.Reset() 402 403 // Fail on setting a bad deadline 404 for _, flag := range []string{"-deadline=0s", "-deadline=-1s"} { 405 if code := cmd.Run([]string{"-address=" + url, "-enable", flag, "12345678-abcd-efab-cdef-123456789abc"}); code != 1 { 406 t.Fatalf("expected exit 1, got: %d", code) 407 } 408 if out := ui.ErrorWriter.String(); !strings.Contains(out, "positive") { 409 t.Fatalf("got: %s", out) 410 } 411 ui.ErrorWriter.Reset() 412 } 413 } 414 415 func TestNodeDrainCommand_AutocompleteArgs(t *testing.T) { 416 ci.Parallel(t) 417 assert := assert.New(t) 418 419 srv, client, url := testServer(t, true, nil) 420 defer srv.Shutdown() 421 422 // Wait for a node to appear 423 var nodeID string 424 testutil.WaitForResult(func() (bool, error) { 425 nodes, _, err := client.Nodes().List(nil) 426 if err != nil { 427 return false, err 428 } 429 if len(nodes) == 0 { 430 return false, fmt.Errorf("missing node") 431 } 432 nodeID = nodes[0].ID 433 return true, nil 434 }, func(err error) { 435 t.Fatalf("err: %s", err) 436 }) 437 438 ui := cli.NewMockUi() 439 cmd := &NodeDrainCommand{Meta: Meta{Ui: ui, flagAddress: url}} 440 441 prefix := nodeID[:len(nodeID)-5] 442 args := complete.Args{Last: prefix} 443 predictor := cmd.AutocompleteArgs() 444 445 res := predictor.Predict(args) 446 assert.Equal(1, len(res)) 447 assert.Equal(nodeID, res[0]) 448 }