github.com/emate/nomad@v0.8.2-wo-binpacking/command/node_drain_test.go (about) 1 package command 2 3 import ( 4 "bytes" 5 "fmt" 6 "strings" 7 "testing" 8 "time" 9 10 "github.com/hashicorp/nomad/api" 11 "github.com/hashicorp/nomad/command/agent" 12 "github.com/hashicorp/nomad/helper" 13 "github.com/hashicorp/nomad/testutil" 14 "github.com/mitchellh/cli" 15 "github.com/posener/complete" 16 "github.com/stretchr/testify/assert" 17 "github.com/stretchr/testify/require" 18 ) 19 20 func TestNodeDrainCommand_Implements(t *testing.T) { 21 t.Parallel() 22 var _ cli.Command = &NodeDrainCommand{} 23 } 24 25 func TestNodeDrainCommand_Detach(t *testing.T) { 26 t.Parallel() 27 require := require.New(t) 28 server, client, url := testServer(t, true, func(c *agent.Config) { 29 c.NodeName = "drain_detach_node" 30 }) 31 defer server.Shutdown() 32 33 // Wait for a node to appear 34 var nodeID string 35 testutil.WaitForResult(func() (bool, error) { 36 nodes, _, err := client.Nodes().List(nil) 37 if err != nil { 38 return false, err 39 } 40 if len(nodes) == 0 { 41 return false, fmt.Errorf("missing node") 42 } 43 nodeID = nodes[0].ID 44 return true, nil 45 }, func(err error) { 46 t.Fatalf("err: %s", err) 47 }) 48 49 // Register a job to create an alloc to drain that will block draining 50 job := &api.Job{ 51 ID: helper.StringToPtr("mock_service"), 52 Name: helper.StringToPtr("mock_service"), 53 Datacenters: []string{"dc1"}, 54 TaskGroups: []*api.TaskGroup{ 55 { 56 Name: helper.StringToPtr("mock_group"), 57 Tasks: []*api.Task{ 58 { 59 Name: "mock_task", 60 Driver: "mock_driver", 61 Config: map[string]interface{}{ 62 "run_for": "10m", 63 "exit_after": "10m", 64 }, 65 }, 66 }, 67 }, 68 }, 69 } 70 71 _, _, err := client.Jobs().Register(job, nil) 72 require.Nil(err) 73 74 testutil.WaitForResult(func() (bool, error) { 75 allocs, _, err := client.Nodes().Allocations(nodeID, nil) 76 if err != nil { 77 return false, err 78 } 79 return len(allocs) > 0, fmt.Errorf("no allocs") 80 }, func(err error) { 81 t.Fatalf("err: %v", err) 82 }) 83 84 ui := new(cli.MockUi) 85 cmd := &NodeDrainCommand{Meta: Meta{Ui: ui}} 86 if code := cmd.Run([]string{"-address=" + url, "-self", "-enable", "-detach"}); code != 0 { 87 t.Fatalf("expected exit 0, got: %d", code) 88 } 89 90 out := ui.OutputWriter.String() 91 expected := "drain strategy set" 92 require.Contains(out, expected) 93 94 node, _, err := client.Nodes().Info(nodeID, nil) 95 require.Nil(err) 96 require.NotNil(node.DrainStrategy) 97 } 98 99 func TestNodeDrainCommand_Monitor(t *testing.T) { 100 t.Parallel() 101 require := require.New(t) 102 server, client, url := testServer(t, true, func(c *agent.Config) { 103 c.NodeName = "drain_monitor_node" 104 }) 105 defer server.Shutdown() 106 107 // Wait for a node to appear 108 var nodeID string 109 testutil.WaitForResult(func() (bool, error) { 110 nodes, _, err := client.Nodes().List(nil) 111 if err != nil { 112 return false, err 113 } 114 if len(nodes) == 0 { 115 return false, fmt.Errorf("missing node") 116 } 117 nodeID = nodes[0].ID 118 return true, nil 119 }, func(err error) { 120 t.Fatalf("err: %s", err) 121 }) 122 123 // Register a service job to create allocs to drain 124 serviceCount := 3 125 job := &api.Job{ 126 ID: helper.StringToPtr("mock_service"), 127 Name: helper.StringToPtr("mock_service"), 128 Datacenters: []string{"dc1"}, 129 Type: helper.StringToPtr("service"), 130 TaskGroups: []*api.TaskGroup{ 131 { 132 Name: helper.StringToPtr("mock_group"), 133 Count: &serviceCount, 134 Migrate: &api.MigrateStrategy{ 135 MaxParallel: helper.IntToPtr(1), 136 HealthCheck: helper.StringToPtr("task_states"), 137 MinHealthyTime: helper.TimeToPtr(10 * time.Millisecond), 138 HealthyDeadline: helper.TimeToPtr(5 * time.Minute), 139 }, 140 Tasks: []*api.Task{ 141 { 142 Name: "mock_task", 143 Driver: "mock_driver", 144 Config: map[string]interface{}{ 145 "run_for": "10m", 146 }, 147 Resources: &api.Resources{ 148 CPU: helper.IntToPtr(50), 149 MemoryMB: helper.IntToPtr(50), 150 }, 151 }, 152 }, 153 }, 154 }, 155 } 156 157 _, _, err := client.Jobs().Register(job, nil) 158 require.Nil(err) 159 160 // Register a system job to ensure it is ignored during draining 161 sysjob := &api.Job{ 162 ID: helper.StringToPtr("mock_system"), 163 Name: helper.StringToPtr("mock_system"), 164 Datacenters: []string{"dc1"}, 165 Type: helper.StringToPtr("system"), 166 TaskGroups: []*api.TaskGroup{ 167 { 168 Name: helper.StringToPtr("mock_sysgroup"), 169 Count: helper.IntToPtr(1), 170 Tasks: []*api.Task{ 171 { 172 Name: "mock_systask", 173 Driver: "mock_driver", 174 Config: map[string]interface{}{ 175 "run_for": "10m", 176 }, 177 Resources: &api.Resources{ 178 CPU: helper.IntToPtr(50), 179 MemoryMB: helper.IntToPtr(50), 180 }, 181 }, 182 }, 183 }, 184 }, 185 } 186 187 _, _, err = client.Jobs().Register(sysjob, nil) 188 require.Nil(err) 189 190 var allocs []*api.Allocation 191 testutil.WaitForResult(func() (bool, error) { 192 allocs, _, err = client.Nodes().Allocations(nodeID, nil) 193 if err != nil { 194 return false, err 195 } 196 if len(allocs) != serviceCount+1 { 197 return false, fmt.Errorf("number of allocs %d != count (%d)", len(allocs), serviceCount+1) 198 } 199 for _, a := range allocs { 200 if a.ClientStatus != "running" { 201 return false, fmt.Errorf("alloc %q still not running: %s", a.ID, a.ClientStatus) 202 } 203 } 204 return true, nil 205 }, func(err error) { 206 t.Fatalf("err: %v", err) 207 }) 208 209 outBuf := bytes.NewBuffer(nil) 210 ui := &cli.BasicUi{ 211 Reader: bytes.NewReader(nil), 212 Writer: outBuf, 213 ErrorWriter: outBuf, 214 } 215 cmd := &NodeDrainCommand{Meta: Meta{Ui: ui}} 216 args := []string{"-address=" + url, "-self", "-enable", "-deadline", "1s", "-ignore-system"} 217 t.Logf("Running: %v", args) 218 if code := cmd.Run(args); code != 0 { 219 t.Fatalf("expected exit 0, got: %d\n%s", code, outBuf.String()) 220 } 221 222 out := outBuf.String() 223 t.Logf("Output:\n%s", out) 224 225 require.Contains(out, "marked all allocations for migration") 226 for _, a := range allocs { 227 if *a.Job.Type == "system" { 228 if strings.Contains(out, a.ID) { 229 t.Fatalf("output should not contain system alloc %q", a.ID) 230 } 231 continue 232 } 233 require.Contains(out, fmt.Sprintf("Alloc %q marked for migration", a.ID)) 234 require.Contains(out, fmt.Sprintf("Alloc %q draining", a.ID)) 235 } 236 expected := fmt.Sprintf("All allocations on node %q have stopped.\n", nodeID) 237 if !strings.HasSuffix(out, expected) { 238 t.Fatalf("expected output to end with:\n%s", expected) 239 } 240 } 241 242 func TestNodeDrainCommand_Fails(t *testing.T) { 243 t.Parallel() 244 srv, _, url := testServer(t, false, nil) 245 defer srv.Shutdown() 246 247 ui := new(cli.MockUi) 248 cmd := &NodeDrainCommand{Meta: Meta{Ui: ui}} 249 250 // Fails on misuse 251 if code := cmd.Run([]string{"some", "bad", "args"}); code != 1 { 252 t.Fatalf("expected exit code 1, got: %d", code) 253 } 254 if out := ui.ErrorWriter.String(); !strings.Contains(out, commandErrorText(cmd)) { 255 t.Fatalf("expected help output, got: %s", out) 256 } 257 ui.ErrorWriter.Reset() 258 259 // Fails on connection failure 260 if code := cmd.Run([]string{"-address=nope", "-enable", "12345678-abcd-efab-cdef-123456789abc"}); code != 1 { 261 t.Fatalf("expected exit code 1, got: %d", code) 262 } 263 if out := ui.ErrorWriter.String(); !strings.Contains(out, "Error toggling") { 264 t.Fatalf("expected failed toggle error, got: %s", out) 265 } 266 ui.ErrorWriter.Reset() 267 268 // Fails on nonexistent node 269 if code := cmd.Run([]string{"-address=" + url, "-enable", "12345678-abcd-efab-cdef-123456789abc"}); code != 1 { 270 t.Fatalf("expected exit 1, got: %d", code) 271 } 272 if out := ui.ErrorWriter.String(); !strings.Contains(out, "No node(s) with prefix or id") { 273 t.Fatalf("expected not exist error, got: %s", out) 274 } 275 ui.ErrorWriter.Reset() 276 277 // Fails if both enable and disable specified 278 if code := cmd.Run([]string{"-enable", "-disable", "12345678-abcd-efab-cdef-123456789abc"}); code != 1 { 279 t.Fatalf("expected exit 1, got: %d", code) 280 } 281 if out := ui.ErrorWriter.String(); !strings.Contains(out, commandErrorText(cmd)) { 282 t.Fatalf("expected help output, got: %s", out) 283 } 284 ui.ErrorWriter.Reset() 285 286 // Fails if neither enable or disable specified 287 if code := cmd.Run([]string{"12345678-abcd-efab-cdef-123456789abc"}); code != 1 { 288 t.Fatalf("expected exit 1, got: %d", code) 289 } 290 if out := ui.ErrorWriter.String(); !strings.Contains(out, commandErrorText(cmd)) { 291 t.Fatalf("expected help output, got: %s", out) 292 } 293 ui.ErrorWriter.Reset() 294 295 // Fail on identifier with too few characters 296 if code := cmd.Run([]string{"-address=" + url, "-enable", "1"}); code != 1 { 297 t.Fatalf("expected exit 1, got: %d", code) 298 } 299 if out := ui.ErrorWriter.String(); !strings.Contains(out, "must contain at least two characters.") { 300 t.Fatalf("expected too few characters error, got: %s", out) 301 } 302 ui.ErrorWriter.Reset() 303 304 // Identifiers with uneven length should produce a query result 305 if code := cmd.Run([]string{"-address=" + url, "-enable", "123"}); code != 1 { 306 t.Fatalf("expected exit 1, got: %d", code) 307 } 308 if out := ui.ErrorWriter.String(); !strings.Contains(out, "No node(s) with prefix or id") { 309 t.Fatalf("expected not exist error, got: %s", out) 310 } 311 ui.ErrorWriter.Reset() 312 313 // Fail on disable being used with drain strategy flags 314 for _, flag := range []string{"-force", "-no-deadline", "-ignore-system"} { 315 if code := cmd.Run([]string{"-address=" + url, "-disable", flag, "12345678-abcd-efab-cdef-123456789abc"}); code != 1 { 316 t.Fatalf("expected exit 1, got: %d", code) 317 } 318 if out := ui.ErrorWriter.String(); !strings.Contains(out, "combined with flags configuring drain strategy") { 319 t.Fatalf("got: %s", out) 320 } 321 ui.ErrorWriter.Reset() 322 } 323 324 // Fail on setting a deadline plus deadline modifying flags 325 for _, flag := range []string{"-force", "-no-deadline"} { 326 if code := cmd.Run([]string{"-address=" + url, "-enable", "-deadline=10s", flag, "12345678-abcd-efab-cdef-123456789abc"}); code != 1 { 327 t.Fatalf("expected exit 1, got: %d", code) 328 } 329 if out := ui.ErrorWriter.String(); !strings.Contains(out, "deadline can't be combined with") { 330 t.Fatalf("got: %s", out) 331 } 332 ui.ErrorWriter.Reset() 333 } 334 335 // Fail on setting a force and no deadline 336 if code := cmd.Run([]string{"-address=" + url, "-enable", "-force", "-no-deadline", "12345678-abcd-efab-cdef-123456789abc"}); code != 1 { 337 t.Fatalf("expected exit 1, got: %d", code) 338 } 339 if out := ui.ErrorWriter.String(); !strings.Contains(out, "mutually exclusive") { 340 t.Fatalf("got: %s", out) 341 } 342 ui.ErrorWriter.Reset() 343 344 // Fail on setting a bad deadline 345 for _, flag := range []string{"-deadline=0s", "-deadline=-1s"} { 346 if code := cmd.Run([]string{"-address=" + url, "-enable", flag, "12345678-abcd-efab-cdef-123456789abc"}); code != 1 { 347 t.Fatalf("expected exit 1, got: %d", code) 348 } 349 if out := ui.ErrorWriter.String(); !strings.Contains(out, "positive") { 350 t.Fatalf("got: %s", out) 351 } 352 ui.ErrorWriter.Reset() 353 } 354 } 355 356 func TestNodeDrainCommand_AutocompleteArgs(t *testing.T) { 357 assert := assert.New(t) 358 t.Parallel() 359 360 srv, client, url := testServer(t, true, nil) 361 defer srv.Shutdown() 362 363 // Wait for a node to appear 364 var nodeID string 365 testutil.WaitForResult(func() (bool, error) { 366 nodes, _, err := client.Nodes().List(nil) 367 if err != nil { 368 return false, err 369 } 370 if len(nodes) == 0 { 371 return false, fmt.Errorf("missing node") 372 } 373 nodeID = nodes[0].ID 374 return true, nil 375 }, func(err error) { 376 t.Fatalf("err: %s", err) 377 }) 378 379 ui := new(cli.MockUi) 380 cmd := &NodeDrainCommand{Meta: Meta{Ui: ui, flagAddress: url}} 381 382 prefix := nodeID[:len(nodeID)-5] 383 args := complete.Args{Last: prefix} 384 predictor := cmd.AutocompleteArgs() 385 386 res := predictor.Predict(args) 387 assert.Equal(1, len(res)) 388 assert.Equal(nodeID, res[0]) 389 }