github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/command/alloc_status_test.go (about) 1 package command 2 3 import ( 4 "fmt" 5 "regexp" 6 "testing" 7 "time" 8 9 "github.com/hashicorp/nomad/ci" 10 "github.com/hashicorp/nomad/command/agent" 11 "github.com/hashicorp/nomad/helper/uuid" 12 "github.com/hashicorp/nomad/nomad/mock" 13 "github.com/hashicorp/nomad/nomad/structs" 14 "github.com/mitchellh/cli" 15 "github.com/posener/complete" 16 "github.com/shoenig/test/must" 17 ) 18 19 func TestAllocStatusCommand_Implements(t *testing.T) { 20 ci.Parallel(t) 21 var _ cli.Command = &AllocStatusCommand{} 22 } 23 24 func TestAllocStatusCommand_Fails(t *testing.T) { 25 ci.Parallel(t) 26 srv, _, url := testServer(t, false, nil) 27 defer stopTestAgent(srv) 28 29 ui := cli.NewMockUi() 30 cmd := &AllocStatusCommand{Meta: Meta{Ui: ui}} 31 32 // Fails on misuse 33 code := cmd.Run([]string{"some", "bad", "args"}) 34 must.One(t, code) 35 36 out := ui.ErrorWriter.String() 37 must.StrContains(t, out, commandErrorText(cmd)) 38 39 ui.ErrorWriter.Reset() 40 41 // Fails on connection failure 42 code = cmd.Run([]string{"-address=nope", "foobar"}) 43 must.One(t, code) 44 45 out = ui.ErrorWriter.String() 46 must.StrContains(t, out, "Error querying allocation") 47 48 ui.ErrorWriter.Reset() 49 50 // Fails on missing alloc 51 code = cmd.Run([]string{"-address=" + url, "26470238-5CF2-438F-8772-DC67CFB0705C"}) 52 must.One(t, code) 53 54 out = ui.ErrorWriter.String() 55 must.StrContains(t, out, "No allocation(s) with prefix or id") 56 57 ui.ErrorWriter.Reset() 58 59 // Fail on identifier with too few characters 60 code = cmd.Run([]string{"-address=" + url, "2"}) 61 must.One(t, code) 62 63 out = ui.ErrorWriter.String() 64 must.StrContains(t, out, "must contain at least two characters.") 65 66 ui.ErrorWriter.Reset() 67 68 // Identifiers with uneven length should produce a query result 69 code = cmd.Run([]string{"-address=" + url, "123"}) 70 must.One(t, code) 71 72 out = ui.ErrorWriter.String() 73 must.StrContains(t, out, "No allocation(s) with prefix or id") 74 75 ui.ErrorWriter.Reset() 76 77 // Failed on both -json and -t options are specified 78 code = cmd.Run([]string{"-address=" + url, "-json", "-t", "{{.ID}}"}) 79 must.One(t, code) 80 81 out = ui.ErrorWriter.String() 82 must.StrContains(t, out, "Both json and template formatting are not allowed") 83 } 84 85 func TestAllocStatusCommand_LifecycleInfo(t *testing.T) { 86 ci.Parallel(t) 87 88 srv, client, url := testServer(t, true, nil) 89 defer stopTestAgent(srv) 90 91 waitForNodes(t, client) 92 93 ui := cli.NewMockUi() 94 cmd := &AllocStatusCommand{Meta: Meta{Ui: ui}} 95 state := srv.Agent.Server().State() 96 97 a := mock.Alloc() 98 a.Metrics = &structs.AllocMetric{} 99 tg := a.Job.LookupTaskGroup(a.TaskGroup) 100 101 initTask := tg.Tasks[0].Copy() 102 initTask.Name = "init_task" 103 initTask.Lifecycle = &structs.TaskLifecycleConfig{ 104 Hook: "prestart", 105 } 106 107 prestartSidecarTask := tg.Tasks[0].Copy() 108 prestartSidecarTask.Name = "prestart_sidecar" 109 prestartSidecarTask.Lifecycle = &structs.TaskLifecycleConfig{ 110 Hook: "prestart", 111 Sidecar: true, 112 } 113 114 tg.Tasks = append(tg.Tasks, initTask, prestartSidecarTask) 115 a.TaskResources["init_task"] = a.TaskResources["web"] 116 a.TaskResources["prestart_sidecar"] = a.TaskResources["web"] 117 a.TaskStates = map[string]*structs.TaskState{ 118 "web": {State: "pending"}, 119 "init_task": {State: "running"}, 120 "prestart_sidecar": {State: "running"}, 121 } 122 123 must.NoError(t, state.UpsertAllocs(structs.MsgTypeTestSetup, 1000, []*structs.Allocation{a})) 124 125 code := cmd.Run([]string{"-address=" + url, a.ID}) 126 must.Zero(t, code) 127 128 out := ui.OutputWriter.String() 129 must.StrContains(t, out, `Task "init_task" (prestart) is "running"`) 130 must.StrContains(t, out, `Task "prestart_sidecar" (prestart sidecar) is "running"`) 131 must.StrContains(t, out, `Task "web" is "pending"`) 132 } 133 134 func TestAllocStatusCommand_Run(t *testing.T) { 135 ci.Parallel(t) 136 srv, client, url := testServer(t, true, nil) 137 defer stopTestAgent(srv) 138 139 waitForNodes(t, client) 140 141 ui := cli.NewMockUi() 142 cmd := &AllocStatusCommand{Meta: Meta{Ui: ui}} 143 144 jobID := "job1_sfx" 145 job1 := testJob(jobID) 146 resp, _, err := client.Jobs().Register(job1, nil) 147 must.NoError(t, err) 148 149 code := waitForSuccess(ui, client, fullId, t, resp.EvalID) 150 must.Zero(t, code) 151 152 // get an alloc id 153 allocID := "" 154 nodeName := "" 155 if allocs, _, err := client.Jobs().Allocations(jobID, false, nil); err == nil { 156 if len(allocs) > 0 { 157 allocID = allocs[0].ID 158 nodeName = allocs[0].NodeName 159 } 160 } 161 must.NotEq(t, "", allocID) 162 163 code = cmd.Run([]string{"-address=" + url, allocID}) 164 must.Zero(t, code) 165 166 out := ui.OutputWriter.String() 167 must.StrContains(t, out, "Created") 168 must.StrContains(t, out, "Modified") 169 170 nodeNameRegexpStr := fmt.Sprintf(`\nNode Name\s+= %s\n`, regexp.QuoteMeta(nodeName)) 171 must.RegexMatch(t, regexp.MustCompile(nodeNameRegexpStr), out) 172 173 ui.OutputWriter.Reset() 174 175 code = cmd.Run([]string{"-address=" + url, "-verbose", allocID}) 176 must.Zero(t, code) 177 178 out = ui.OutputWriter.String() 179 must.StrContains(t, out, allocID) 180 must.StrContains(t, out, "Created") 181 182 ui.OutputWriter.Reset() 183 184 // Try the query with an even prefix that includes the hyphen 185 code = cmd.Run([]string{"-address=" + url, allocID[:13]}) 186 must.Zero(t, code) 187 188 out = ui.OutputWriter.String() 189 must.StrContains(t, out, "Created") 190 ui.OutputWriter.Reset() 191 192 code = cmd.Run([]string{"-address=" + url, "-verbose", allocID}) 193 must.Zero(t, code) 194 195 out = ui.OutputWriter.String() 196 must.StrContains(t, out, allocID) 197 198 // make sure nsd checks status output is elided if none exist 199 must.StrNotContains(t, out, `Nomad Service Checks:`) 200 } 201 202 func TestAllocStatusCommand_RescheduleInfo(t *testing.T) { 203 ci.Parallel(t) 204 srv, client, url := testServer(t, true, nil) 205 defer stopTestAgent(srv) 206 207 waitForNodes(t, client) 208 209 ui := cli.NewMockUi() 210 cmd := &AllocStatusCommand{Meta: Meta{Ui: ui}} 211 // Test reschedule attempt info 212 state := srv.Agent.Server().State() 213 a := mock.Alloc() 214 a.Metrics = &structs.AllocMetric{} 215 nextAllocId := uuid.Generate() 216 a.NextAllocation = nextAllocId 217 a.RescheduleTracker = &structs.RescheduleTracker{ 218 Events: []*structs.RescheduleEvent{ 219 { 220 RescheduleTime: time.Now().Add(-2 * time.Minute).UTC().UnixNano(), 221 PrevAllocID: uuid.Generate(), 222 PrevNodeID: uuid.Generate(), 223 }, 224 }, 225 } 226 must.NoError(t, state.UpsertAllocs(structs.MsgTypeTestSetup, 1000, []*structs.Allocation{a})) 227 228 if code := cmd.Run([]string{"-address=" + url, a.ID}); code != 0 { 229 t.Fatalf("expected exit 0, got: %d", code) 230 } 231 out := ui.OutputWriter.String() 232 must.StrContains(t, out, "Replacement Alloc ID") 233 must.RegexMatch(t, regexp.MustCompile(".*Reschedule Attempts\\s*=\\s*1/2"), out) 234 } 235 236 func TestAllocStatusCommand_ScoreMetrics(t *testing.T) { 237 ci.Parallel(t) 238 srv, client, url := testServer(t, true, nil) 239 defer stopTestAgent(srv) 240 241 waitForNodes(t, client) 242 243 ui := cli.NewMockUi() 244 cmd := &AllocStatusCommand{Meta: Meta{Ui: ui}} 245 246 // Test node metrics 247 state := srv.Agent.Server().State() 248 a := mock.Alloc() 249 mockNode1 := mock.Node() 250 mockNode2 := mock.Node() 251 a.Metrics = &structs.AllocMetric{ 252 ScoreMetaData: []*structs.NodeScoreMeta{ 253 { 254 NodeID: mockNode1.ID, 255 Scores: map[string]float64{ 256 "binpack": 0.77, 257 "node-affinity": 0.5, 258 }, 259 }, 260 { 261 NodeID: mockNode2.ID, 262 Scores: map[string]float64{ 263 "binpack": 0.75, 264 "node-affinity": 0.33, 265 }, 266 }, 267 }, 268 } 269 must.NoError(t, state.UpsertAllocs(structs.MsgTypeTestSetup, 1000, []*structs.Allocation{a})) 270 271 code := cmd.Run([]string{"-address=" + url, "-verbose", a.ID}) 272 must.Zero(t, code) 273 274 out := ui.OutputWriter.String() 275 must.StrContains(t, out, "Placement Metrics") 276 must.StrContains(t, out, mockNode1.ID) 277 must.StrContains(t, out, mockNode2.ID) 278 279 // assert we sort headers alphabetically 280 must.StrContains(t, out, "binpack node-affinity") 281 must.StrContains(t, out, "final score") 282 } 283 284 func TestAllocStatusCommand_AutocompleteArgs(t *testing.T) { 285 ci.Parallel(t) 286 287 srv, _, url := testServer(t, true, nil) 288 defer stopTestAgent(srv) 289 290 ui := cli.NewMockUi() 291 cmd := &AllocStatusCommand{Meta: Meta{Ui: ui, flagAddress: url}} 292 293 // Create a fake alloc 294 state := srv.Agent.Server().State() 295 a := mock.Alloc() 296 must.NoError(t, state.UpsertAllocs(structs.MsgTypeTestSetup, 1000, []*structs.Allocation{a})) 297 298 prefix := a.ID[:5] 299 args := complete.Args{Last: prefix} 300 predictor := cmd.AutocompleteArgs() 301 302 res := predictor.Predict(args) 303 must.Len(t, 1, res) 304 must.Eq(t, a.ID, res[0]) 305 } 306 307 func TestAllocStatusCommand_HostVolumes(t *testing.T) { 308 ci.Parallel(t) 309 // We have to create a tempdir for the host volume even though we're 310 // not going to use it b/c the server validates the config on startup 311 tmpDir := t.TempDir() 312 313 vol0 := uuid.Generate() 314 srv, _, url := testServer(t, true, func(c *agent.Config) { 315 c.Client.HostVolumes = []*structs.ClientHostVolumeConfig{ 316 { 317 Name: vol0, 318 Path: tmpDir, 319 ReadOnly: false, 320 }, 321 } 322 }) 323 defer stopTestAgent(srv) 324 325 state := srv.Agent.Server().State() 326 327 // Upsert the job and alloc 328 node := mock.Node() 329 alloc := mock.Alloc() 330 alloc.Metrics = &structs.AllocMetric{} 331 alloc.NodeID = node.ID 332 job := alloc.Job 333 job.TaskGroups[0].Volumes = map[string]*structs.VolumeRequest{ 334 vol0: { 335 Name: vol0, 336 Type: structs.VolumeTypeHost, 337 Source: tmpDir, 338 }, 339 } 340 job.TaskGroups[0].Tasks[0].VolumeMounts = []*structs.VolumeMount{ 341 { 342 Volume: vol0, 343 Destination: "/var/www", 344 ReadOnly: true, 345 PropagationMode: "private", 346 }, 347 } 348 // fakes the placement enough so that we have something to iterate 349 // on in 'nomad alloc status' 350 alloc.TaskStates = map[string]*structs.TaskState{ 351 "web": { 352 Events: []*structs.TaskEvent{ 353 structs.NewTaskEvent("test event").SetMessage("test msg"), 354 }, 355 }, 356 } 357 summary := mock.JobSummary(alloc.JobID) 358 must.NoError(t, state.UpsertJobSummary(1004, summary)) 359 must.NoError(t, state.UpsertAllocs(structs.MsgTypeTestSetup, 1005, []*structs.Allocation{alloc})) 360 361 ui := cli.NewMockUi() 362 cmd := &AllocStatusCommand{Meta: Meta{Ui: ui}} 363 code := cmd.Run([]string{"-address=" + url, "-verbose", alloc.ID}) 364 must.Zero(t, code) 365 366 out := ui.OutputWriter.String() 367 must.StrContains(t, out, "Host Volumes") 368 must.StrContains(t, out, fmt.Sprintf("%s true", vol0)) 369 must.StrNotContains(t, out, "CSI Volumes") 370 } 371 372 func TestAllocStatusCommand_CSIVolumes(t *testing.T) { 373 ci.Parallel(t) 374 srv, _, url := testServer(t, true, nil) 375 defer stopTestAgent(srv) 376 377 state := srv.Agent.Server().State() 378 379 // Upsert the node, plugin, and volume 380 vol0 := uuid.Generate() 381 node := mock.Node() 382 node.CSINodePlugins = map[string]*structs.CSIInfo{ 383 "minnie": { 384 PluginID: "minnie", 385 Healthy: true, 386 NodeInfo: &structs.CSINodeInfo{}, 387 }, 388 } 389 err := state.UpsertNode(structs.MsgTypeTestSetup, 1001, node) 390 must.NoError(t, err) 391 392 vols := []*structs.CSIVolume{{ 393 ID: vol0, 394 Namespace: structs.DefaultNamespace, 395 PluginID: "minnie", 396 AccessMode: structs.CSIVolumeAccessModeMultiNodeSingleWriter, 397 AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem, 398 Topologies: []*structs.CSITopology{{ 399 Segments: map[string]string{"foo": "bar"}, 400 }}, 401 }} 402 err = state.UpsertCSIVolume(1002, vols) 403 must.NoError(t, err) 404 405 // Upsert the job and alloc 406 alloc := mock.Alloc() 407 alloc.Metrics = &structs.AllocMetric{} 408 alloc.NodeID = node.ID 409 job := alloc.Job 410 job.TaskGroups[0].Volumes = map[string]*structs.VolumeRequest{ 411 vol0: { 412 Name: vol0, 413 Type: structs.VolumeTypeCSI, 414 Source: vol0, 415 }, 416 } 417 job.TaskGroups[0].Tasks[0].VolumeMounts = []*structs.VolumeMount{ 418 { 419 Volume: vol0, 420 Destination: "/var/www", 421 ReadOnly: true, 422 PropagationMode: "private", 423 }, 424 } 425 // if we don't set a task state, there's nothing to iterate on alloc status 426 alloc.TaskStates = map[string]*structs.TaskState{ 427 "web": { 428 Events: []*structs.TaskEvent{ 429 structs.NewTaskEvent("test event").SetMessage("test msg"), 430 }, 431 }, 432 } 433 summary := mock.JobSummary(alloc.JobID) 434 must.NoError(t, state.UpsertJobSummary(1004, summary)) 435 must.NoError(t, state.UpsertAllocs(structs.MsgTypeTestSetup, 1005, []*structs.Allocation{alloc})) 436 437 ui := cli.NewMockUi() 438 cmd := &AllocStatusCommand{Meta: Meta{Ui: ui}} 439 if code := cmd.Run([]string{"-address=" + url, "-verbose", alloc.ID}); code != 0 { 440 t.Fatalf("expected exit 0, got: %d", code) 441 } 442 out := ui.OutputWriter.String() 443 must.StrContains(t, out, "CSI Volumes") 444 must.StrContains(t, out, fmt.Sprintf("%s minnie", vol0)) 445 must.StrNotContains(t, out, "Host Volumes") 446 } 447 448 func TestAllocStatusCommand_NSD_Checks(t *testing.T) { 449 ci.Parallel(t) 450 srv, client, url := testServer(t, true, nil) 451 defer stopTestAgent(srv) 452 453 // wait for nodes 454 waitForNodes(t, client) 455 456 jobID := "job1_checks" 457 job1 := testNomadServiceJob(jobID) 458 459 resp, _, err := client.Jobs().Register(job1, nil) 460 must.NoError(t, err) 461 462 // wait for registration success 463 ui := cli.NewMockUi() 464 code := waitForSuccess(ui, client, fullId, t, resp.EvalID) 465 must.Zero(t, code) 466 467 // Get an alloc id 468 allocID := getAllocFromJob(t, client, jobID) 469 470 // wait for the check to be marked failure 471 waitForCheckStatus(t, client, allocID, "failure") 472 473 // Run command 474 cmd := &AllocStatusCommand{Meta: Meta{Ui: ui, flagAddress: url}} 475 code = cmd.Run([]string{"-address=" + url, allocID}) 476 must.Zero(t, code) 477 478 // check output 479 out := ui.OutputWriter.String() 480 must.StrContains(t, out, `Nomad Service Checks:`) 481 must.RegexMatch(t, regexp.MustCompile(`Service\s+Task\s+Name\s+Mode\s+Status`), out) 482 must.RegexMatch(t, regexp.MustCompile(`service1\s+\(group\)\s+check1\s+healthiness\s+(pending|failure)`), out) 483 }