github.com/hernad/nomad@v1.6.112/nomad/alloc_endpoint_test.go (about) 1 // Copyright (c) HashiCorp, Inc. 2 // SPDX-License-Identifier: MPL-2.0 3 4 package nomad 5 6 import ( 7 "reflect" 8 "testing" 9 "time" 10 11 msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc" 12 "github.com/hernad/nomad/acl" 13 "github.com/hernad/nomad/ci" 14 "github.com/hernad/nomad/helper/pointer" 15 "github.com/hernad/nomad/helper/uuid" 16 "github.com/hernad/nomad/nomad/mock" 17 "github.com/hernad/nomad/nomad/structs" 18 "github.com/hernad/nomad/testutil" 19 "github.com/stretchr/testify/assert" 20 "github.com/stretchr/testify/require" 21 ) 22 23 func TestAllocEndpoint_List(t *testing.T) { 24 ci.Parallel(t) 25 26 s1, cleanupS1 := TestServer(t, nil) 27 defer cleanupS1() 28 29 codec := rpcClient(t, s1) 30 testutil.WaitForLeader(t, s1.RPC) 31 32 // Create the register request 33 alloc := mock.Alloc() 34 summary := mock.JobSummary(alloc.JobID) 35 state := s1.fsm.State() 36 37 if err := state.UpsertJobSummary(999, summary); err != nil { 38 t.Fatalf("err: %v", err) 39 } 40 if err := state.UpsertAllocs(structs.MsgTypeTestSetup, 1000, []*structs.Allocation{alloc}); err != nil { 41 t.Fatalf("err: %v", err) 42 } 43 44 // Lookup the allocations 45 get := &structs.AllocListRequest{ 46 QueryOptions: structs.QueryOptions{ 47 Region: "global", 48 Namespace: structs.DefaultNamespace, 49 }, 50 } 51 var resp structs.AllocListResponse 52 if err := msgpackrpc.CallWithCodec(codec, "Alloc.List", get, &resp); err != nil { 53 t.Fatalf("err: %v", err) 54 } 55 if resp.Index != 1000 { 56 t.Fatalf("Bad index: %d %d", resp.Index, 1000) 57 } 58 59 if len(resp.Allocations) != 1 { 60 t.Fatalf("bad: %#v", resp.Allocations) 61 } 62 if resp.Allocations[0].ID != alloc.ID { 63 t.Fatalf("bad: %#v", resp.Allocations[0]) 64 } 65 66 // Lookup the allocations by prefix 67 get = &structs.AllocListRequest{ 68 QueryOptions: structs.QueryOptions{ 69 Region: "global", 70 Namespace: structs.DefaultNamespace, 71 Prefix: alloc.ID[:4], 72 }, 73 } 74 75 var resp2 structs.AllocListResponse 76 require.NoError(t, msgpackrpc.CallWithCodec(codec, "Alloc.List", get, &resp2)) 77 require.Equal(t, uint64(1000), resp2.Index) 78 require.Len(t, resp2.Allocations, 1) 79 require.Equal(t, alloc.ID, resp2.Allocations[0].ID) 80 81 // Lookup allocations with a filter 82 get = &structs.AllocListRequest{ 83 QueryOptions: structs.QueryOptions{ 84 Region: "global", 85 Namespace: structs.DefaultNamespace, 86 Filter: "TaskGroup == web", 87 }, 88 } 89 90 var resp3 structs.AllocListResponse 91 require.NoError(t, msgpackrpc.CallWithCodec(codec, "Alloc.List", get, &resp3)) 92 require.Equal(t, uint64(1000), resp3.Index) 93 require.Len(t, resp3.Allocations, 1) 94 require.Equal(t, alloc.ID, resp3.Allocations[0].ID) 95 } 96 97 func TestAllocEndpoint_List_PaginationFiltering(t *testing.T) { 98 ci.Parallel(t) 99 s1, _, cleanupS1 := TestACLServer(t, nil) 100 defer cleanupS1() 101 codec := rpcClient(t, s1) 102 testutil.WaitForLeader(t, s1.RPC) 103 104 // create a set of allocs and field values to filter on. these are in the order 105 // that the state store will return them from the iterator (sorted by create 106 // index), for ease of writing tests. 107 mocks := []struct { 108 ids []string 109 namespace string 110 group string 111 }{ 112 {ids: []string{"aaaa1111-3350-4b4b-d185-0e1992ed43e9"}}, // 0 113 {ids: []string{"aaaaaa22-3350-4b4b-d185-0e1992ed43e9"}}, // 1 114 {ids: []string{"aaaaaa33-3350-4b4b-d185-0e1992ed43e9"}, namespace: "non-default"}, // 2 115 {ids: []string{"aaaaaaaa-3350-4b4b-d185-0e1992ed43e9"}, group: "bar"}, // 3 116 {ids: []string{"aaaaaabb-3350-4b4b-d185-0e1992ed43e9"}, group: "goo"}, // 4 117 {ids: []string{"aaaaaacc-3350-4b4b-d185-0e1992ed43e9"}}, // 5 118 {ids: []string{"aaaaaadd-3350-4b4b-d185-0e1992ed43e9"}, group: "bar"}, // 6 119 {ids: []string{"aaaaaaee-3350-4b4b-d185-0e1992ed43e9"}, group: "goo"}, // 7 120 {ids: []string{"aaaaaaff-3350-4b4b-d185-0e1992ed43e9"}, group: "bar"}, // 8 121 {ids: []string{"00000111-3350-4b4b-d185-0e1992ed43e9"}}, // 9 122 {ids: []string{ // 10 123 "00000222-3350-4b4b-d185-0e1992ed43e9", 124 "00000333-3350-4b4b-d185-0e1992ed43e9", 125 }}, 126 {}, // 11, index missing 127 {ids: []string{"bbbb1111-3350-4b4b-d185-0e1992ed43e9"}}, // 12 128 } 129 130 state := s1.fsm.State() 131 132 require.NoError(t, state.UpsertNamespaces(1099, []*structs.Namespace{ 133 {Name: "non-default"}, 134 })) 135 136 var allocs []*structs.Allocation 137 for i, m := range mocks { 138 allocsInTx := []*structs.Allocation{} 139 for _, id := range m.ids { 140 alloc := mock.Alloc() 141 alloc.ID = id 142 if m.namespace != "" { 143 alloc.Namespace = m.namespace 144 } 145 if m.group != "" { 146 alloc.TaskGroup = m.group 147 } 148 allocs = append(allocs, alloc) 149 allocsInTx = append(allocsInTx, alloc) 150 } 151 // other fields 152 index := 1000 + uint64(i) 153 require.NoError(t, state.UpsertAllocs(structs.MsgTypeTestSetup, index, allocsInTx)) 154 } 155 156 aclToken := mock.CreatePolicyAndToken(t, 157 state, 1100, "test-valid-read", 158 mock.NamespacePolicy("*", "read", nil), 159 ).SecretID 160 161 cases := []struct { 162 name string 163 namespace string 164 prefix string 165 nextToken string 166 pageSize int32 167 filter string 168 expIDs []string 169 expNextToken string 170 expErr string 171 }{ 172 { 173 name: "test01 size-2 page-1 ns-default", 174 pageSize: 2, 175 expIDs: []string{ // first two items 176 "aaaa1111-3350-4b4b-d185-0e1992ed43e9", 177 "aaaaaa22-3350-4b4b-d185-0e1992ed43e9", 178 }, 179 expNextToken: "1003.aaaaaaaa-3350-4b4b-d185-0e1992ed43e9", // next one in default ns 180 }, 181 { 182 name: "test02 size-2 page-1 ns-default with-prefix", 183 prefix: "aaaa", 184 pageSize: 2, 185 expIDs: []string{ 186 "aaaa1111-3350-4b4b-d185-0e1992ed43e9", 187 "aaaaaa22-3350-4b4b-d185-0e1992ed43e9", 188 }, 189 expNextToken: "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9", 190 }, 191 { 192 name: "test03 size-2 page-2 ns-default", 193 pageSize: 2, 194 nextToken: "1003.aaaaaaaa-3350-4b4b-d185-0e1992ed43e9", 195 expNextToken: "1005.aaaaaacc-3350-4b4b-d185-0e1992ed43e9", 196 expIDs: []string{ 197 "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9", 198 "aaaaaabb-3350-4b4b-d185-0e1992ed43e9", 199 }, 200 }, 201 { 202 name: "test04 size-2 page-2 ns-default with prefix", 203 prefix: "aaaa", 204 pageSize: 2, 205 nextToken: "aaaaaabb-3350-4b4b-d185-0e1992ed43e9", 206 expNextToken: "aaaaaadd-3350-4b4b-d185-0e1992ed43e9", 207 expIDs: []string{ 208 "aaaaaabb-3350-4b4b-d185-0e1992ed43e9", 209 "aaaaaacc-3350-4b4b-d185-0e1992ed43e9", 210 }, 211 }, 212 { 213 name: "test05 go-bexpr filter", 214 filter: `TaskGroup == "goo"`, 215 nextToken: "", 216 expIDs: []string{ 217 "aaaaaabb-3350-4b4b-d185-0e1992ed43e9", 218 "aaaaaaee-3350-4b4b-d185-0e1992ed43e9", 219 }, 220 }, 221 { 222 name: "test06 go-bexpr filter with pagination", 223 filter: `TaskGroup == "bar"`, 224 pageSize: 2, 225 expNextToken: "1008.aaaaaaff-3350-4b4b-d185-0e1992ed43e9", 226 expIDs: []string{ 227 "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9", 228 "aaaaaadd-3350-4b4b-d185-0e1992ed43e9", 229 }, 230 }, 231 { 232 name: "test07 go-bexpr filter namespace", 233 namespace: "non-default", 234 filter: `ID contains "aaa"`, 235 expIDs: []string{ 236 "aaaaaa33-3350-4b4b-d185-0e1992ed43e9", 237 }, 238 }, 239 { 240 name: "test08 go-bexpr wrong namespace", 241 namespace: "default", 242 filter: `Namespace == "non-default"`, 243 expIDs: []string(nil), 244 }, 245 { 246 name: "test09 go-bexpr invalid expression", 247 filter: `NotValid`, 248 expErr: "failed to read filter expression", 249 }, 250 { 251 name: "test10 go-bexpr invalid field", 252 filter: `InvalidField == "value"`, 253 expErr: "error finding value in datum", 254 }, 255 { 256 name: "test11 non-lexicographic order", 257 pageSize: 1, 258 nextToken: "1009.00000111-3350-4b4b-d185-0e1992ed43e9", 259 expNextToken: "1010.00000222-3350-4b4b-d185-0e1992ed43e9", 260 expIDs: []string{ 261 "00000111-3350-4b4b-d185-0e1992ed43e9", 262 }, 263 }, 264 { 265 name: "test12 same index", 266 pageSize: 1, 267 nextToken: "1010.00000222-3350-4b4b-d185-0e1992ed43e9", 268 expNextToken: "1010.00000333-3350-4b4b-d185-0e1992ed43e9", 269 expIDs: []string{ 270 "00000222-3350-4b4b-d185-0e1992ed43e9", 271 }, 272 }, 273 { 274 name: "test13 missing index", 275 pageSize: 1, 276 nextToken: "1011.e9522802-0cd8-4b1d-9c9e-ab3d97938371", 277 expIDs: []string{ 278 "bbbb1111-3350-4b4b-d185-0e1992ed43e9", 279 }, 280 }, 281 } 282 283 for _, tc := range cases { 284 t.Run(tc.name, func(t *testing.T) { 285 var req = &structs.AllocListRequest{ 286 QueryOptions: structs.QueryOptions{ 287 Region: "global", 288 Namespace: tc.namespace, 289 Prefix: tc.prefix, 290 PerPage: tc.pageSize, 291 NextToken: tc.nextToken, 292 Filter: tc.filter, 293 }, 294 Fields: &structs.AllocStubFields{ 295 Resources: false, 296 TaskStates: false, 297 }, 298 } 299 req.AuthToken = aclToken 300 var resp structs.AllocListResponse 301 err := msgpackrpc.CallWithCodec(codec, "Alloc.List", req, &resp) 302 if tc.expErr == "" { 303 require.NoError(t, err) 304 } else { 305 require.Contains(t, err, tc.expErr) 306 } 307 308 var gotIDs []string 309 for _, alloc := range resp.Allocations { 310 gotIDs = append(gotIDs, alloc.ID) 311 } 312 require.Equal(t, tc.expIDs, gotIDs) 313 require.Equal(t, tc.expNextToken, resp.QueryMeta.NextToken) 314 }) 315 } 316 } 317 318 func TestAllocEndpoint_List_order(t *testing.T) { 319 ci.Parallel(t) 320 321 s1, cleanupS1 := TestServer(t, nil) 322 defer cleanupS1() 323 codec := rpcClient(t, s1) 324 testutil.WaitForLeader(t, s1.RPC) 325 326 // Create register requests 327 uuid1 := uuid.Generate() 328 alloc1 := mock.Alloc() 329 alloc1.ID = uuid1 330 331 uuid2 := uuid.Generate() 332 alloc2 := mock.Alloc() 333 alloc2.ID = uuid2 334 335 uuid3 := uuid.Generate() 336 alloc3 := mock.Alloc() 337 alloc3.ID = uuid3 338 339 err := s1.fsm.State().UpsertAllocs(structs.MsgTypeTestSetup, 1000, []*structs.Allocation{alloc1}) 340 require.NoError(t, err) 341 342 err = s1.fsm.State().UpsertAllocs(structs.MsgTypeTestSetup, 1001, []*structs.Allocation{alloc2}) 343 require.NoError(t, err) 344 345 err = s1.fsm.State().UpsertAllocs(structs.MsgTypeTestSetup, 1002, []*structs.Allocation{alloc3}) 346 require.NoError(t, err) 347 348 // update alloc2 again so we can later assert create index order did not change 349 err = s1.fsm.State().UpsertAllocs(structs.MsgTypeTestSetup, 1003, []*structs.Allocation{alloc2}) 350 require.NoError(t, err) 351 352 t.Run("default", func(t *testing.T) { 353 // Lookup the allocations in the default order (oldest first) 354 get := &structs.AllocListRequest{ 355 QueryOptions: structs.QueryOptions{ 356 Region: "global", 357 Namespace: "*", 358 }, 359 } 360 361 var resp structs.AllocListResponse 362 err = msgpackrpc.CallWithCodec(codec, "Alloc.List", get, &resp) 363 require.NoError(t, err) 364 require.Equal(t, uint64(1003), resp.Index) 365 require.Len(t, resp.Allocations, 3) 366 367 // Assert returned order is by CreateIndex (ascending) 368 require.Equal(t, uint64(1000), resp.Allocations[0].CreateIndex) 369 require.Equal(t, uuid1, resp.Allocations[0].ID) 370 371 require.Equal(t, uint64(1001), resp.Allocations[1].CreateIndex) 372 require.Equal(t, uuid2, resp.Allocations[1].ID) 373 374 require.Equal(t, uint64(1002), resp.Allocations[2].CreateIndex) 375 require.Equal(t, uuid3, resp.Allocations[2].ID) 376 }) 377 378 t.Run("reverse", func(t *testing.T) { 379 // Lookup the allocations in reverse order (newest first) 380 get := &structs.AllocListRequest{ 381 QueryOptions: structs.QueryOptions{ 382 Region: "global", 383 Namespace: "*", 384 Reverse: true, 385 }, 386 } 387 388 var resp structs.AllocListResponse 389 err = msgpackrpc.CallWithCodec(codec, "Alloc.List", get, &resp) 390 require.NoError(t, err) 391 require.Equal(t, uint64(1003), resp.Index) 392 require.Len(t, resp.Allocations, 3) 393 394 // Assert returned order is by CreateIndex (descending) 395 require.Equal(t, uint64(1002), resp.Allocations[0].CreateIndex) 396 require.Equal(t, uuid3, resp.Allocations[0].ID) 397 398 require.Equal(t, uint64(1001), resp.Allocations[1].CreateIndex) 399 require.Equal(t, uuid2, resp.Allocations[1].ID) 400 401 require.Equal(t, uint64(1000), resp.Allocations[2].CreateIndex) 402 require.Equal(t, uuid1, resp.Allocations[2].ID) 403 }) 404 } 405 406 func TestAllocEndpoint_List_Fields(t *testing.T) { 407 ci.Parallel(t) 408 409 s1, cleanupS1 := TestServer(t, nil) 410 defer cleanupS1() 411 412 codec := rpcClient(t, s1) 413 testutil.WaitForLeader(t, s1.RPC) 414 415 // Create a running alloc 416 alloc := mock.Alloc() 417 alloc.ClientStatus = structs.AllocClientStatusRunning 418 alloc.TaskStates = map[string]*structs.TaskState{ 419 "web": { 420 State: structs.TaskStateRunning, 421 StartedAt: time.Now(), 422 }, 423 } 424 summary := mock.JobSummary(alloc.JobID) 425 state := s1.fsm.State() 426 427 require.NoError(t, state.UpsertJobSummary(999, summary)) 428 require.NoError(t, state.UpsertAllocs(structs.MsgTypeTestSetup, 1000, []*structs.Allocation{alloc})) 429 430 cases := []struct { 431 Name string 432 Fields *structs.AllocStubFields 433 Assert func(t *testing.T, allocs []*structs.AllocListStub) 434 }{ 435 { 436 Name: "None", 437 Fields: nil, 438 Assert: func(t *testing.T, allocs []*structs.AllocListStub) { 439 require.Nil(t, allocs[0].AllocatedResources) 440 require.Len(t, allocs[0].TaskStates, 1) 441 }, 442 }, 443 { 444 Name: "Default", 445 Fields: structs.NewAllocStubFields(), 446 Assert: func(t *testing.T, allocs []*structs.AllocListStub) { 447 require.Nil(t, allocs[0].AllocatedResources) 448 require.Len(t, allocs[0].TaskStates, 1) 449 }, 450 }, 451 { 452 Name: "Resources", 453 Fields: &structs.AllocStubFields{ 454 Resources: true, 455 TaskStates: false, 456 }, 457 Assert: func(t *testing.T, allocs []*structs.AllocListStub) { 458 require.NotNil(t, allocs[0].AllocatedResources) 459 require.Len(t, allocs[0].TaskStates, 0) 460 }, 461 }, 462 { 463 Name: "NoTaskStates", 464 Fields: &structs.AllocStubFields{ 465 Resources: false, 466 TaskStates: false, 467 }, 468 Assert: func(t *testing.T, allocs []*structs.AllocListStub) { 469 require.Nil(t, allocs[0].AllocatedResources) 470 require.Len(t, allocs[0].TaskStates, 0) 471 }, 472 }, 473 { 474 Name: "Both", 475 Fields: &structs.AllocStubFields{ 476 Resources: true, 477 TaskStates: true, 478 }, 479 Assert: func(t *testing.T, allocs []*structs.AllocListStub) { 480 require.NotNil(t, allocs[0].AllocatedResources) 481 require.Len(t, allocs[0].TaskStates, 1) 482 }, 483 }, 484 } 485 486 for i := range cases { 487 tc := cases[i] 488 t.Run(tc.Name, func(t *testing.T) { 489 get := &structs.AllocListRequest{ 490 QueryOptions: structs.QueryOptions{ 491 Region: "global", 492 Namespace: structs.DefaultNamespace, 493 }, 494 Fields: tc.Fields, 495 } 496 var resp structs.AllocListResponse 497 require.NoError(t, msgpackrpc.CallWithCodec(codec, "Alloc.List", get, &resp)) 498 require.Equal(t, uint64(1000), resp.Index) 499 require.Len(t, resp.Allocations, 1) 500 require.Equal(t, alloc.ID, resp.Allocations[0].ID) 501 tc.Assert(t, resp.Allocations) 502 }) 503 } 504 505 } 506 507 func TestAllocEndpoint_List_ACL(t *testing.T) { 508 ci.Parallel(t) 509 510 s1, root, cleanupS1 := TestACLServer(t, nil) 511 defer cleanupS1() 512 codec := rpcClient(t, s1) 513 testutil.WaitForLeader(t, s1.RPC) 514 assert := assert.New(t) 515 516 // Create the alloc 517 alloc := mock.Alloc() 518 allocs := []*structs.Allocation{alloc} 519 summary := mock.JobSummary(alloc.JobID) 520 state := s1.fsm.State() 521 522 assert.Nil(state.UpsertJobSummary(999, summary), "UpsertJobSummary") 523 assert.Nil(state.UpsertAllocs(structs.MsgTypeTestSetup, 1000, allocs), "UpsertAllocs") 524 525 stubAllocs := []*structs.AllocListStub{alloc.Stub(nil)} 526 stubAllocs[0].CreateIndex = 1000 527 stubAllocs[0].ModifyIndex = 1000 528 529 // Create the namespace policy and tokens 530 validToken := mock.CreatePolicyAndToken(t, state, 1001, "test-valid", 531 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 532 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 533 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 534 535 // Lookup the allocs without a token and expect failure 536 get := &structs.AllocListRequest{ 537 QueryOptions: structs.QueryOptions{ 538 Region: "global", 539 Namespace: structs.DefaultNamespace, 540 }, 541 } 542 var resp structs.AllocListResponse 543 assert.NotNil(msgpackrpc.CallWithCodec(codec, "Alloc.List", get, &resp), "RPC") 544 545 // Try with a valid token 546 get.AuthToken = validToken.SecretID 547 assert.Nil(msgpackrpc.CallWithCodec(codec, "Alloc.List", get, &resp), "RPC") 548 assert.EqualValues(resp.Index, 1000, "resp.Index") 549 assert.Equal(stubAllocs, resp.Allocations, "Returned alloc list not equal") 550 551 // Try with a invalid token 552 get.AuthToken = invalidToken.SecretID 553 err := msgpackrpc.CallWithCodec(codec, "Alloc.List", get, &resp) 554 assert.NotNil(err, "RPC") 555 assert.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 556 557 // Try with a root token 558 get.AuthToken = root.SecretID 559 assert.Nil(msgpackrpc.CallWithCodec(codec, "Alloc.List", get, &resp), "RPC") 560 assert.EqualValues(resp.Index, 1000, "resp.Index") 561 assert.Equal(stubAllocs, resp.Allocations, "Returned alloc list not equal") 562 } 563 564 func TestAllocEndpoint_List_Blocking(t *testing.T) { 565 ci.Parallel(t) 566 567 s1, cleanupS1 := TestServer(t, nil) 568 defer cleanupS1() 569 state := s1.fsm.State() 570 codec := rpcClient(t, s1) 571 testutil.WaitForLeader(t, s1.RPC) 572 573 // Create the alloc 574 alloc := mock.Alloc() 575 576 summary := mock.JobSummary(alloc.JobID) 577 if err := state.UpsertJobSummary(1, summary); err != nil { 578 t.Fatalf("err: %v", err) 579 } 580 // Upsert alloc triggers watches 581 time.AfterFunc(100*time.Millisecond, func() { 582 if err := state.UpsertAllocs(structs.MsgTypeTestSetup, 2, []*structs.Allocation{alloc}); err != nil { 583 t.Fatalf("err: %v", err) 584 } 585 }) 586 587 req := &structs.AllocListRequest{ 588 QueryOptions: structs.QueryOptions{ 589 Region: "global", 590 Namespace: structs.DefaultNamespace, 591 MinQueryIndex: 1, 592 }, 593 } 594 start := time.Now() 595 var resp structs.AllocListResponse 596 if err := msgpackrpc.CallWithCodec(codec, "Alloc.List", req, &resp); err != nil { 597 t.Fatalf("err: %v", err) 598 } 599 600 if elapsed := time.Since(start); elapsed < 100*time.Millisecond { 601 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 602 } 603 if resp.Index != 2 { 604 t.Fatalf("Bad index: %d %d", resp.Index, 2) 605 } 606 if len(resp.Allocations) != 1 || resp.Allocations[0].ID != alloc.ID { 607 t.Fatalf("bad: %#v", resp.Allocations) 608 } 609 610 // Client updates trigger watches 611 alloc2 := mock.Alloc() 612 alloc2.ID = alloc.ID 613 alloc2.ClientStatus = structs.AllocClientStatusRunning 614 time.AfterFunc(100*time.Millisecond, func() { 615 state.UpsertJobSummary(3, mock.JobSummary(alloc2.JobID)) 616 if err := state.UpdateAllocsFromClient(structs.MsgTypeTestSetup, 4, []*structs.Allocation{alloc2}); err != nil { 617 t.Fatalf("err: %v", err) 618 } 619 }) 620 621 req.MinQueryIndex = 3 622 start = time.Now() 623 var resp2 structs.AllocListResponse 624 if err := msgpackrpc.CallWithCodec(codec, "Alloc.List", req, &resp2); err != nil { 625 t.Fatalf("err: %v", err) 626 } 627 628 if elapsed := time.Since(start); elapsed < 100*time.Millisecond { 629 t.Fatalf("should block (returned in %s) %#v", elapsed, resp2) 630 } 631 if resp2.Index != 4 { 632 t.Fatalf("Bad index: %d %d", resp2.Index, 4) 633 } 634 if len(resp2.Allocations) != 1 || resp.Allocations[0].ID != alloc.ID || 635 resp2.Allocations[0].ClientStatus != structs.AllocClientStatusRunning { 636 t.Fatalf("bad: %#v", resp2.Allocations) 637 } 638 } 639 640 // TestAllocEndpoint_List_AllNamespaces_OSS asserts that server 641 // returns all allocations across namespaces. 642 func TestAllocEndpoint_List_AllNamespaces_OSS(t *testing.T) { 643 ci.Parallel(t) 644 645 s1, cleanupS1 := TestServer(t, nil) 646 defer cleanupS1() 647 codec := rpcClient(t, s1) 648 testutil.WaitForLeader(t, s1.RPC) 649 state := s1.fsm.State() 650 651 // two namespaces 652 ns1 := mock.Namespace() 653 ns2 := mock.Namespace() 654 require.NoError(t, state.UpsertNamespaces(900, []*structs.Namespace{ns1, ns2})) 655 656 // Create the allocations 657 uuid1 := uuid.Generate() 658 alloc1 := mock.Alloc() 659 alloc1.ID = uuid1 660 alloc1.Namespace = ns1.Name 661 662 uuid2 := uuid.Generate() 663 alloc2 := mock.Alloc() 664 alloc2.ID = uuid2 665 alloc2.Namespace = ns2.Name 666 667 summary1 := mock.JobSummary(alloc1.JobID) 668 summary2 := mock.JobSummary(alloc2.JobID) 669 670 require.NoError(t, state.UpsertJobSummary(1000, summary1)) 671 require.NoError(t, state.UpsertJobSummary(1001, summary2)) 672 require.NoError(t, state.UpsertAllocs(structs.MsgTypeTestSetup, 1002, []*structs.Allocation{alloc1})) 673 require.NoError(t, state.UpsertAllocs(structs.MsgTypeTestSetup, 1003, []*structs.Allocation{alloc2})) 674 675 t.Run("looking up all allocations", func(t *testing.T) { 676 get := &structs.AllocListRequest{ 677 QueryOptions: structs.QueryOptions{ 678 Region: "global", 679 Namespace: "*", 680 }, 681 } 682 var resp structs.AllocListResponse 683 require.NoError(t, msgpackrpc.CallWithCodec(codec, "Alloc.List", get, &resp)) 684 require.Equal(t, uint64(1003), resp.Index) 685 require.Len(t, resp.Allocations, 2) 686 require.ElementsMatch(t, 687 []string{resp.Allocations[0].ID, resp.Allocations[1].ID}, 688 []string{alloc1.ID, alloc2.ID}) 689 }) 690 691 t.Run("looking up allocations with prefix", func(t *testing.T) { 692 get := &structs.AllocListRequest{ 693 QueryOptions: structs.QueryOptions{ 694 Region: "global", 695 Namespace: "*", 696 // allocations were constructed above to have non-matching prefix 697 Prefix: alloc1.ID[:4], 698 }, 699 } 700 var resp structs.AllocListResponse 701 require.NoError(t, msgpackrpc.CallWithCodec(codec, "Alloc.List", get, &resp)) 702 require.Equal(t, uint64(1003), resp.Index) 703 require.Len(t, resp.Allocations, 1) 704 require.Equal(t, alloc1.ID, resp.Allocations[0].ID) 705 require.Equal(t, alloc1.Namespace, resp.Allocations[0].Namespace) 706 }) 707 708 t.Run("looking up allocations with mismatch prefix", func(t *testing.T) { 709 get := &structs.AllocListRequest{ 710 QueryOptions: structs.QueryOptions{ 711 Region: "global", 712 Namespace: "*", 713 Prefix: "000000", // unlikely to match 714 }, 715 } 716 var resp structs.AllocListResponse 717 require.NoError(t, msgpackrpc.CallWithCodec(codec, "Alloc.List", get, &resp)) 718 require.Equal(t, uint64(1003), resp.Index) 719 require.Empty(t, resp.Allocations) 720 }) 721 } 722 723 func TestAllocEndpoint_GetAlloc(t *testing.T) { 724 ci.Parallel(t) 725 726 s1, cleanupS1 := TestServer(t, nil) 727 defer cleanupS1() 728 codec := rpcClient(t, s1) 729 testutil.WaitForLeader(t, s1.RPC) 730 731 // Create the register request 732 prevAllocID := uuid.Generate() 733 alloc := mock.Alloc() 734 alloc.RescheduleTracker = &structs.RescheduleTracker{ 735 Events: []*structs.RescheduleEvent{ 736 {RescheduleTime: time.Now().UTC().UnixNano(), PrevNodeID: "boom", PrevAllocID: prevAllocID}, 737 }, 738 } 739 state := s1.fsm.State() 740 state.UpsertJobSummary(999, mock.JobSummary(alloc.JobID)) 741 err := state.UpsertAllocs(structs.MsgTypeTestSetup, 1000, []*structs.Allocation{alloc}) 742 if err != nil { 743 t.Fatalf("err: %v", err) 744 } 745 746 // Lookup the alloc 747 get := &structs.AllocSpecificRequest{ 748 AllocID: alloc.ID, 749 QueryOptions: structs.QueryOptions{Region: "global"}, 750 } 751 var resp structs.SingleAllocResponse 752 if err := msgpackrpc.CallWithCodec(codec, "Alloc.GetAlloc", get, &resp); err != nil { 753 t.Fatalf("err: %v", err) 754 } 755 if resp.Index != 1000 { 756 t.Fatalf("Bad index: %d %d", resp.Index, 1000) 757 } 758 759 if !reflect.DeepEqual(alloc, resp.Alloc) { 760 t.Fatalf("bad: %#v", resp.Alloc) 761 } 762 } 763 764 func TestAllocEndpoint_GetAlloc_ACL(t *testing.T) { 765 ci.Parallel(t) 766 767 s1, root, cleanupS1 := TestACLServer(t, nil) 768 defer cleanupS1() 769 codec := rpcClient(t, s1) 770 testutil.WaitForLeader(t, s1.RPC) 771 assert := assert.New(t) 772 773 // Create the alloc 774 alloc := mock.Alloc() 775 allocs := []*structs.Allocation{alloc} 776 summary := mock.JobSummary(alloc.JobID) 777 state := s1.fsm.State() 778 779 assert.Nil(state.UpsertJobSummary(999, summary), "UpsertJobSummary") 780 assert.Nil(state.UpsertAllocs(structs.MsgTypeTestSetup, 1000, allocs), "UpsertAllocs") 781 782 // Create the namespace policy and tokens 783 validToken := mock.CreatePolicyAndToken(t, state, 1001, "test-valid", 784 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 785 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 786 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 787 788 getReq := func() *structs.AllocSpecificRequest { 789 return &structs.AllocSpecificRequest{ 790 AllocID: alloc.ID, 791 QueryOptions: structs.QueryOptions{ 792 Region: "global", 793 }, 794 } 795 } 796 797 cases := []struct { 798 Name string 799 F func(t *testing.T) 800 }{ 801 // Lookup the alloc without a token and expect failure 802 { 803 Name: "no-token", 804 F: func(t *testing.T) { 805 var resp structs.SingleAllocResponse 806 err := msgpackrpc.CallWithCodec(codec, "Alloc.GetAlloc", getReq(), &resp) 807 require.True(t, structs.IsErrUnknownAllocation(err), "expected unknown alloc but found: %v", err) 808 }, 809 }, 810 811 // Try with a valid ACL token 812 { 813 Name: "valid-token", 814 F: func(t *testing.T) { 815 get := getReq() 816 get.AuthToken = validToken.SecretID 817 get.AllocID = alloc.ID 818 var resp structs.SingleAllocResponse 819 require.NoError(t, msgpackrpc.CallWithCodec(codec, "Alloc.GetAlloc", get, &resp), "RPC") 820 require.EqualValues(t, resp.Index, 1000, "resp.Index") 821 require.Equal(t, alloc, resp.Alloc, "Returned alloc not equal") 822 }, 823 }, 824 825 // Try with a valid Node.SecretID 826 { 827 Name: "valid-node-secret", 828 F: func(t *testing.T) { 829 node := mock.Node() 830 assert.Nil(state.UpsertNode(structs.MsgTypeTestSetup, 1005, node)) 831 get := getReq() 832 get.AuthToken = node.SecretID 833 get.AllocID = alloc.ID 834 var resp structs.SingleAllocResponse 835 require.NoError(t, msgpackrpc.CallWithCodec(codec, "Alloc.GetAlloc", get, &resp), "RPC") 836 require.EqualValues(t, resp.Index, 1000, "resp.Index") 837 require.Equal(t, alloc, resp.Alloc, "Returned alloc not equal") 838 }, 839 }, 840 841 // Try with a invalid token 842 { 843 Name: "invalid-token", 844 F: func(t *testing.T) { 845 get := getReq() 846 get.AuthToken = invalidToken.SecretID 847 get.AllocID = alloc.ID 848 var resp structs.SingleAllocResponse 849 err := msgpackrpc.CallWithCodec(codec, "Alloc.GetAlloc", get, &resp) 850 require.NotNil(t, err, "RPC") 851 require.True(t, structs.IsErrUnknownAllocation(err), "expected unknown alloc but found: %v", err) 852 }, 853 }, 854 855 // Try with a root token 856 { 857 Name: "root-token", 858 F: func(t *testing.T) { 859 get := getReq() 860 get.AuthToken = root.SecretID 861 get.AllocID = alloc.ID 862 var resp structs.SingleAllocResponse 863 require.NoError(t, msgpackrpc.CallWithCodec(codec, "Alloc.GetAlloc", get, &resp), "RPC") 864 require.EqualValues(t, resp.Index, 1000, "resp.Index") 865 require.Equal(t, alloc, resp.Alloc, "Returned alloc not equal") 866 }, 867 }, 868 } 869 870 for _, tc := range cases { 871 t.Run(tc.Name, tc.F) 872 } 873 } 874 875 func TestAllocEndpoint_GetAlloc_Blocking(t *testing.T) { 876 ci.Parallel(t) 877 878 s1, cleanupS1 := TestServer(t, nil) 879 defer cleanupS1() 880 state := s1.fsm.State() 881 codec := rpcClient(t, s1) 882 testutil.WaitForLeader(t, s1.RPC) 883 884 // Create the allocs 885 alloc1 := mock.Alloc() 886 alloc2 := mock.Alloc() 887 888 // First create an unrelated alloc 889 time.AfterFunc(100*time.Millisecond, func() { 890 state.UpsertJobSummary(99, mock.JobSummary(alloc1.JobID)) 891 err := state.UpsertAllocs(structs.MsgTypeTestSetup, 100, []*structs.Allocation{alloc1}) 892 if err != nil { 893 t.Fatalf("err: %v", err) 894 } 895 }) 896 897 // Create the alloc we are watching later 898 time.AfterFunc(200*time.Millisecond, func() { 899 state.UpsertJobSummary(199, mock.JobSummary(alloc2.JobID)) 900 err := state.UpsertAllocs(structs.MsgTypeTestSetup, 200, []*structs.Allocation{alloc2}) 901 if err != nil { 902 t.Fatalf("err: %v", err) 903 } 904 }) 905 906 // Lookup the allocs 907 get := &structs.AllocSpecificRequest{ 908 AllocID: alloc2.ID, 909 QueryOptions: structs.QueryOptions{ 910 Region: "global", 911 MinQueryIndex: 150, 912 }, 913 } 914 var resp structs.SingleAllocResponse 915 start := time.Now() 916 if err := msgpackrpc.CallWithCodec(codec, "Alloc.GetAlloc", get, &resp); err != nil { 917 t.Fatalf("err: %v", err) 918 } 919 920 if elapsed := time.Since(start); elapsed < 200*time.Millisecond { 921 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 922 } 923 if resp.Index != 200 { 924 t.Fatalf("Bad index: %d %d", resp.Index, 200) 925 } 926 if resp.Alloc == nil || resp.Alloc.ID != alloc2.ID { 927 t.Fatalf("bad: %#v", resp.Alloc) 928 } 929 } 930 931 func TestAllocEndpoint_GetAllocs(t *testing.T) { 932 ci.Parallel(t) 933 934 s1, cleanupS1 := TestServer(t, nil) 935 defer cleanupS1() 936 codec := rpcClient(t, s1) 937 testutil.WaitForLeader(t, s1.RPC) 938 939 // Create the register request 940 alloc := mock.Alloc() 941 alloc2 := mock.Alloc() 942 state := s1.fsm.State() 943 state.UpsertJobSummary(998, mock.JobSummary(alloc.JobID)) 944 state.UpsertJobSummary(999, mock.JobSummary(alloc2.JobID)) 945 err := state.UpsertAllocs(structs.MsgTypeTestSetup, 1000, []*structs.Allocation{alloc, alloc2}) 946 if err != nil { 947 t.Fatalf("err: %v", err) 948 } 949 950 // Lookup the allocs 951 get := &structs.AllocsGetRequest{ 952 AllocIDs: []string{alloc.ID, alloc2.ID}, 953 QueryOptions: structs.QueryOptions{ 954 Region: "global", 955 }, 956 } 957 var resp structs.AllocsGetResponse 958 if err := msgpackrpc.CallWithCodec(codec, "Alloc.GetAllocs", get, &resp); err != nil { 959 t.Fatalf("err: %v", err) 960 } 961 if resp.Index != 1000 { 962 t.Fatalf("Bad index: %d %d", resp.Index, 1000) 963 } 964 965 if len(resp.Allocs) != 2 { 966 t.Fatalf("bad: %#v", resp.Allocs) 967 } 968 969 // Lookup nonexistent allocs. 970 get = &structs.AllocsGetRequest{ 971 AllocIDs: []string{"foo"}, 972 QueryOptions: structs.QueryOptions{Region: "global"}, 973 } 974 if err := msgpackrpc.CallWithCodec(codec, "Alloc.GetAllocs", get, &resp); err == nil { 975 t.Fatalf("expect error") 976 } 977 } 978 979 func TestAllocEndpoint_GetAllocs_Blocking(t *testing.T) { 980 ci.Parallel(t) 981 982 s1, cleanupS1 := TestServer(t, nil) 983 defer cleanupS1() 984 state := s1.fsm.State() 985 codec := rpcClient(t, s1) 986 testutil.WaitForLeader(t, s1.RPC) 987 988 // Create the allocs 989 alloc1 := mock.Alloc() 990 alloc2 := mock.Alloc() 991 992 // First create an unrelated alloc 993 time.AfterFunc(100*time.Millisecond, func() { 994 state.UpsertJobSummary(99, mock.JobSummary(alloc1.JobID)) 995 err := state.UpsertAllocs(structs.MsgTypeTestSetup, 100, []*structs.Allocation{alloc1}) 996 if err != nil { 997 t.Fatalf("err: %v", err) 998 } 999 }) 1000 1001 // Create the alloc we are watching later 1002 time.AfterFunc(200*time.Millisecond, func() { 1003 state.UpsertJobSummary(199, mock.JobSummary(alloc2.JobID)) 1004 err := state.UpsertAllocs(structs.MsgTypeTestSetup, 200, []*structs.Allocation{alloc2}) 1005 if err != nil { 1006 t.Fatalf("err: %v", err) 1007 } 1008 }) 1009 1010 // Lookup the allocs 1011 get := &structs.AllocsGetRequest{ 1012 AllocIDs: []string{alloc1.ID, alloc2.ID}, 1013 QueryOptions: structs.QueryOptions{ 1014 Region: "global", 1015 MinQueryIndex: 150, 1016 }, 1017 } 1018 var resp structs.AllocsGetResponse 1019 start := time.Now() 1020 if err := msgpackrpc.CallWithCodec(codec, "Alloc.GetAllocs", get, &resp); err != nil { 1021 t.Fatalf("err: %v", err) 1022 } 1023 1024 if elapsed := time.Since(start); elapsed < 200*time.Millisecond { 1025 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 1026 } 1027 if resp.Index != 200 { 1028 t.Fatalf("Bad index: %d %d", resp.Index, 200) 1029 } 1030 if len(resp.Allocs) != 2 { 1031 t.Fatalf("bad: %#v", resp.Allocs) 1032 } 1033 } 1034 1035 func TestAllocEndpoint_UpdateDesiredTransition(t *testing.T) { 1036 ci.Parallel(t) 1037 require := require.New(t) 1038 1039 s1, _, cleanupS1 := TestACLServer(t, nil) 1040 defer cleanupS1() 1041 codec := rpcClient(t, s1) 1042 testutil.WaitForLeader(t, s1.RPC) 1043 1044 // Create the register request 1045 alloc := mock.Alloc() 1046 alloc2 := mock.Alloc() 1047 state := s1.fsm.State() 1048 require.Nil(state.UpsertJobSummary(998, mock.JobSummary(alloc.JobID))) 1049 require.Nil(state.UpsertJobSummary(999, mock.JobSummary(alloc2.JobID))) 1050 require.Nil(state.UpsertAllocs(structs.MsgTypeTestSetup, 1000, []*structs.Allocation{alloc, alloc2})) 1051 1052 t1 := &structs.DesiredTransition{ 1053 Migrate: pointer.Of(true), 1054 } 1055 1056 // Update the allocs desired status 1057 get := &structs.AllocUpdateDesiredTransitionRequest{ 1058 Allocs: map[string]*structs.DesiredTransition{ 1059 alloc.ID: t1, 1060 alloc2.ID: t1, 1061 }, 1062 Evals: []*structs.Evaluation{ 1063 { 1064 ID: uuid.Generate(), 1065 Namespace: alloc.Namespace, 1066 Priority: alloc.Job.Priority, 1067 Type: alloc.Job.Type, 1068 TriggeredBy: structs.EvalTriggerNodeDrain, 1069 JobID: alloc.Job.ID, 1070 JobModifyIndex: alloc.Job.ModifyIndex, 1071 Status: structs.EvalStatusPending, 1072 }, 1073 { 1074 ID: uuid.Generate(), 1075 Namespace: alloc2.Namespace, 1076 Priority: alloc2.Job.Priority, 1077 Type: alloc2.Job.Type, 1078 TriggeredBy: structs.EvalTriggerNodeDrain, 1079 JobID: alloc2.Job.ID, 1080 JobModifyIndex: alloc2.Job.ModifyIndex, 1081 Status: structs.EvalStatusPending, 1082 }, 1083 }, 1084 WriteRequest: structs.WriteRequest{ 1085 Region: "global", 1086 }, 1087 } 1088 1089 // Try without permissions 1090 var resp structs.GenericResponse 1091 err := msgpackrpc.CallWithCodec(codec, "Alloc.UpdateDesiredTransition", get, &resp) 1092 require.NotNil(err) 1093 require.True(structs.IsErrPermissionDenied(err)) 1094 1095 // Try with permissions 1096 get.WriteRequest.AuthToken = s1.getLeaderAcl() 1097 var resp2 structs.GenericResponse 1098 require.Nil(msgpackrpc.CallWithCodec(codec, "Alloc.UpdateDesiredTransition", get, &resp2)) 1099 require.NotZero(resp2.Index) 1100 1101 // Look up the allocations 1102 out1, err := state.AllocByID(nil, alloc.ID) 1103 require.Nil(err) 1104 out2, err := state.AllocByID(nil, alloc.ID) 1105 require.Nil(err) 1106 e1, err := state.EvalByID(nil, get.Evals[0].ID) 1107 require.Nil(err) 1108 e2, err := state.EvalByID(nil, get.Evals[1].ID) 1109 require.Nil(err) 1110 1111 require.NotNil(out1.DesiredTransition.Migrate) 1112 require.NotNil(out2.DesiredTransition.Migrate) 1113 require.NotNil(e1) 1114 require.NotNil(e2) 1115 require.True(*out1.DesiredTransition.Migrate) 1116 require.True(*out2.DesiredTransition.Migrate) 1117 } 1118 1119 func TestAllocEndpoint_Stop_ACL(t *testing.T) { 1120 ci.Parallel(t) 1121 require := require.New(t) 1122 1123 s1, _, cleanupS1 := TestACLServer(t, nil) 1124 defer cleanupS1() 1125 codec := rpcClient(t, s1) 1126 testutil.WaitForLeader(t, s1.RPC) 1127 1128 // Create the register request 1129 alloc := mock.Alloc() 1130 alloc2 := mock.Alloc() 1131 state := s1.fsm.State() 1132 require.Nil(state.UpsertJobSummary(998, mock.JobSummary(alloc.JobID))) 1133 require.Nil(state.UpsertJobSummary(999, mock.JobSummary(alloc2.JobID))) 1134 require.Nil(state.UpsertAllocs(structs.MsgTypeTestSetup, 1000, []*structs.Allocation{alloc, alloc2})) 1135 1136 req := &structs.AllocStopRequest{ 1137 AllocID: alloc.ID, 1138 } 1139 req.Namespace = structs.DefaultNamespace 1140 req.Region = alloc.Job.Region 1141 1142 // Try without permissions 1143 var resp structs.AllocStopResponse 1144 err := msgpackrpc.CallWithCodec(codec, "Alloc.Stop", req, &resp) 1145 require.True(structs.IsErrPermissionDenied(err), "expected permissions error, got: %v", err) 1146 1147 // Try with management permissions 1148 req.WriteRequest.AuthToken = s1.getLeaderAcl() 1149 var resp2 structs.AllocStopResponse 1150 require.Nil(msgpackrpc.CallWithCodec(codec, "Alloc.Stop", req, &resp2)) 1151 require.NotZero(resp2.Index) 1152 1153 // Try with alloc-lifecycle permissions 1154 validToken := mock.CreatePolicyAndToken(t, state, 1002, "valid", 1155 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityAllocLifecycle})) 1156 req.WriteRequest.AuthToken = validToken.SecretID 1157 req.AllocID = alloc2.ID 1158 1159 var resp3 structs.AllocStopResponse 1160 require.Nil(msgpackrpc.CallWithCodec(codec, "Alloc.Stop", req, &resp3)) 1161 require.NotZero(resp3.Index) 1162 1163 // Look up the allocations 1164 out1, err := state.AllocByID(nil, alloc.ID) 1165 require.Nil(err) 1166 out2, err := state.AllocByID(nil, alloc2.ID) 1167 require.Nil(err) 1168 e1, err := state.EvalByID(nil, resp2.EvalID) 1169 require.Nil(err) 1170 e2, err := state.EvalByID(nil, resp3.EvalID) 1171 require.Nil(err) 1172 1173 require.NotNil(out1.DesiredTransition.Migrate) 1174 require.NotNil(out2.DesiredTransition.Migrate) 1175 require.NotNil(e1) 1176 require.NotNil(e2) 1177 require.True(*out1.DesiredTransition.Migrate) 1178 require.True(*out2.DesiredTransition.Migrate) 1179 } 1180 1181 func TestAllocEndpoint_List_AllNamespaces_ACL_OSS(t *testing.T) { 1182 ci.Parallel(t) 1183 1184 s1, root, cleanupS1 := TestACLServer(t, nil) 1185 defer cleanupS1() 1186 codec := rpcClient(t, s1) 1187 testutil.WaitForLeader(t, s1.RPC) 1188 state := s1.fsm.State() 1189 1190 // two namespaces 1191 ns1 := mock.Namespace() 1192 ns2 := mock.Namespace() 1193 require.NoError(t, state.UpsertNamespaces(900, []*structs.Namespace{ns1, ns2})) 1194 1195 // Create the allocations 1196 alloc1 := mock.Alloc() 1197 alloc1.ID = "a" + alloc1.ID[1:] 1198 alloc1.Namespace = ns1.Name 1199 alloc2 := mock.Alloc() 1200 alloc2.ID = "b" + alloc2.ID[1:] 1201 alloc2.Namespace = ns2.Name 1202 summary1 := mock.JobSummary(alloc1.JobID) 1203 summary2 := mock.JobSummary(alloc2.JobID) 1204 1205 require.NoError(t, state.UpsertJobSummary(999, summary1)) 1206 require.NoError(t, state.UpsertJobSummary(999, summary2)) 1207 require.NoError(t, state.UpsertAllocs(structs.MsgTypeTestSetup, 1000, []*structs.Allocation{alloc1, alloc2})) 1208 alloc1.CreateIndex = 1000 1209 alloc1.ModifyIndex = 1000 1210 alloc2.CreateIndex = 1000 1211 alloc2.ModifyIndex = 1000 1212 1213 everythingButReadJob := []string{ 1214 acl.NamespaceCapabilityDeny, 1215 acl.NamespaceCapabilityListJobs, 1216 // acl.NamespaceCapabilityReadJob, 1217 acl.NamespaceCapabilitySubmitJob, 1218 acl.NamespaceCapabilityDispatchJob, 1219 acl.NamespaceCapabilityReadLogs, 1220 acl.NamespaceCapabilityReadFS, 1221 acl.NamespaceCapabilityAllocExec, 1222 acl.NamespaceCapabilityAllocNodeExec, 1223 acl.NamespaceCapabilityAllocLifecycle, 1224 acl.NamespaceCapabilitySentinelOverride, 1225 acl.NamespaceCapabilityCSIRegisterPlugin, 1226 acl.NamespaceCapabilityCSIWriteVolume, 1227 acl.NamespaceCapabilityCSIReadVolume, 1228 acl.NamespaceCapabilityCSIListVolume, 1229 acl.NamespaceCapabilityCSIMountVolume, 1230 acl.NamespaceCapabilityListScalingPolicies, 1231 acl.NamespaceCapabilityReadScalingPolicy, 1232 acl.NamespaceCapabilityReadJobScaling, 1233 acl.NamespaceCapabilityScaleJob, 1234 acl.NamespaceCapabilitySubmitRecommendation, 1235 } 1236 1237 ns1token := mock.CreatePolicyAndToken(t, state, 1001, "ns1", 1238 mock.NamespacePolicy(ns1.Name, "", []string{acl.NamespaceCapabilityReadJob})) 1239 ns1tokenInsufficient := mock.CreatePolicyAndToken(t, state, 1001, "ns1-insufficient", 1240 mock.NamespacePolicy(ns1.Name, "", everythingButReadJob)) 1241 ns2token := mock.CreatePolicyAndToken(t, state, 1001, "ns2", 1242 mock.NamespacePolicy(ns2.Name, "", []string{acl.NamespaceCapabilityReadJob})) 1243 bothToken := mock.CreatePolicyAndToken(t, state, 1001, "nsBoth", 1244 mock.NamespacePolicy(ns1.Name, "", []string{acl.NamespaceCapabilityReadJob})+ 1245 mock.NamespacePolicy(ns2.Name, "", []string{acl.NamespaceCapabilityReadJob})) 1246 1247 cases := []struct { 1248 Label string 1249 Namespace string 1250 Token string 1251 Allocs []*structs.Allocation 1252 Error bool 1253 Message string 1254 Prefix string 1255 }{ 1256 { 1257 Label: "all namespaces with sufficient token", 1258 Namespace: "*", 1259 Token: bothToken.SecretID, 1260 Allocs: []*structs.Allocation{alloc1, alloc2}, 1261 }, 1262 { 1263 Label: "all namespaces with root token", 1264 Namespace: "*", 1265 Token: root.SecretID, 1266 Allocs: []*structs.Allocation{alloc1, alloc2}, 1267 }, 1268 { 1269 Label: "all namespaces with ns1 token", 1270 Namespace: "*", 1271 Token: ns1token.SecretID, 1272 Allocs: []*structs.Allocation{alloc1}, 1273 }, 1274 { 1275 Label: "all namespaces with ns2 token", 1276 Namespace: "*", 1277 Token: ns2token.SecretID, 1278 Allocs: []*structs.Allocation{alloc2}, 1279 }, 1280 { 1281 Label: "all namespaces with bad token", 1282 Namespace: "*", 1283 Token: uuid.Generate(), 1284 Error: true, 1285 Message: structs.ErrPermissionDenied.Error(), 1286 }, 1287 { 1288 Label: "all namespaces with insufficient token", 1289 Namespace: "*", 1290 Token: ns1tokenInsufficient.SecretID, 1291 Error: true, 1292 Message: structs.ErrPermissionDenied.Error(), 1293 }, 1294 { 1295 Label: "ns1 with ns1 token", 1296 Namespace: ns1.Name, 1297 Token: ns1token.SecretID, 1298 Allocs: []*structs.Allocation{alloc1}, 1299 }, 1300 { 1301 Label: "ns1 with root token", 1302 Namespace: ns1.Name, 1303 Token: root.SecretID, 1304 Allocs: []*structs.Allocation{alloc1}, 1305 }, 1306 { 1307 Label: "ns1 with ns2 token", 1308 Namespace: ns1.Name, 1309 Token: ns2token.SecretID, 1310 Error: true, 1311 }, 1312 { 1313 Label: "ns1 with invalid token", 1314 Namespace: ns1.Name, 1315 Token: uuid.Generate(), 1316 Error: true, 1317 Message: structs.ErrPermissionDenied.Error(), 1318 }, 1319 { 1320 Label: "bad namespace with root token", 1321 Namespace: uuid.Generate(), 1322 Token: root.SecretID, 1323 Allocs: []*structs.Allocation{}, 1324 }, 1325 { 1326 Label: "all namespaces with prefix", 1327 Namespace: "*", 1328 Prefix: alloc1.ID[:2], 1329 Token: root.SecretID, 1330 Allocs: []*structs.Allocation{alloc1}, 1331 }, 1332 } 1333 1334 for _, tc := range cases { 1335 t.Run(tc.Label, func(t *testing.T) { 1336 1337 get := &structs.AllocListRequest{ 1338 QueryOptions: structs.QueryOptions{ 1339 Region: "global", 1340 Namespace: tc.Namespace, 1341 Prefix: tc.Prefix, 1342 AuthToken: tc.Token, 1343 }, 1344 } 1345 var resp structs.AllocListResponse 1346 err := msgpackrpc.CallWithCodec(codec, "Alloc.List", get, &resp) 1347 if tc.Error { 1348 require.Error(t, err) 1349 if tc.Message != "" { 1350 require.Equal(t, err.Error(), tc.Message) 1351 } else { 1352 require.Equal(t, err.Error(), structs.ErrPermissionDenied.Error()) 1353 } 1354 } else { 1355 require.NoError(t, err) 1356 require.Equal(t, uint64(1000), resp.Index) 1357 exp := make([]*structs.AllocListStub, len(tc.Allocs)) 1358 for i, a := range tc.Allocs { 1359 exp[i] = a.Stub(nil) 1360 } 1361 require.ElementsMatch(t, exp, resp.Allocations) 1362 } 1363 }) 1364 } 1365 1366 } 1367 1368 func TestAlloc_GetServiceRegistrations(t *testing.T) { 1369 ci.Parallel(t) 1370 1371 // This function is a helper function to set up an allocation and service 1372 // which can be queried. 1373 correctSetupFn := func(s *Server) (error, string, *structs.ServiceRegistration) { 1374 // Generate an upsert an allocation. 1375 alloc := mock.Alloc() 1376 err := s.State().UpsertAllocs(structs.MsgTypeTestSetup, 10, []*structs.Allocation{alloc}) 1377 if err != nil { 1378 return nil, "", nil 1379 } 1380 1381 // Generate services. Set the allocation ID to the first, so it 1382 // matches the allocation. The alloc and first service both 1383 // reside in the default namespace. 1384 services := mock.ServiceRegistrations() 1385 services[0].AllocID = alloc.ID 1386 err = s.State().UpsertServiceRegistrations(structs.MsgTypeTestSetup, 20, services) 1387 1388 return err, alloc.ID, services[0] 1389 } 1390 1391 testCases := []struct { 1392 serverFn func(t *testing.T) (*Server, *structs.ACLToken, func()) 1393 testFn func(t *testing.T, s *Server, token *structs.ACLToken) 1394 name string 1395 }{ 1396 { 1397 serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { 1398 server, cleanup := TestServer(t, nil) 1399 return server, nil, cleanup 1400 }, 1401 testFn: func(t *testing.T, s *Server, _ *structs.ACLToken) { 1402 codec := rpcClient(t, s) 1403 testutil.WaitForLeader(t, s.RPC) 1404 1405 err, allocID, service := correctSetupFn(s) 1406 require.NoError(t, err) 1407 1408 // Perform a lookup on the first service. 1409 serviceRegReq := &structs.AllocServiceRegistrationsRequest{ 1410 AllocID: allocID, 1411 QueryOptions: structs.QueryOptions{ 1412 Namespace: service.Namespace, 1413 Region: s.Region(), 1414 }, 1415 } 1416 var serviceRegResp structs.AllocServiceRegistrationsResponse 1417 err = msgpackrpc.CallWithCodec(codec, structs.AllocServiceRegistrationsRPCMethod, serviceRegReq, &serviceRegResp) 1418 require.NoError(t, err) 1419 require.EqualValues(t, uint64(20), serviceRegResp.Index) 1420 require.ElementsMatch(t, serviceRegResp.Services, []*structs.ServiceRegistration{service}) 1421 }, 1422 name: "ACLs disabled alloc found with regs", 1423 }, 1424 { 1425 serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { 1426 server, cleanup := TestServer(t, nil) 1427 return server, nil, cleanup 1428 }, 1429 testFn: func(t *testing.T, s *Server, _ *structs.ACLToken) { 1430 codec := rpcClient(t, s) 1431 testutil.WaitForLeader(t, s.RPC) 1432 1433 // Generate and upsert our services. 1434 services := mock.ServiceRegistrations() 1435 require.NoError(t, s.State().UpsertServiceRegistrations(structs.MsgTypeTestSetup, 20, services)) 1436 1437 // Perform a lookup on the first service using the allocation 1438 // ID. This allocation does not exist within the Nomad state 1439 // meaning the service is orphaned or the caller used an 1440 // incorrect allocation ID. 1441 serviceRegReq := &structs.AllocServiceRegistrationsRequest{ 1442 AllocID: services[0].AllocID, 1443 QueryOptions: structs.QueryOptions{ 1444 Namespace: services[0].Namespace, 1445 Region: s.Region(), 1446 }, 1447 } 1448 var serviceRegResp structs.AllocServiceRegistrationsResponse 1449 err := msgpackrpc.CallWithCodec(codec, structs.AllocServiceRegistrationsRPCMethod, serviceRegReq, &serviceRegResp) 1450 require.NoError(t, err) 1451 require.Nil(t, serviceRegResp.Services) 1452 }, 1453 name: "ACLs disabled alloc not found", 1454 }, 1455 { 1456 serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { 1457 server, cleanup := TestServer(t, nil) 1458 return server, nil, cleanup 1459 }, 1460 testFn: func(t *testing.T, s *Server, _ *structs.ACLToken) { 1461 codec := rpcClient(t, s) 1462 testutil.WaitForLeader(t, s.RPC) 1463 1464 err, allocID, _ := correctSetupFn(s) 1465 require.NoError(t, err) 1466 1467 // Perform a lookup on the first service using the allocation 1468 // ID but a random namespace. The namespace on the allocation 1469 // does therefore not match the request args. 1470 serviceRegReq := &structs.AllocServiceRegistrationsRequest{ 1471 AllocID: allocID, 1472 QueryOptions: structs.QueryOptions{ 1473 Namespace: "platform", 1474 Region: s.Region(), 1475 }, 1476 } 1477 var serviceRegResp structs.AllocServiceRegistrationsResponse 1478 err = msgpackrpc.CallWithCodec(codec, structs.AllocServiceRegistrationsRPCMethod, serviceRegReq, &serviceRegResp) 1479 require.NoError(t, err) 1480 require.ElementsMatch(t, serviceRegResp.Services, []*structs.ServiceRegistration{}) 1481 }, 1482 name: "ACLs disabled alloc found in different namespace than request", 1483 }, 1484 { 1485 serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { 1486 server, cleanup := TestServer(t, nil) 1487 return server, nil, cleanup 1488 }, 1489 testFn: func(t *testing.T, s *Server, _ *structs.ACLToken) { 1490 codec := rpcClient(t, s) 1491 testutil.WaitForLeader(t, s.RPC) 1492 1493 // Generate an upsert an allocation. 1494 alloc := mock.Alloc() 1495 require.NoError(t, s.State().UpsertAllocs( 1496 structs.MsgTypeTestSetup, 10, []*structs.Allocation{alloc})) 1497 1498 // Perform a lookup using the allocation information. 1499 serviceRegReq := &structs.AllocServiceRegistrationsRequest{ 1500 AllocID: alloc.ID, 1501 QueryOptions: structs.QueryOptions{ 1502 Namespace: alloc.Namespace, 1503 Region: s.Region(), 1504 }, 1505 } 1506 var serviceRegResp structs.AllocServiceRegistrationsResponse 1507 err := msgpackrpc.CallWithCodec(codec, structs.AllocServiceRegistrationsRPCMethod, serviceRegReq, &serviceRegResp) 1508 require.NoError(t, err) 1509 require.ElementsMatch(t, serviceRegResp.Services, []*structs.ServiceRegistration{}) 1510 }, 1511 name: "ACLs disabled alloc found without regs", 1512 }, 1513 { 1514 serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { 1515 return TestACLServer(t, nil) 1516 }, 1517 testFn: func(t *testing.T, s *Server, token *structs.ACLToken) { 1518 codec := rpcClient(t, s) 1519 testutil.WaitForLeader(t, s.RPC) 1520 1521 err, allocID, service := correctSetupFn(s) 1522 require.NoError(t, err) 1523 1524 // Perform a lookup using the allocation information. 1525 serviceRegReq := &structs.AllocServiceRegistrationsRequest{ 1526 AllocID: allocID, 1527 QueryOptions: structs.QueryOptions{ 1528 Namespace: service.Namespace, 1529 Region: s.Region(), 1530 AuthToken: token.SecretID, 1531 }, 1532 } 1533 var serviceRegResp structs.AllocServiceRegistrationsResponse 1534 err = msgpackrpc.CallWithCodec(codec, structs.AllocServiceRegistrationsRPCMethod, serviceRegReq, &serviceRegResp) 1535 require.NoError(t, err) 1536 require.ElementsMatch(t, serviceRegResp.Services, []*structs.ServiceRegistration{service}) 1537 }, 1538 name: "ACLs enabled use management token", 1539 }, 1540 { 1541 serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { 1542 return TestACLServer(t, nil) 1543 }, 1544 testFn: func(t *testing.T, s *Server, _ *structs.ACLToken) { 1545 codec := rpcClient(t, s) 1546 testutil.WaitForLeader(t, s.RPC) 1547 1548 err, allocID, service := correctSetupFn(s) 1549 require.NoError(t, err) 1550 1551 // Create and policy and grab the auth token. 1552 authToken := mock.CreatePolicyAndToken(t, s.State(), 30, "test-node-get-service-reg", 1553 mock.NamespacePolicy(service.Namespace, "", []string{acl.NamespaceCapabilityReadJob})).SecretID 1554 1555 // Perform a lookup using the allocation information. 1556 serviceRegReq := &structs.AllocServiceRegistrationsRequest{ 1557 AllocID: allocID, 1558 QueryOptions: structs.QueryOptions{ 1559 Namespace: service.Namespace, 1560 Region: s.Region(), 1561 AuthToken: authToken, 1562 }, 1563 } 1564 var serviceRegResp structs.AllocServiceRegistrationsResponse 1565 err = msgpackrpc.CallWithCodec(codec, structs.AllocServiceRegistrationsRPCMethod, serviceRegReq, &serviceRegResp) 1566 require.NoError(t, err) 1567 require.ElementsMatch(t, serviceRegResp.Services, []*structs.ServiceRegistration{service}) 1568 }, 1569 name: "ACLs enabled use read-job namespace capability token", 1570 }, 1571 { 1572 serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { 1573 return TestACLServer(t, nil) 1574 }, 1575 testFn: func(t *testing.T, s *Server, _ *structs.ACLToken) { 1576 codec := rpcClient(t, s) 1577 testutil.WaitForLeader(t, s.RPC) 1578 1579 err, allocID, service := correctSetupFn(s) 1580 require.NoError(t, err) 1581 1582 // Create and policy and grab the auth token. 1583 authToken := mock.CreatePolicyAndToken(t, s.State(), 30, "test-node-get-service-reg", 1584 mock.NamespacePolicy(service.Namespace, "read", nil)).SecretID 1585 1586 // Perform a lookup using the allocation information. 1587 serviceRegReq := &structs.AllocServiceRegistrationsRequest{ 1588 AllocID: allocID, 1589 QueryOptions: structs.QueryOptions{ 1590 Namespace: service.Namespace, 1591 Region: s.Region(), 1592 AuthToken: authToken, 1593 }, 1594 } 1595 var serviceRegResp structs.AllocServiceRegistrationsResponse 1596 err = msgpackrpc.CallWithCodec(codec, structs.AllocServiceRegistrationsRPCMethod, serviceRegReq, &serviceRegResp) 1597 require.NoError(t, err) 1598 require.ElementsMatch(t, serviceRegResp.Services, []*structs.ServiceRegistration{service}) 1599 }, 1600 name: "ACLs enabled use read namespace policy token", 1601 }, 1602 { 1603 serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { 1604 return TestACLServer(t, nil) 1605 }, 1606 testFn: func(t *testing.T, s *Server, _ *structs.ACLToken) { 1607 codec := rpcClient(t, s) 1608 testutil.WaitForLeader(t, s.RPC) 1609 1610 err, allocID, service := correctSetupFn(s) 1611 require.NoError(t, err) 1612 1613 // Create and policy and grab the auth token. 1614 authToken := mock.CreatePolicyAndToken(t, s.State(), 30, "test-node-get-service-reg", 1615 mock.NamespacePolicy("ohno", "read", nil)).SecretID 1616 1617 // Perform a lookup using the allocation information. 1618 serviceRegReq := &structs.AllocServiceRegistrationsRequest{ 1619 AllocID: allocID, 1620 QueryOptions: structs.QueryOptions{ 1621 Namespace: service.Namespace, 1622 Region: s.Region(), 1623 AuthToken: authToken, 1624 }, 1625 } 1626 var serviceRegResp structs.AllocServiceRegistrationsResponse 1627 err = msgpackrpc.CallWithCodec(codec, structs.AllocServiceRegistrationsRPCMethod, serviceRegReq, &serviceRegResp) 1628 require.Error(t, err) 1629 require.Contains(t, err.Error(), "Permission denied") 1630 require.Empty(t, serviceRegResp.Services) 1631 }, 1632 name: "ACLs enabled use read incorrect namespace policy token", 1633 }, 1634 { 1635 serverFn: func(t *testing.T) (*Server, *structs.ACLToken, func()) { 1636 return TestACLServer(t, nil) 1637 }, 1638 testFn: func(t *testing.T, s *Server, _ *structs.ACLToken) { 1639 codec := rpcClient(t, s) 1640 testutil.WaitForLeader(t, s.RPC) 1641 1642 err, allocID, service := correctSetupFn(s) 1643 require.NoError(t, err) 1644 1645 // Create and policy and grab the auth token. 1646 authToken := mock.CreatePolicyAndToken(t, s.State(), 30, "test-node-get-service-reg", 1647 mock.NamespacePolicy(service.Namespace, "", []string{acl.NamespaceCapabilityReadScalingPolicy})).SecretID 1648 1649 // Perform a lookup using the allocation information. 1650 serviceRegReq := &structs.AllocServiceRegistrationsRequest{ 1651 AllocID: allocID, 1652 QueryOptions: structs.QueryOptions{ 1653 Namespace: service.Namespace, 1654 Region: s.Region(), 1655 AuthToken: authToken, 1656 }, 1657 } 1658 var serviceRegResp structs.AllocServiceRegistrationsResponse 1659 err = msgpackrpc.CallWithCodec(codec, structs.AllocServiceRegistrationsRPCMethod, serviceRegReq, &serviceRegResp) 1660 require.Error(t, err) 1661 require.Contains(t, err.Error(), "Permission denied") 1662 require.Empty(t, serviceRegResp.Services) 1663 }, 1664 name: "ACLs enabled use incorrect capability", 1665 }, 1666 } 1667 1668 for _, tc := range testCases { 1669 t.Run(tc.name, func(t *testing.T) { 1670 server, aclToken, cleanup := tc.serverFn(t) 1671 defer cleanup() 1672 tc.testFn(t, server, aclToken) 1673 }) 1674 } 1675 }