github.com/aminovpavel/nomad@v0.11.8/nomad/deployment_endpoint_test.go (about) 1 package nomad 2 3 import ( 4 "testing" 5 "time" 6 7 memdb "github.com/hashicorp/go-memdb" 8 msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc" 9 "github.com/hashicorp/nomad/acl" 10 "github.com/hashicorp/nomad/helper" 11 "github.com/hashicorp/nomad/nomad/mock" 12 "github.com/hashicorp/nomad/nomad/structs" 13 "github.com/hashicorp/nomad/testutil" 14 "github.com/stretchr/testify/assert" 15 ) 16 17 func TestDeploymentEndpoint_GetDeployment(t *testing.T) { 18 t.Parallel() 19 20 s1, cleanupS1 := TestServer(t, nil) 21 defer cleanupS1() 22 codec := rpcClient(t, s1) 23 testutil.WaitForLeader(t, s1.RPC) 24 assert := assert.New(t) 25 26 // Create the deployment 27 j := mock.Job() 28 d := mock.Deployment() 29 d.JobID = j.ID 30 state := s1.fsm.State() 31 32 assert.Nil(state.UpsertJob(999, j), "UpsertJob") 33 assert.Nil(state.UpsertDeployment(1000, d), "UpsertDeployment") 34 35 // Lookup the deployments 36 get := &structs.DeploymentSpecificRequest{ 37 DeploymentID: d.ID, 38 QueryOptions: structs.QueryOptions{ 39 Region: "global", 40 Namespace: structs.DefaultNamespace, 41 }, 42 } 43 var resp structs.SingleDeploymentResponse 44 assert.Nil(msgpackrpc.CallWithCodec(codec, "Deployment.GetDeployment", get, &resp), "RPC") 45 assert.EqualValues(resp.Index, 1000, "resp.Index") 46 assert.Equal(d, resp.Deployment, "Returned deployment not equal") 47 } 48 49 func TestDeploymentEndpoint_GetDeployment_ACL(t *testing.T) { 50 t.Parallel() 51 52 s1, root, cleanupS1 := TestACLServer(t, nil) 53 defer cleanupS1() 54 codec := rpcClient(t, s1) 55 testutil.WaitForLeader(t, s1.RPC) 56 assert := assert.New(t) 57 58 // Create the deployment 59 j := mock.Job() 60 d := mock.Deployment() 61 d.JobID = j.ID 62 state := s1.fsm.State() 63 64 assert.Nil(state.UpsertJob(999, j), "UpsertJob") 65 assert.Nil(state.UpsertDeployment(1000, d), "UpsertDeployment") 66 67 // Create the namespace policy and tokens 68 validToken := mock.CreatePolicyAndToken(t, state, 1001, "test-valid", 69 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 70 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 71 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 72 73 // Lookup the deployments without a token and expect failure 74 get := &structs.DeploymentSpecificRequest{ 75 DeploymentID: d.ID, 76 QueryOptions: structs.QueryOptions{ 77 Region: "global", 78 Namespace: structs.DefaultNamespace, 79 }, 80 } 81 var resp structs.SingleDeploymentResponse 82 assert.NotNil(msgpackrpc.CallWithCodec(codec, "Deployment.GetDeployment", get, &resp), "RPC") 83 84 // Try with a good token 85 get.AuthToken = validToken.SecretID 86 assert.Nil(msgpackrpc.CallWithCodec(codec, "Deployment.GetDeployment", get, &resp), "RPC") 87 assert.EqualValues(resp.Index, 1000, "resp.Index") 88 assert.Equal(d, resp.Deployment, "Returned deployment not equal") 89 90 // Try with a bad token 91 get.AuthToken = invalidToken.SecretID 92 err := msgpackrpc.CallWithCodec(codec, "Deployment.GetDeployment", get, &resp) 93 assert.NotNil(err, "RPC") 94 assert.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 95 96 // Try with a root token 97 get.AuthToken = root.SecretID 98 assert.Nil(msgpackrpc.CallWithCodec(codec, "Deployment.GetDeployment", get, &resp), "RPC") 99 assert.EqualValues(resp.Index, 1000, "resp.Index") 100 assert.Equal(d, resp.Deployment, "Returned deployment not equal") 101 } 102 103 func TestDeploymentEndpoint_GetDeployment_Blocking(t *testing.T) { 104 t.Parallel() 105 106 s1, cleanupS1 := TestServer(t, nil) 107 defer cleanupS1() 108 codec := rpcClient(t, s1) 109 testutil.WaitForLeader(t, s1.RPC) 110 state := s1.fsm.State() 111 assert := assert.New(t) 112 113 // Create the deployments 114 j1 := mock.Job() 115 j2 := mock.Job() 116 d1 := mock.Deployment() 117 d1.JobID = j1.ID 118 d2 := mock.Deployment() 119 d2.JobID = j2.ID 120 121 assert.Nil(state.UpsertJob(98, j1), "UpsertJob") 122 assert.Nil(state.UpsertJob(99, j2), "UpsertJob") 123 124 // Upsert a deployment we are not interested in first. 125 time.AfterFunc(100*time.Millisecond, func() { 126 assert.Nil(state.UpsertDeployment(100, d1), "UpsertDeployment") 127 }) 128 129 // Upsert another deployment later which should trigger the watch. 130 time.AfterFunc(200*time.Millisecond, func() { 131 assert.Nil(state.UpsertDeployment(200, d2), "UpsertDeployment") 132 }) 133 134 // Lookup the deployments 135 get := &structs.DeploymentSpecificRequest{ 136 DeploymentID: d2.ID, 137 QueryOptions: structs.QueryOptions{ 138 Region: "global", 139 Namespace: structs.DefaultNamespace, 140 MinQueryIndex: 150, 141 }, 142 } 143 start := time.Now() 144 var resp structs.SingleDeploymentResponse 145 assert.Nil(msgpackrpc.CallWithCodec(codec, "Deployment.GetDeployment", get, &resp), "RPC") 146 if elapsed := time.Since(start); elapsed < 200*time.Millisecond { 147 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 148 } 149 assert.EqualValues(resp.Index, 200, "resp.Index") 150 assert.Equal(d2, resp.Deployment, "deployments equal") 151 } 152 153 func TestDeploymentEndpoint_Fail(t *testing.T) { 154 t.Parallel() 155 156 s1, cleanupS1 := TestServer(t, func(c *Config) { 157 c.NumSchedulers = 0 // Prevent automatic dequeue 158 }) 159 defer cleanupS1() 160 codec := rpcClient(t, s1) 161 testutil.WaitForLeader(t, s1.RPC) 162 assert := assert.New(t) 163 164 // Create the deployment 165 j := mock.Job() 166 d := mock.Deployment() 167 d.JobID = j.ID 168 state := s1.fsm.State() 169 170 assert.Nil(state.UpsertJob(999, j), "UpsertJob") 171 assert.Nil(state.UpsertDeployment(1000, d), "UpsertDeployment") 172 173 // Mark the deployment as failed 174 req := &structs.DeploymentFailRequest{ 175 DeploymentID: d.ID, 176 WriteRequest: structs.WriteRequest{Region: "global"}, 177 } 178 179 // Fetch the response 180 var resp structs.DeploymentUpdateResponse 181 assert.Nil(msgpackrpc.CallWithCodec(codec, "Deployment.Fail", req, &resp), "RPC") 182 assert.NotEqual(resp.Index, uint64(0), "bad response index") 183 184 // Lookup the evaluation 185 ws := memdb.NewWatchSet() 186 eval, err := state.EvalByID(ws, resp.EvalID) 187 assert.Nil(err, "EvalByID failed") 188 assert.NotNil(eval, "Expect eval") 189 assert.Equal(eval.CreateIndex, resp.EvalCreateIndex, "eval index mismatch") 190 assert.Equal(eval.TriggeredBy, structs.EvalTriggerDeploymentWatcher, "eval trigger") 191 assert.Equal(eval.JobID, d.JobID, "eval job id") 192 assert.Equal(eval.DeploymentID, d.ID, "eval deployment id") 193 assert.Equal(eval.Status, structs.EvalStatusPending, "eval status") 194 195 // Lookup the deployment 196 dout, err := state.DeploymentByID(ws, d.ID) 197 assert.Nil(err, "DeploymentByID failed") 198 assert.Equal(dout.Status, structs.DeploymentStatusFailed, "wrong status") 199 assert.Equal(dout.StatusDescription, structs.DeploymentStatusDescriptionFailedByUser, "wrong status description") 200 assert.Equal(dout.ModifyIndex, resp.DeploymentModifyIndex, "wrong modify index") 201 } 202 203 func TestDeploymentEndpoint_Fail_ACL(t *testing.T) { 204 t.Parallel() 205 206 s1, _, cleanupS1 := TestACLServer(t, func(c *Config) { 207 c.NumSchedulers = 0 // Prevent automatic dequeue 208 }) 209 defer cleanupS1() 210 codec := rpcClient(t, s1) 211 testutil.WaitForLeader(t, s1.RPC) 212 assert := assert.New(t) 213 214 // Create the deployment 215 j := mock.Job() 216 d := mock.Deployment() 217 d.JobID = j.ID 218 state := s1.fsm.State() 219 220 assert.Nil(state.UpsertJob(999, j), "UpsertJob") 221 assert.Nil(state.UpsertDeployment(1000, d), "UpsertDeployment") 222 223 // Create the namespace policy and tokens 224 validToken := mock.CreatePolicyAndToken(t, state, 1001, "test-valid", 225 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilitySubmitJob})) 226 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 227 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 228 229 // Mark the deployment as failed 230 req := &structs.DeploymentFailRequest{ 231 DeploymentID: d.ID, 232 WriteRequest: structs.WriteRequest{Region: "global"}, 233 } 234 235 // Try with no token and expect permission denied 236 { 237 var resp structs.DeploymentUpdateResponse 238 err := msgpackrpc.CallWithCodec(codec, "Deployment.Fail", req, &resp) 239 assert.NotNil(err) 240 assert.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 241 } 242 243 // Try with an invalid token 244 { 245 req.AuthToken = invalidToken.SecretID 246 var resp structs.DeploymentUpdateResponse 247 err := msgpackrpc.CallWithCodec(codec, "Deployment.Fail", req, &resp) 248 assert.NotNil(err) 249 assert.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 250 } 251 252 // Try with a valid token 253 { 254 req.AuthToken = validToken.SecretID 255 var resp structs.DeploymentUpdateResponse 256 assert.Nil(msgpackrpc.CallWithCodec(codec, "Deployment.Fail", req, &resp), "RPC") 257 assert.NotEqual(resp.Index, uint64(0), "bad response index") 258 259 // Lookup the evaluation 260 ws := memdb.NewWatchSet() 261 eval, err := state.EvalByID(ws, resp.EvalID) 262 assert.Nil(err, "EvalByID failed") 263 assert.NotNil(eval, "Expect eval") 264 assert.Equal(eval.CreateIndex, resp.EvalCreateIndex, "eval index mismatch") 265 assert.Equal(eval.TriggeredBy, structs.EvalTriggerDeploymentWatcher, "eval trigger") 266 assert.Equal(eval.JobID, d.JobID, "eval job id") 267 assert.Equal(eval.DeploymentID, d.ID, "eval deployment id") 268 assert.Equal(eval.Status, structs.EvalStatusPending, "eval status") 269 270 // Lookup the deployment 271 dout, err := state.DeploymentByID(ws, d.ID) 272 assert.Nil(err, "DeploymentByID failed") 273 assert.Equal(dout.Status, structs.DeploymentStatusFailed, "wrong status") 274 assert.Equal(dout.StatusDescription, structs.DeploymentStatusDescriptionFailedByUser, "wrong status description") 275 assert.Equal(dout.ModifyIndex, resp.DeploymentModifyIndex, "wrong modify index") 276 } 277 } 278 279 func TestDeploymentEndpoint_Fail_Rollback(t *testing.T) { 280 t.Parallel() 281 282 s1, cleanupS1 := TestServer(t, func(c *Config) { 283 c.NumSchedulers = 0 // Prevent automatic dequeue 284 }) 285 defer cleanupS1() 286 codec := rpcClient(t, s1) 287 testutil.WaitForLeader(t, s1.RPC) 288 assert := assert.New(t) 289 state := s1.fsm.State() 290 291 // Create the original job 292 j := mock.Job() 293 j.Stable = true 294 j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() 295 j.TaskGroups[0].Update.MaxParallel = 2 296 j.TaskGroups[0].Update.AutoRevert = true 297 assert.Nil(state.UpsertJob(998, j), "UpsertJob") 298 299 // Create the second job, deployment and alloc 300 j2 := j.Copy() 301 j2.Stable = false 302 // Modify the job to make its specification different 303 j2.Meta["foo"] = "bar" 304 305 d := mock.Deployment() 306 d.TaskGroups["web"].AutoRevert = true 307 d.JobID = j2.ID 308 d.JobVersion = j2.Version 309 310 a := mock.Alloc() 311 a.JobID = j.ID 312 a.DeploymentID = d.ID 313 314 assert.Nil(state.UpsertJob(999, j2), "UpsertJob") 315 assert.Nil(state.UpsertDeployment(1000, d), "UpsertDeployment") 316 assert.Nil(state.UpsertAllocs(1001, []*structs.Allocation{a}), "UpsertAllocs") 317 318 // Mark the deployment as failed 319 req := &structs.DeploymentFailRequest{ 320 DeploymentID: d.ID, 321 WriteRequest: structs.WriteRequest{Region: "global"}, 322 } 323 324 // Fetch the response 325 var resp structs.DeploymentUpdateResponse 326 assert.Nil(msgpackrpc.CallWithCodec(codec, "Deployment.Fail", req, &resp), "RPC") 327 assert.NotEqual(resp.Index, uint64(0), "bad response index") 328 assert.NotNil(resp.RevertedJobVersion, "bad revert version") 329 assert.EqualValues(0, *resp.RevertedJobVersion, "bad revert version") 330 331 // Lookup the evaluation 332 ws := memdb.NewWatchSet() 333 eval, err := state.EvalByID(ws, resp.EvalID) 334 assert.Nil(err, "EvalByID failed") 335 assert.NotNil(eval, "Expect eval") 336 assert.Equal(eval.CreateIndex, resp.EvalCreateIndex, "eval index mismatch") 337 assert.Equal(eval.TriggeredBy, structs.EvalTriggerDeploymentWatcher, "eval trigger") 338 assert.Equal(eval.JobID, d.JobID, "eval job id") 339 assert.Equal(eval.DeploymentID, d.ID, "eval deployment id") 340 assert.Equal(eval.Status, structs.EvalStatusPending, "eval status") 341 342 // Lookup the deployment 343 expectedDesc := structs.DeploymentStatusDescriptionRollback(structs.DeploymentStatusDescriptionFailedByUser, 0) 344 dout, err := state.DeploymentByID(ws, d.ID) 345 assert.Nil(err, "DeploymentByID failed") 346 assert.Equal(dout.Status, structs.DeploymentStatusFailed, "wrong status") 347 assert.Equal(dout.StatusDescription, expectedDesc, "wrong status description") 348 assert.Equal(resp.DeploymentModifyIndex, dout.ModifyIndex, "wrong modify index") 349 350 // Lookup the job 351 jout, err := state.JobByID(ws, j.Namespace, j.ID) 352 assert.Nil(err, "JobByID") 353 assert.NotNil(jout, "job") 354 assert.EqualValues(2, jout.Version, "reverted job version") 355 } 356 357 func TestDeploymentEndpoint_Pause(t *testing.T) { 358 t.Parallel() 359 360 s1, cleanupS1 := TestServer(t, func(c *Config) { 361 c.NumSchedulers = 0 // Prevent automatic dequeue 362 }) 363 defer cleanupS1() 364 codec := rpcClient(t, s1) 365 testutil.WaitForLeader(t, s1.RPC) 366 assert := assert.New(t) 367 368 // Create the deployment 369 j := mock.Job() 370 d := mock.Deployment() 371 d.JobID = j.ID 372 state := s1.fsm.State() 373 374 assert.Nil(state.UpsertJob(999, j), "UpsertJob") 375 assert.Nil(state.UpsertDeployment(1000, d), "UpsertDeployment") 376 377 // Mark the deployment as failed 378 req := &structs.DeploymentPauseRequest{ 379 DeploymentID: d.ID, 380 Pause: true, 381 WriteRequest: structs.WriteRequest{Region: "global"}, 382 } 383 384 // Fetch the response 385 var resp structs.DeploymentUpdateResponse 386 assert.Nil(msgpackrpc.CallWithCodec(codec, "Deployment.Pause", req, &resp), "RPC") 387 assert.NotEqual(resp.Index, uint64(0), "bad response index") 388 assert.Zero(resp.EvalCreateIndex, "Shouldn't create eval") 389 assert.Zero(resp.EvalID, "Shouldn't create eval") 390 391 // Lookup the deployment 392 ws := memdb.NewWatchSet() 393 dout, err := state.DeploymentByID(ws, d.ID) 394 assert.Nil(err, "DeploymentByID failed") 395 assert.Equal(dout.Status, structs.DeploymentStatusPaused, "wrong status") 396 assert.Equal(dout.StatusDescription, structs.DeploymentStatusDescriptionPaused, "wrong status description") 397 assert.Equal(dout.ModifyIndex, resp.DeploymentModifyIndex, "wrong modify index") 398 } 399 400 func TestDeploymentEndpoint_Pause_ACL(t *testing.T) { 401 t.Parallel() 402 403 s1, _, cleanupS1 := TestACLServer(t, func(c *Config) { 404 c.NumSchedulers = 0 // Prevent automatic dequeue 405 }) 406 defer cleanupS1() 407 codec := rpcClient(t, s1) 408 testutil.WaitForLeader(t, s1.RPC) 409 assert := assert.New(t) 410 411 // Create the deployment 412 j := mock.Job() 413 d := mock.Deployment() 414 d.JobID = j.ID 415 state := s1.fsm.State() 416 417 assert.Nil(state.UpsertJob(999, j), "UpsertJob") 418 assert.Nil(state.UpsertDeployment(1000, d), "UpsertDeployment") 419 420 // Create the namespace policy and tokens 421 validToken := mock.CreatePolicyAndToken(t, state, 1001, "test-valid", 422 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilitySubmitJob})) 423 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 424 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 425 426 // Mark the deployment as failed 427 req := &structs.DeploymentPauseRequest{ 428 DeploymentID: d.ID, 429 Pause: true, 430 WriteRequest: structs.WriteRequest{Region: "global"}, 431 } 432 433 // Try with no token and expect permission denied 434 { 435 var resp structs.DeploymentUpdateResponse 436 err := msgpackrpc.CallWithCodec(codec, "Deployment.Pause", req, &resp) 437 assert.NotNil(err) 438 assert.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 439 } 440 441 // Try with an invalid token 442 { 443 req.AuthToken = invalidToken.SecretID 444 var resp structs.DeploymentUpdateResponse 445 err := msgpackrpc.CallWithCodec(codec, "Deployment.Pause", req, &resp) 446 assert.NotNil(err) 447 assert.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 448 } 449 450 // Fetch the response with a valid token 451 { 452 req.AuthToken = validToken.SecretID 453 var resp structs.DeploymentUpdateResponse 454 assert.Nil(msgpackrpc.CallWithCodec(codec, "Deployment.Pause", req, &resp), "RPC") 455 assert.NotEqual(resp.Index, uint64(0), "bad response index") 456 assert.Zero(resp.EvalCreateIndex, "Shouldn't create eval") 457 assert.Zero(resp.EvalID, "Shouldn't create eval") 458 459 // Lookup the deployment 460 ws := memdb.NewWatchSet() 461 dout, err := state.DeploymentByID(ws, d.ID) 462 assert.Nil(err, "DeploymentByID failed") 463 assert.Equal(dout.Status, structs.DeploymentStatusPaused, "wrong status") 464 assert.Equal(dout.StatusDescription, structs.DeploymentStatusDescriptionPaused, "wrong status description") 465 assert.Equal(dout.ModifyIndex, resp.DeploymentModifyIndex, "wrong modify index") 466 } 467 } 468 469 func TestDeploymentEndpoint_Promote(t *testing.T) { 470 t.Parallel() 471 472 s1, cleanupS1 := TestServer(t, func(c *Config) { 473 c.NumSchedulers = 0 // Prevent automatic dequeue 474 }) 475 defer cleanupS1() 476 codec := rpcClient(t, s1) 477 testutil.WaitForLeader(t, s1.RPC) 478 assert := assert.New(t) 479 480 // Create the deployment, job and canary 481 j := mock.Job() 482 j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() 483 j.TaskGroups[0].Update.MaxParallel = 2 484 j.TaskGroups[0].Update.Canary = 1 485 d := mock.Deployment() 486 d.TaskGroups["web"].DesiredCanaries = 1 487 d.JobID = j.ID 488 a := mock.Alloc() 489 d.TaskGroups[a.TaskGroup].PlacedCanaries = []string{a.ID} 490 a.DeploymentID = d.ID 491 a.DeploymentStatus = &structs.AllocDeploymentStatus{ 492 Healthy: helper.BoolToPtr(true), 493 } 494 495 state := s1.fsm.State() 496 assert.Nil(state.UpsertJob(999, j), "UpsertJob") 497 assert.Nil(state.UpsertDeployment(1000, d), "UpsertDeployment") 498 assert.Nil(state.UpsertAllocs(1001, []*structs.Allocation{a}), "UpsertAllocs") 499 500 // Promote the deployment 501 req := &structs.DeploymentPromoteRequest{ 502 DeploymentID: d.ID, 503 All: true, 504 WriteRequest: structs.WriteRequest{Region: "global"}, 505 } 506 507 // Fetch the response 508 var resp structs.DeploymentUpdateResponse 509 assert.Nil(msgpackrpc.CallWithCodec(codec, "Deployment.Promote", req, &resp), "RPC") 510 assert.NotEqual(resp.Index, uint64(0), "bad response index") 511 512 // Lookup the evaluation 513 ws := memdb.NewWatchSet() 514 eval, err := state.EvalByID(ws, resp.EvalID) 515 assert.Nil(err, "EvalByID failed") 516 assert.NotNil(eval, "Expect eval") 517 assert.Equal(eval.CreateIndex, resp.EvalCreateIndex, "eval index mismatch") 518 assert.Equal(eval.TriggeredBy, structs.EvalTriggerDeploymentWatcher, "eval trigger") 519 assert.Equal(eval.JobID, d.JobID, "eval job id") 520 assert.Equal(eval.DeploymentID, d.ID, "eval deployment id") 521 assert.Equal(eval.Status, structs.EvalStatusPending, "eval status") 522 523 // Lookup the deployment 524 dout, err := state.DeploymentByID(ws, d.ID) 525 assert.Nil(err, "DeploymentByID failed") 526 assert.Equal(dout.Status, structs.DeploymentStatusRunning, "wrong status") 527 assert.Equal(dout.StatusDescription, structs.DeploymentStatusDescriptionRunning, "wrong status description") 528 assert.Equal(dout.ModifyIndex, resp.DeploymentModifyIndex, "wrong modify index") 529 assert.Len(dout.TaskGroups, 1, "should have one group") 530 assert.Contains(dout.TaskGroups, "web", "should have web group") 531 assert.True(dout.TaskGroups["web"].Promoted, "web group should be promoted") 532 } 533 534 func TestDeploymentEndpoint_Promote_ACL(t *testing.T) { 535 t.Parallel() 536 537 s1, _, cleanupS1 := TestACLServer(t, func(c *Config) { 538 c.NumSchedulers = 0 // Prevent automatic dequeue 539 }) 540 defer cleanupS1() 541 codec := rpcClient(t, s1) 542 testutil.WaitForLeader(t, s1.RPC) 543 assert := assert.New(t) 544 545 // Create the deployment, job and canary 546 j := mock.Job() 547 j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() 548 j.TaskGroups[0].Update.MaxParallel = 2 549 j.TaskGroups[0].Update.Canary = 1 550 d := mock.Deployment() 551 d.TaskGroups["web"].DesiredCanaries = 1 552 d.JobID = j.ID 553 a := mock.Alloc() 554 d.TaskGroups[a.TaskGroup].PlacedCanaries = []string{a.ID} 555 a.DeploymentID = d.ID 556 a.DeploymentStatus = &structs.AllocDeploymentStatus{ 557 Healthy: helper.BoolToPtr(true), 558 } 559 560 state := s1.fsm.State() 561 assert.Nil(state.UpsertJob(999, j), "UpsertJob") 562 assert.Nil(state.UpsertDeployment(1000, d), "UpsertDeployment") 563 assert.Nil(state.UpsertAllocs(1001, []*structs.Allocation{a}), "UpsertAllocs") 564 565 // Create the namespace policy and tokens 566 validToken := mock.CreatePolicyAndToken(t, state, 1001, "test-valid", 567 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilitySubmitJob})) 568 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 569 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 570 571 // Promote the deployment 572 req := &structs.DeploymentPromoteRequest{ 573 DeploymentID: d.ID, 574 All: true, 575 WriteRequest: structs.WriteRequest{Region: "global"}, 576 } 577 578 // Try with no token and expect permission denied 579 { 580 var resp structs.DeploymentUpdateResponse 581 err := msgpackrpc.CallWithCodec(codec, "Deployment.Promote", req, &resp) 582 assert.NotNil(err) 583 assert.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 584 } 585 586 // Try with an invalid token 587 { 588 req.AuthToken = invalidToken.SecretID 589 var resp structs.DeploymentUpdateResponse 590 err := msgpackrpc.CallWithCodec(codec, "Deployment.Promote", req, &resp) 591 assert.NotNil(err) 592 assert.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 593 } 594 595 // Fetch the response with a valid token 596 { 597 req.AuthToken = validToken.SecretID 598 var resp structs.DeploymentUpdateResponse 599 assert.Nil(msgpackrpc.CallWithCodec(codec, "Deployment.Promote", req, &resp), "RPC") 600 assert.NotEqual(resp.Index, uint64(0), "bad response index") 601 602 // Lookup the evaluation 603 ws := memdb.NewWatchSet() 604 eval, err := state.EvalByID(ws, resp.EvalID) 605 assert.Nil(err, "EvalByID failed") 606 assert.NotNil(eval, "Expect eval") 607 assert.Equal(eval.CreateIndex, resp.EvalCreateIndex, "eval index mismatch") 608 assert.Equal(eval.TriggeredBy, structs.EvalTriggerDeploymentWatcher, "eval trigger") 609 assert.Equal(eval.JobID, d.JobID, "eval job id") 610 assert.Equal(eval.DeploymentID, d.ID, "eval deployment id") 611 assert.Equal(eval.Status, structs.EvalStatusPending, "eval status") 612 613 // Lookup the deployment 614 dout, err := state.DeploymentByID(ws, d.ID) 615 assert.Nil(err, "DeploymentByID failed") 616 assert.Equal(dout.Status, structs.DeploymentStatusRunning, "wrong status") 617 assert.Equal(dout.StatusDescription, structs.DeploymentStatusDescriptionRunning, "wrong status description") 618 assert.Equal(dout.ModifyIndex, resp.DeploymentModifyIndex, "wrong modify index") 619 assert.Len(dout.TaskGroups, 1, "should have one group") 620 assert.Contains(dout.TaskGroups, "web", "should have web group") 621 assert.True(dout.TaskGroups["web"].Promoted, "web group should be promoted") 622 } 623 } 624 625 func TestDeploymentEndpoint_SetAllocHealth(t *testing.T) { 626 t.Parallel() 627 628 s1, cleanupS1 := TestServer(t, func(c *Config) { 629 c.NumSchedulers = 0 // Prevent automatic dequeue 630 }) 631 defer cleanupS1() 632 codec := rpcClient(t, s1) 633 testutil.WaitForLeader(t, s1.RPC) 634 assert := assert.New(t) 635 636 // Create the deployment, job and canary 637 j := mock.Job() 638 j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() 639 j.TaskGroups[0].Update.MaxParallel = 2 640 d := mock.Deployment() 641 d.JobID = j.ID 642 a := mock.Alloc() 643 a.JobID = j.ID 644 a.DeploymentID = d.ID 645 646 state := s1.fsm.State() 647 assert.Nil(state.UpsertJob(999, j), "UpsertJob") 648 assert.Nil(state.UpsertDeployment(1000, d), "UpsertDeployment") 649 assert.Nil(state.UpsertAllocs(1001, []*structs.Allocation{a}), "UpsertAllocs") 650 651 // Set the alloc as healthy 652 req := &structs.DeploymentAllocHealthRequest{ 653 DeploymentID: d.ID, 654 HealthyAllocationIDs: []string{a.ID}, 655 WriteRequest: structs.WriteRequest{Region: "global"}, 656 } 657 658 // Fetch the response 659 var resp structs.DeploymentUpdateResponse 660 assert.Nil(msgpackrpc.CallWithCodec(codec, "Deployment.SetAllocHealth", req, &resp), "RPC") 661 assert.NotZero(resp.Index, "bad response index") 662 663 // Lookup the evaluation 664 ws := memdb.NewWatchSet() 665 eval, err := state.EvalByID(ws, resp.EvalID) 666 assert.Nil(err, "EvalByID failed") 667 assert.NotNil(eval, "Expect eval") 668 assert.Equal(eval.CreateIndex, resp.EvalCreateIndex, "eval index mismatch") 669 assert.Equal(eval.TriggeredBy, structs.EvalTriggerDeploymentWatcher, "eval trigger") 670 assert.Equal(eval.JobID, d.JobID, "eval job id") 671 assert.Equal(eval.DeploymentID, d.ID, "eval deployment id") 672 assert.Equal(eval.Status, structs.EvalStatusPending, "eval status") 673 674 // Lookup the deployment 675 dout, err := state.DeploymentByID(ws, d.ID) 676 assert.Nil(err, "DeploymentByID failed") 677 assert.Equal(dout.Status, structs.DeploymentStatusRunning, "wrong status") 678 assert.Equal(dout.StatusDescription, structs.DeploymentStatusDescriptionRunning, "wrong status description") 679 assert.Equal(resp.DeploymentModifyIndex, dout.ModifyIndex, "wrong modify index") 680 assert.Len(dout.TaskGroups, 1, "should have one group") 681 assert.Contains(dout.TaskGroups, "web", "should have web group") 682 assert.Equal(1, dout.TaskGroups["web"].HealthyAllocs, "should have one healthy") 683 684 // Lookup the allocation 685 aout, err := state.AllocByID(ws, a.ID) 686 assert.Nil(err, "AllocByID") 687 assert.NotNil(aout, "alloc") 688 assert.NotNil(aout.DeploymentStatus, "alloc deployment status") 689 assert.NotNil(aout.DeploymentStatus.Healthy, "alloc deployment healthy") 690 assert.True(*aout.DeploymentStatus.Healthy, "alloc deployment healthy") 691 } 692 693 func TestDeploymentEndpoint_SetAllocHealth_ACL(t *testing.T) { 694 t.Parallel() 695 696 s1, _, cleanupS1 := TestACLServer(t, func(c *Config) { 697 c.NumSchedulers = 0 // Prevent automatic dequeue 698 }) 699 defer cleanupS1() 700 codec := rpcClient(t, s1) 701 testutil.WaitForLeader(t, s1.RPC) 702 assert := assert.New(t) 703 704 // Create the deployment, job and canary 705 j := mock.Job() 706 j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() 707 j.TaskGroups[0].Update.MaxParallel = 2 708 d := mock.Deployment() 709 d.JobID = j.ID 710 a := mock.Alloc() 711 a.JobID = j.ID 712 a.DeploymentID = d.ID 713 714 state := s1.fsm.State() 715 assert.Nil(state.UpsertJob(999, j), "UpsertJob") 716 assert.Nil(state.UpsertDeployment(1000, d), "UpsertDeployment") 717 assert.Nil(state.UpsertAllocs(1001, []*structs.Allocation{a}), "UpsertAllocs") 718 719 // Create the namespace policy and tokens 720 validToken := mock.CreatePolicyAndToken(t, state, 1001, "test-valid", 721 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilitySubmitJob})) 722 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 723 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 724 725 // Set the alloc as healthy 726 req := &structs.DeploymentAllocHealthRequest{ 727 DeploymentID: d.ID, 728 HealthyAllocationIDs: []string{a.ID}, 729 WriteRequest: structs.WriteRequest{Region: "global"}, 730 } 731 732 // Try with no token and expect permission denied 733 { 734 var resp structs.DeploymentUpdateResponse 735 err := msgpackrpc.CallWithCodec(codec, "Deployment.SetAllocHealth", req, &resp) 736 assert.NotNil(err) 737 assert.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 738 } 739 740 // Try with an invalid token 741 { 742 req.AuthToken = invalidToken.SecretID 743 var resp structs.DeploymentUpdateResponse 744 err := msgpackrpc.CallWithCodec(codec, "Deployment.SetAllocHealth", req, &resp) 745 assert.NotNil(err) 746 assert.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 747 } 748 749 // Fetch the response with a valid token 750 { 751 req.AuthToken = validToken.SecretID 752 var resp structs.DeploymentUpdateResponse 753 assert.Nil(msgpackrpc.CallWithCodec(codec, "Deployment.SetAllocHealth", req, &resp), "RPC") 754 assert.NotZero(resp.Index, "bad response index") 755 756 // Lookup the evaluation 757 ws := memdb.NewWatchSet() 758 eval, err := state.EvalByID(ws, resp.EvalID) 759 assert.Nil(err, "EvalByID failed") 760 assert.NotNil(eval, "Expect eval") 761 assert.Equal(eval.CreateIndex, resp.EvalCreateIndex, "eval index mismatch") 762 assert.Equal(eval.TriggeredBy, structs.EvalTriggerDeploymentWatcher, "eval trigger") 763 assert.Equal(eval.JobID, d.JobID, "eval job id") 764 assert.Equal(eval.DeploymentID, d.ID, "eval deployment id") 765 assert.Equal(eval.Status, structs.EvalStatusPending, "eval status") 766 767 // Lookup the deployment 768 dout, err := state.DeploymentByID(ws, d.ID) 769 assert.Nil(err, "DeploymentByID failed") 770 assert.Equal(dout.Status, structs.DeploymentStatusRunning, "wrong status") 771 assert.Equal(dout.StatusDescription, structs.DeploymentStatusDescriptionRunning, "wrong status description") 772 assert.Equal(resp.DeploymentModifyIndex, dout.ModifyIndex, "wrong modify index") 773 assert.Len(dout.TaskGroups, 1, "should have one group") 774 assert.Contains(dout.TaskGroups, "web", "should have web group") 775 assert.Equal(1, dout.TaskGroups["web"].HealthyAllocs, "should have one healthy") 776 777 // Lookup the allocation 778 aout, err := state.AllocByID(ws, a.ID) 779 assert.Nil(err, "AllocByID") 780 assert.NotNil(aout, "alloc") 781 assert.NotNil(aout.DeploymentStatus, "alloc deployment status") 782 assert.NotNil(aout.DeploymentStatus.Healthy, "alloc deployment healthy") 783 assert.True(*aout.DeploymentStatus.Healthy, "alloc deployment healthy") 784 } 785 } 786 787 func TestDeploymentEndpoint_SetAllocHealth_Rollback(t *testing.T) { 788 t.Parallel() 789 790 s1, cleanupS1 := TestServer(t, func(c *Config) { 791 c.NumSchedulers = 0 // Prevent automatic dequeue 792 }) 793 defer cleanupS1() 794 codec := rpcClient(t, s1) 795 testutil.WaitForLeader(t, s1.RPC) 796 assert := assert.New(t) 797 state := s1.fsm.State() 798 799 // Create the original job 800 j := mock.Job() 801 j.Stable = true 802 j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() 803 j.TaskGroups[0].Update.MaxParallel = 2 804 j.TaskGroups[0].Update.AutoRevert = true 805 assert.Nil(state.UpsertJob(998, j), "UpsertJob") 806 807 // Create the second job, deployment and alloc 808 j2 := j.Copy() 809 j2.Stable = false 810 // Modify the job to make its specification different 811 j2.Meta["foo"] = "bar" 812 d := mock.Deployment() 813 d.TaskGroups["web"].AutoRevert = true 814 d.JobID = j2.ID 815 d.JobVersion = j2.Version 816 817 a := mock.Alloc() 818 a.JobID = j.ID 819 a.DeploymentID = d.ID 820 821 assert.Nil(state.UpsertJob(999, j2), "UpsertJob") 822 assert.Nil(state.UpsertDeployment(1000, d), "UpsertDeployment") 823 assert.Nil(state.UpsertAllocs(1001, []*structs.Allocation{a}), "UpsertAllocs") 824 825 // Set the alloc as unhealthy 826 req := &structs.DeploymentAllocHealthRequest{ 827 DeploymentID: d.ID, 828 UnhealthyAllocationIDs: []string{a.ID}, 829 WriteRequest: structs.WriteRequest{Region: "global"}, 830 } 831 832 // Fetch the response 833 var resp structs.DeploymentUpdateResponse 834 assert.Nil(msgpackrpc.CallWithCodec(codec, "Deployment.SetAllocHealth", req, &resp), "RPC") 835 assert.NotZero(resp.Index, "bad response index") 836 assert.NotNil(resp.RevertedJobVersion, "bad revert version") 837 assert.EqualValues(0, *resp.RevertedJobVersion, "bad revert version") 838 839 // Lookup the evaluation 840 ws := memdb.NewWatchSet() 841 eval, err := state.EvalByID(ws, resp.EvalID) 842 assert.Nil(err, "EvalByID failed") 843 assert.NotNil(eval, "Expect eval") 844 assert.Equal(eval.CreateIndex, resp.EvalCreateIndex, "eval index mismatch") 845 assert.Equal(eval.TriggeredBy, structs.EvalTriggerDeploymentWatcher, "eval trigger") 846 assert.Equal(eval.JobID, d.JobID, "eval job id") 847 assert.Equal(eval.DeploymentID, d.ID, "eval deployment id") 848 assert.Equal(eval.Status, structs.EvalStatusPending, "eval status") 849 850 // Lookup the deployment 851 expectedDesc := structs.DeploymentStatusDescriptionRollback(structs.DeploymentStatusDescriptionFailedAllocations, 0) 852 dout, err := state.DeploymentByID(ws, d.ID) 853 assert.Nil(err, "DeploymentByID failed") 854 assert.Equal(dout.Status, structs.DeploymentStatusFailed, "wrong status") 855 assert.Equal(dout.StatusDescription, expectedDesc, "wrong status description") 856 assert.Equal(resp.DeploymentModifyIndex, dout.ModifyIndex, "wrong modify index") 857 assert.Len(dout.TaskGroups, 1, "should have one group") 858 assert.Contains(dout.TaskGroups, "web", "should have web group") 859 assert.Equal(1, dout.TaskGroups["web"].UnhealthyAllocs, "should have one healthy") 860 861 // Lookup the allocation 862 aout, err := state.AllocByID(ws, a.ID) 863 assert.Nil(err, "AllocByID") 864 assert.NotNil(aout, "alloc") 865 assert.NotNil(aout.DeploymentStatus, "alloc deployment status") 866 assert.NotNil(aout.DeploymentStatus.Healthy, "alloc deployment healthy") 867 assert.False(*aout.DeploymentStatus.Healthy, "alloc deployment healthy") 868 869 // Lookup the job 870 jout, err := state.JobByID(ws, j.Namespace, j.ID) 871 assert.Nil(err, "JobByID") 872 assert.NotNil(jout, "job") 873 assert.EqualValues(2, jout.Version, "reverted job version") 874 } 875 876 // tests rollback upon alloc health failure to job with identical spec does not succeed 877 func TestDeploymentEndpoint_SetAllocHealth_NoRollback(t *testing.T) { 878 t.Parallel() 879 880 s1, cleanupS1 := TestServer(t, func(c *Config) { 881 c.NumSchedulers = 0 // Prevent automatic dequeue 882 }) 883 defer cleanupS1() 884 codec := rpcClient(t, s1) 885 testutil.WaitForLeader(t, s1.RPC) 886 assert := assert.New(t) 887 state := s1.fsm.State() 888 889 // Create the original job 890 j := mock.Job() 891 j.Stable = true 892 j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() 893 j.TaskGroups[0].Update.MaxParallel = 2 894 j.TaskGroups[0].Update.AutoRevert = true 895 assert.Nil(state.UpsertJob(998, j), "UpsertJob") 896 897 // Create the second job, deployment and alloc. Job has same spec as original 898 j2 := j.Copy() 899 j2.Stable = false 900 901 d := mock.Deployment() 902 d.TaskGroups["web"].AutoRevert = true 903 d.JobID = j2.ID 904 d.JobVersion = j2.Version 905 906 a := mock.Alloc() 907 a.JobID = j.ID 908 a.DeploymentID = d.ID 909 910 assert.Nil(state.UpsertJob(999, j2), "UpsertJob") 911 assert.Nil(state.UpsertDeployment(1000, d), "UpsertDeployment") 912 assert.Nil(state.UpsertAllocs(1001, []*structs.Allocation{a}), "UpsertAllocs") 913 914 // Set the alloc as unhealthy 915 req := &structs.DeploymentAllocHealthRequest{ 916 DeploymentID: d.ID, 917 UnhealthyAllocationIDs: []string{a.ID}, 918 WriteRequest: structs.WriteRequest{Region: "global"}, 919 } 920 921 // Fetch the response 922 var resp structs.DeploymentUpdateResponse 923 assert.Nil(msgpackrpc.CallWithCodec(codec, "Deployment.SetAllocHealth", req, &resp), "RPC") 924 assert.NotZero(resp.Index, "bad response index") 925 assert.Nil(resp.RevertedJobVersion, "revert version must be nil") 926 927 // Lookup the evaluation 928 ws := memdb.NewWatchSet() 929 eval, err := state.EvalByID(ws, resp.EvalID) 930 assert.Nil(err, "EvalByID failed") 931 assert.NotNil(eval, "Expect eval") 932 assert.Equal(eval.CreateIndex, resp.EvalCreateIndex, "eval index mismatch") 933 assert.Equal(eval.TriggeredBy, structs.EvalTriggerDeploymentWatcher, "eval trigger") 934 assert.Equal(eval.JobID, d.JobID, "eval job id") 935 assert.Equal(eval.DeploymentID, d.ID, "eval deployment id") 936 assert.Equal(eval.Status, structs.EvalStatusPending, "eval status") 937 938 // Lookup the deployment 939 expectedDesc := structs.DeploymentStatusDescriptionRollbackNoop(structs.DeploymentStatusDescriptionFailedAllocations, 0) 940 dout, err := state.DeploymentByID(ws, d.ID) 941 assert.Nil(err, "DeploymentByID failed") 942 assert.Equal(dout.Status, structs.DeploymentStatusFailed, "wrong status") 943 assert.Equal(dout.StatusDescription, expectedDesc, "wrong status description") 944 assert.Equal(resp.DeploymentModifyIndex, dout.ModifyIndex, "wrong modify index") 945 assert.Len(dout.TaskGroups, 1, "should have one group") 946 assert.Contains(dout.TaskGroups, "web", "should have web group") 947 assert.Equal(1, dout.TaskGroups["web"].UnhealthyAllocs, "should have one healthy") 948 949 // Lookup the allocation 950 aout, err := state.AllocByID(ws, a.ID) 951 assert.Nil(err, "AllocByID") 952 assert.NotNil(aout, "alloc") 953 assert.NotNil(aout.DeploymentStatus, "alloc deployment status") 954 assert.NotNil(aout.DeploymentStatus.Healthy, "alloc deployment healthy") 955 assert.False(*aout.DeploymentStatus.Healthy, "alloc deployment healthy") 956 957 // Lookup the job, its version should not have changed 958 jout, err := state.JobByID(ws, j.Namespace, j.ID) 959 assert.Nil(err, "JobByID") 960 assert.NotNil(jout, "job") 961 assert.EqualValues(1, jout.Version, "original job version") 962 } 963 964 func TestDeploymentEndpoint_List(t *testing.T) { 965 t.Parallel() 966 967 s1, cleanupS1 := TestServer(t, nil) 968 defer cleanupS1() 969 codec := rpcClient(t, s1) 970 testutil.WaitForLeader(t, s1.RPC) 971 assert := assert.New(t) 972 973 // Create the register request 974 j := mock.Job() 975 d := mock.Deployment() 976 d.JobID = j.ID 977 state := s1.fsm.State() 978 979 assert.Nil(state.UpsertJob(999, j), "UpsertJob") 980 assert.Nil(state.UpsertDeployment(1000, d), "UpsertDeployment") 981 982 // Lookup the deployments 983 get := &structs.DeploymentListRequest{ 984 QueryOptions: structs.QueryOptions{ 985 Region: "global", 986 Namespace: structs.DefaultNamespace, 987 }, 988 } 989 var resp structs.DeploymentListResponse 990 assert.Nil(msgpackrpc.CallWithCodec(codec, "Deployment.List", get, &resp), "RPC") 991 assert.EqualValues(resp.Index, 1000, "Wrong Index") 992 assert.Len(resp.Deployments, 1, "Deployments") 993 assert.Equal(resp.Deployments[0].ID, d.ID, "Deployment ID") 994 995 // Lookup the deploys by prefix 996 get = &structs.DeploymentListRequest{ 997 QueryOptions: structs.QueryOptions{ 998 Region: "global", 999 Namespace: structs.DefaultNamespace, 1000 Prefix: d.ID[:4], 1001 }, 1002 } 1003 1004 var resp2 structs.DeploymentListResponse 1005 assert.Nil(msgpackrpc.CallWithCodec(codec, "Deployment.List", get, &resp2), "RPC") 1006 assert.EqualValues(resp.Index, 1000, "Wrong Index") 1007 assert.Len(resp2.Deployments, 1, "Deployments") 1008 assert.Equal(resp2.Deployments[0].ID, d.ID, "Deployment ID") 1009 } 1010 1011 func TestDeploymentEndpoint_List_ACL(t *testing.T) { 1012 t.Parallel() 1013 1014 s1, root, cleanupS1 := TestACLServer(t, nil) 1015 defer cleanupS1() 1016 codec := rpcClient(t, s1) 1017 testutil.WaitForLeader(t, s1.RPC) 1018 assert := assert.New(t) 1019 1020 // Create the register request 1021 j := mock.Job() 1022 d := mock.Deployment() 1023 d.JobID = j.ID 1024 state := s1.fsm.State() 1025 1026 assert.Nil(state.UpsertJob(999, j), "UpsertJob") 1027 assert.Nil(state.UpsertDeployment(1000, d), "UpsertDeployment") 1028 1029 // Create the namespace policy and tokens 1030 validToken := mock.CreatePolicyAndToken(t, state, 1001, "test-valid", 1031 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 1032 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 1033 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 1034 1035 get := &structs.DeploymentListRequest{ 1036 QueryOptions: structs.QueryOptions{ 1037 Region: "global", 1038 Namespace: structs.DefaultNamespace, 1039 }, 1040 } 1041 1042 // Try with no token and expect permission denied 1043 { 1044 var resp structs.DeploymentUpdateResponse 1045 err := msgpackrpc.CallWithCodec(codec, "Deployment.List", get, &resp) 1046 assert.NotNil(err) 1047 assert.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 1048 } 1049 1050 // Try with an invalid token 1051 { 1052 get.AuthToken = invalidToken.SecretID 1053 var resp structs.DeploymentUpdateResponse 1054 err := msgpackrpc.CallWithCodec(codec, "Deployment.List", get, &resp) 1055 assert.NotNil(err) 1056 assert.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 1057 } 1058 1059 // Lookup the deployments with a root token 1060 { 1061 get.AuthToken = root.SecretID 1062 var resp structs.DeploymentListResponse 1063 assert.Nil(msgpackrpc.CallWithCodec(codec, "Deployment.List", get, &resp), "RPC") 1064 assert.EqualValues(resp.Index, 1000, "Wrong Index") 1065 assert.Len(resp.Deployments, 1, "Deployments") 1066 assert.Equal(resp.Deployments[0].ID, d.ID, "Deployment ID") 1067 } 1068 1069 // Lookup the deployments with a valid token 1070 { 1071 get.AuthToken = validToken.SecretID 1072 var resp structs.DeploymentListResponse 1073 assert.Nil(msgpackrpc.CallWithCodec(codec, "Deployment.List", get, &resp), "RPC") 1074 assert.EqualValues(resp.Index, 1000, "Wrong Index") 1075 assert.Len(resp.Deployments, 1, "Deployments") 1076 assert.Equal(resp.Deployments[0].ID, d.ID, "Deployment ID") 1077 } 1078 } 1079 1080 func TestDeploymentEndpoint_List_Blocking(t *testing.T) { 1081 t.Parallel() 1082 1083 s1, cleanupS1 := TestServer(t, nil) 1084 defer cleanupS1() 1085 state := s1.fsm.State() 1086 codec := rpcClient(t, s1) 1087 testutil.WaitForLeader(t, s1.RPC) 1088 assert := assert.New(t) 1089 1090 // Create the deployment 1091 j := mock.Job() 1092 d := mock.Deployment() 1093 d.JobID = j.ID 1094 1095 assert.Nil(state.UpsertJob(999, j), "UpsertJob") 1096 1097 // Upsert alloc triggers watches 1098 time.AfterFunc(100*time.Millisecond, func() { 1099 assert.Nil(state.UpsertDeployment(3, d), "UpsertDeployment") 1100 }) 1101 1102 req := &structs.DeploymentListRequest{ 1103 QueryOptions: structs.QueryOptions{ 1104 Region: "global", 1105 Namespace: structs.DefaultNamespace, 1106 MinQueryIndex: 1, 1107 }, 1108 } 1109 start := time.Now() 1110 var resp structs.DeploymentListResponse 1111 assert.Nil(msgpackrpc.CallWithCodec(codec, "Deployment.List", req, &resp), "RPC") 1112 assert.EqualValues(resp.Index, 3, "Wrong Index") 1113 assert.Len(resp.Deployments, 1, "Deployments") 1114 assert.Equal(resp.Deployments[0].ID, d.ID, "Deployment ID") 1115 if elapsed := time.Since(start); elapsed < 100*time.Millisecond { 1116 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 1117 } 1118 1119 // Deployment updates trigger watches 1120 d2 := d.Copy() 1121 d2.Status = structs.DeploymentStatusPaused 1122 time.AfterFunc(100*time.Millisecond, func() { 1123 assert.Nil(state.UpsertDeployment(5, d2), "UpsertDeployment") 1124 }) 1125 1126 req.MinQueryIndex = 3 1127 start = time.Now() 1128 var resp2 structs.DeploymentListResponse 1129 assert.Nil(msgpackrpc.CallWithCodec(codec, "Deployment.List", req, &resp2), "RPC") 1130 assert.EqualValues(5, resp2.Index, "Wrong Index") 1131 assert.Len(resp2.Deployments, 1, "Deployments") 1132 assert.Equal(d2.ID, resp2.Deployments[0].ID, "Deployment ID") 1133 if elapsed := time.Since(start); elapsed < 100*time.Millisecond { 1134 t.Fatalf("should block (returned in %s) %#v", elapsed, resp2) 1135 } 1136 } 1137 1138 func TestDeploymentEndpoint_Allocations(t *testing.T) { 1139 t.Parallel() 1140 1141 s1, cleanupS1 := TestServer(t, nil) 1142 defer cleanupS1() 1143 codec := rpcClient(t, s1) 1144 testutil.WaitForLeader(t, s1.RPC) 1145 assert := assert.New(t) 1146 1147 // Create the register request 1148 j := mock.Job() 1149 d := mock.Deployment() 1150 d.JobID = j.ID 1151 a := mock.Alloc() 1152 a.DeploymentID = d.ID 1153 summary := mock.JobSummary(a.JobID) 1154 state := s1.fsm.State() 1155 1156 assert.Nil(state.UpsertJob(998, j), "UpsertJob") 1157 assert.Nil(state.UpsertJobSummary(999, summary), "UpsertJobSummary") 1158 assert.Nil(state.UpsertDeployment(1000, d), "UpsertDeployment") 1159 assert.Nil(state.UpsertAllocs(1001, []*structs.Allocation{a}), "UpsertAllocs") 1160 1161 // Lookup the allocations 1162 get := &structs.DeploymentSpecificRequest{ 1163 DeploymentID: d.ID, 1164 QueryOptions: structs.QueryOptions{ 1165 Region: "global", 1166 Namespace: structs.DefaultNamespace, 1167 }, 1168 } 1169 var resp structs.AllocListResponse 1170 assert.Nil(msgpackrpc.CallWithCodec(codec, "Deployment.Allocations", get, &resp), "RPC") 1171 assert.EqualValues(1001, resp.Index, "Wrong Index") 1172 assert.Len(resp.Allocations, 1, "Allocations") 1173 assert.Equal(a.ID, resp.Allocations[0].ID, "Allocation ID") 1174 } 1175 1176 func TestDeploymentEndpoint_Allocations_ACL(t *testing.T) { 1177 t.Parallel() 1178 1179 s1, root, cleanupS1 := TestACLServer(t, nil) 1180 defer cleanupS1() 1181 codec := rpcClient(t, s1) 1182 testutil.WaitForLeader(t, s1.RPC) 1183 assert := assert.New(t) 1184 1185 // Create the register request 1186 j := mock.Job() 1187 d := mock.Deployment() 1188 d.JobID = j.ID 1189 a := mock.Alloc() 1190 a.DeploymentID = d.ID 1191 summary := mock.JobSummary(a.JobID) 1192 state := s1.fsm.State() 1193 1194 assert.Nil(state.UpsertJob(998, j), "UpsertJob") 1195 assert.Nil(state.UpsertJobSummary(999, summary), "UpsertJobSummary") 1196 assert.Nil(state.UpsertDeployment(1000, d), "UpsertDeployment") 1197 assert.Nil(state.UpsertAllocs(1001, []*structs.Allocation{a}), "UpsertAllocs") 1198 1199 // Create the namespace policy and tokens 1200 validToken := mock.CreatePolicyAndToken(t, state, 1001, "test-valid", 1201 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 1202 invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", 1203 mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) 1204 1205 get := &structs.DeploymentSpecificRequest{ 1206 DeploymentID: d.ID, 1207 QueryOptions: structs.QueryOptions{ 1208 Region: "global", 1209 Namespace: structs.DefaultNamespace, 1210 }, 1211 } 1212 1213 // Try with no token and expect permission denied 1214 { 1215 var resp structs.DeploymentUpdateResponse 1216 err := msgpackrpc.CallWithCodec(codec, "Deployment.Allocations", get, &resp) 1217 assert.NotNil(err) 1218 assert.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 1219 } 1220 1221 // Try with an invalid token 1222 { 1223 get.AuthToken = invalidToken.SecretID 1224 var resp structs.DeploymentUpdateResponse 1225 err := msgpackrpc.CallWithCodec(codec, "Deployment.Allocations", get, &resp) 1226 assert.NotNil(err) 1227 assert.Equal(err.Error(), structs.ErrPermissionDenied.Error()) 1228 } 1229 1230 // Lookup the allocations with a valid token 1231 { 1232 get.AuthToken = validToken.SecretID 1233 var resp structs.AllocListResponse 1234 assert.Nil(msgpackrpc.CallWithCodec(codec, "Deployment.Allocations", get, &resp), "RPC") 1235 assert.EqualValues(1001, resp.Index, "Wrong Index") 1236 assert.Len(resp.Allocations, 1, "Allocations") 1237 assert.Equal(a.ID, resp.Allocations[0].ID, "Allocation ID") 1238 } 1239 1240 // Lookup the allocations with a root token 1241 { 1242 get.AuthToken = root.SecretID 1243 var resp structs.AllocListResponse 1244 assert.Nil(msgpackrpc.CallWithCodec(codec, "Deployment.Allocations", get, &resp), "RPC") 1245 assert.EqualValues(1001, resp.Index, "Wrong Index") 1246 assert.Len(resp.Allocations, 1, "Allocations") 1247 assert.Equal(a.ID, resp.Allocations[0].ID, "Allocation ID") 1248 } 1249 } 1250 1251 func TestDeploymentEndpoint_Allocations_Blocking(t *testing.T) { 1252 t.Parallel() 1253 1254 s1, cleanupS1 := TestServer(t, nil) 1255 defer cleanupS1() 1256 state := s1.fsm.State() 1257 codec := rpcClient(t, s1) 1258 testutil.WaitForLeader(t, s1.RPC) 1259 assert := assert.New(t) 1260 1261 // Create the alloc 1262 j := mock.Job() 1263 d := mock.Deployment() 1264 d.JobID = j.ID 1265 a := mock.Alloc() 1266 a.DeploymentID = d.ID 1267 summary := mock.JobSummary(a.JobID) 1268 1269 assert.Nil(state.UpsertJob(1, j), "UpsertJob") 1270 assert.Nil(state.UpsertDeployment(2, d), "UpsertDeployment") 1271 assert.Nil(state.UpsertJobSummary(3, summary), "UpsertJobSummary") 1272 1273 // Upsert alloc triggers watches 1274 time.AfterFunc(100*time.Millisecond, func() { 1275 assert.Nil(state.UpsertAllocs(4, []*structs.Allocation{a}), "UpsertAllocs") 1276 }) 1277 1278 req := &structs.DeploymentSpecificRequest{ 1279 DeploymentID: d.ID, 1280 QueryOptions: structs.QueryOptions{ 1281 Region: "global", 1282 Namespace: structs.DefaultNamespace, 1283 MinQueryIndex: 1, 1284 }, 1285 } 1286 start := time.Now() 1287 var resp structs.AllocListResponse 1288 assert.Nil(msgpackrpc.CallWithCodec(codec, "Deployment.Allocations", req, &resp), "RPC") 1289 assert.EqualValues(4, resp.Index, "Wrong Index") 1290 assert.Len(resp.Allocations, 1, "Allocations") 1291 assert.Equal(a.ID, resp.Allocations[0].ID, "Allocation ID") 1292 if elapsed := time.Since(start); elapsed < 100*time.Millisecond { 1293 t.Fatalf("should block (returned in %s) %#v", elapsed, resp) 1294 } 1295 1296 // Client updates trigger watches 1297 a2 := mock.Alloc() 1298 a2.ID = a.ID 1299 a2.DeploymentID = a.DeploymentID 1300 a2.ClientStatus = structs.AllocClientStatusRunning 1301 time.AfterFunc(100*time.Millisecond, func() { 1302 assert.Nil(state.UpsertJobSummary(5, mock.JobSummary(a2.JobID)), "UpsertJobSummary") 1303 assert.Nil(state.UpdateAllocsFromClient(6, []*structs.Allocation{a2}), "updateAllocsFromClient") 1304 }) 1305 1306 req.MinQueryIndex = 4 1307 start = time.Now() 1308 var resp2 structs.AllocListResponse 1309 assert.Nil(msgpackrpc.CallWithCodec(codec, "Deployment.Allocations", req, &resp2), "RPC") 1310 assert.EqualValues(6, resp2.Index, "Wrong Index") 1311 assert.Len(resp2.Allocations, 1, "Allocations") 1312 assert.Equal(a.ID, resp2.Allocations[0].ID, "Allocation ID") 1313 assert.Equal(structs.AllocClientStatusRunning, resp2.Allocations[0].ClientStatus, "Client Status") 1314 if elapsed := time.Since(start); elapsed < 100*time.Millisecond { 1315 t.Fatalf("should block (returned in %s) %#v", elapsed, resp2) 1316 } 1317 } 1318 1319 func TestDeploymentEndpoint_Reap(t *testing.T) { 1320 t.Parallel() 1321 1322 s1, cleanupS1 := TestServer(t, nil) 1323 defer cleanupS1() 1324 codec := rpcClient(t, s1) 1325 testutil.WaitForLeader(t, s1.RPC) 1326 assert := assert.New(t) 1327 1328 // Create the register request 1329 d1 := mock.Deployment() 1330 assert.Nil(s1.fsm.State().UpsertDeployment(1000, d1), "UpsertDeployment") 1331 1332 // Reap the eval 1333 get := &structs.DeploymentDeleteRequest{ 1334 Deployments: []string{d1.ID}, 1335 WriteRequest: structs.WriteRequest{Region: "global"}, 1336 } 1337 var resp structs.GenericResponse 1338 assert.Nil(msgpackrpc.CallWithCodec(codec, "Deployment.Reap", get, &resp), "RPC") 1339 assert.NotZero(resp.Index, "bad response index") 1340 1341 // Ensure deleted 1342 ws := memdb.NewWatchSet() 1343 outD, err := s1.fsm.State().DeploymentByID(ws, d1.ID) 1344 assert.Nil(err, "DeploymentByID") 1345 assert.Nil(outD, "Deleted Deployment") 1346 }