github.com/Ilhicas/nomad@v1.0.4-0.20210304152020-e86851182bc3/nomad/stream/event_broker_test.go (about) 1 package stream 2 3 import ( 4 "context" 5 "sync/atomic" 6 "testing" 7 "time" 8 9 "github.com/hashicorp/go-memdb" 10 "github.com/hashicorp/nomad/acl" 11 "github.com/hashicorp/nomad/nomad/mock" 12 "github.com/hashicorp/nomad/nomad/structs" 13 14 "github.com/stretchr/testify/require" 15 ) 16 17 func TestEventBroker_PublishChangesAndSubscribe(t *testing.T) { 18 subscription := &SubscribeRequest{ 19 Topics: map[structs.Topic][]string{ 20 "Test": {"sub-key"}, 21 }, 22 } 23 ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) 24 defer cancel() 25 26 publisher, err := NewEventBroker(ctx, nil, EventBrokerCfg{EventBufferSize: 100}) 27 require.NoError(t, err) 28 29 sub, err := publisher.Subscribe(subscription) 30 require.NoError(t, err) 31 eventCh := consumeSubscription(ctx, sub) 32 33 // Now subscriber should block waiting for updates 34 assertNoResult(t, eventCh) 35 36 events := []structs.Event{{ 37 Index: 1, 38 Topic: "Test", 39 Key: "sub-key", 40 Payload: "sample payload", 41 }} 42 publisher.Publish(&structs.Events{Index: 1, Events: events}) 43 44 // Subscriber should see the published event 45 result := nextResult(t, eventCh) 46 require.NoError(t, result.Err) 47 expected := []structs.Event{{Payload: "sample payload", Key: "sub-key", Topic: "Test", Index: 1}} 48 require.Equal(t, expected, result.Events) 49 50 // Now subscriber should block waiting for updates 51 assertNoResult(t, eventCh) 52 53 // Publish a second event 54 events = []structs.Event{{ 55 Index: 2, 56 Topic: "Test", 57 Key: "sub-key", 58 Payload: "sample payload 2", 59 }} 60 publisher.Publish(&structs.Events{Index: 2, Events: events}) 61 62 result = nextResult(t, eventCh) 63 require.NoError(t, result.Err) 64 expected = []structs.Event{{Payload: "sample payload 2", Key: "sub-key", Topic: "Test", Index: 2}} 65 require.Equal(t, expected, result.Events) 66 } 67 68 func TestEventBroker_ShutdownClosesSubscriptions(t *testing.T) { 69 ctx, cancel := context.WithCancel(context.Background()) 70 t.Cleanup(cancel) 71 72 publisher, err := NewEventBroker(ctx, nil, EventBrokerCfg{}) 73 require.NoError(t, err) 74 75 sub1, err := publisher.Subscribe(&SubscribeRequest{}) 76 require.NoError(t, err) 77 defer sub1.Unsubscribe() 78 79 sub2, err := publisher.Subscribe(&SubscribeRequest{}) 80 require.NoError(t, err) 81 defer sub2.Unsubscribe() 82 83 cancel() // Shutdown 84 85 err = consumeSub(context.Background(), sub1) 86 require.Equal(t, err, ErrSubscriptionClosed) 87 88 _, err = sub2.Next(context.Background()) 89 require.Equal(t, err, ErrSubscriptionClosed) 90 } 91 92 // TestEventBroker_EmptyReqToken_DistinctSubscriptions tests subscription 93 // hanlding behavior when ACLs are disabled (request Token is empty). 94 // Subscriptions are mapped by their request token. when that token is empty, 95 // the subscriptions should still be handled indeppendtly of each other when 96 // unssubscribing. 97 func TestEventBroker_EmptyReqToken_DistinctSubscriptions(t *testing.T) { 98 ctx, cancel := context.WithCancel(context.Background()) 99 t.Cleanup(cancel) 100 101 publisher, err := NewEventBroker(ctx, nil, EventBrokerCfg{}) 102 require.NoError(t, err) 103 104 // first subscription, empty token 105 sub1, err := publisher.Subscribe(&SubscribeRequest{}) 106 require.NoError(t, err) 107 defer sub1.Unsubscribe() 108 109 // second subscription, empty token 110 sub2, err := publisher.Subscribe(&SubscribeRequest{}) 111 require.NoError(t, err) 112 require.NotNil(t, sub2) 113 114 sub1.Unsubscribe() 115 116 require.Equal(t, subscriptionStateOpen, atomic.LoadUint32(&sub2.state)) 117 } 118 119 func TestEventBroker_handleACLUpdates_TokenDeleted(t *testing.T) { 120 ctx, cancel := context.WithCancel(context.Background()) 121 t.Cleanup(cancel) 122 123 publisher, err := NewEventBroker(ctx, nil, EventBrokerCfg{}) 124 require.NoError(t, err) 125 126 sub1, err := publisher.Subscribe(&SubscribeRequest{ 127 Topics: map[structs.Topic][]string{ 128 "*": {"*"}, 129 }, 130 Token: "foo", 131 }) 132 require.NoError(t, err) 133 defer sub1.Unsubscribe() 134 135 aclEvent := structs.Event{ 136 Topic: structs.TopicACLToken, 137 Type: structs.TypeACLTokenDeleted, 138 Payload: structs.NewACLTokenEvent(&structs.ACLToken{SecretID: "foo"}), 139 } 140 141 publisher.Publish(&structs.Events{Index: 100, Events: []structs.Event{aclEvent}}) 142 for { 143 _, err := sub1.Next(ctx) 144 if err == ErrSubscriptionClosed { 145 break 146 } 147 } 148 149 out, err := sub1.Next(ctx) 150 require.Error(t, err) 151 require.Equal(t, ErrSubscriptionClosed, err) 152 require.Equal(t, structs.Events{}, out) 153 } 154 155 type fakeACLDelegate struct { 156 tokenProvider ACLTokenProvider 157 } 158 159 func (d *fakeACLDelegate) TokenProvider() ACLTokenProvider { 160 return d.tokenProvider 161 } 162 163 type fakeACLTokenProvider struct { 164 policy *structs.ACLPolicy 165 policyErr error 166 token *structs.ACLToken 167 tokenErr error 168 } 169 170 func (p *fakeACLTokenProvider) ACLTokenBySecretID(ws memdb.WatchSet, secretID string) (*structs.ACLToken, error) { 171 return p.token, p.tokenErr 172 } 173 174 func (p *fakeACLTokenProvider) ACLPolicyByName(ws memdb.WatchSet, policyName string) (*structs.ACLPolicy, error) { 175 return p.policy, p.policyErr 176 } 177 178 func TestEventBroker_handleACLUpdates_policyupdated(t *testing.T) { 179 ctx, cancel := context.WithCancel(context.Background()) 180 t.Cleanup(cancel) 181 182 secretID := "some-secret-id" 183 cases := []struct { 184 policyBeforeRules string 185 policyAfterRules string 186 topics map[structs.Topic][]string 187 desc string 188 event structs.Event 189 policyEvent structs.Event 190 shouldUnsubscribe bool 191 initialSubErr bool 192 }{ 193 { 194 desc: "subscribed to deployments and removed access", 195 policyBeforeRules: mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob}), 196 policyAfterRules: mock.NamespacePolicy(structs.DefaultNamespace, "", []string{}), 197 shouldUnsubscribe: true, 198 event: structs.Event{ 199 Topic: structs.TopicDeployment, 200 Type: structs.TypeDeploymentUpdate, 201 Payload: structs.DeploymentEvent{ 202 Deployment: &structs.Deployment{ 203 ID: "some-id", 204 }, 205 }, 206 }, 207 policyEvent: structs.Event{ 208 Topic: structs.TopicACLToken, 209 Type: structs.TypeACLTokenUpserted, 210 Payload: structs.NewACLTokenEvent(&structs.ACLToken{SecretID: secretID}), 211 }, 212 }, 213 { 214 desc: "subscribed to evals and removed access", 215 policyBeforeRules: mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob}), 216 policyAfterRules: mock.NamespacePolicy(structs.DefaultNamespace, "", []string{}), 217 shouldUnsubscribe: true, 218 event: structs.Event{ 219 Topic: structs.TopicEvaluation, 220 Type: structs.TypeEvalUpdated, 221 Payload: structs.EvaluationEvent{ 222 Evaluation: &structs.Evaluation{ 223 ID: "some-id", 224 }, 225 }, 226 }, 227 policyEvent: structs.Event{ 228 Topic: structs.TopicACLToken, 229 Type: structs.TypeACLTokenUpserted, 230 Payload: structs.NewACLTokenEvent(&structs.ACLToken{SecretID: secretID}), 231 }, 232 }, 233 { 234 desc: "subscribed to allocs and removed access", 235 policyBeforeRules: mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob}), 236 policyAfterRules: mock.NamespacePolicy(structs.DefaultNamespace, "", []string{}), 237 shouldUnsubscribe: true, 238 event: structs.Event{ 239 Topic: structs.TopicAllocation, 240 Type: structs.TypeAllocationUpdated, 241 Payload: structs.AllocationEvent{ 242 Allocation: &structs.Allocation{ 243 ID: "some-id", 244 }, 245 }, 246 }, 247 policyEvent: structs.Event{ 248 Topic: structs.TopicACLToken, 249 Type: structs.TypeACLTokenUpserted, 250 Payload: structs.NewACLTokenEvent(&structs.ACLToken{SecretID: secretID}), 251 }, 252 }, 253 { 254 desc: "subscribed to nodes and removed access", 255 policyBeforeRules: mock.NodePolicy(acl.PolicyRead), 256 policyAfterRules: mock.NodePolicy(acl.PolicyDeny), 257 shouldUnsubscribe: true, 258 event: structs.Event{ 259 Topic: structs.TopicNode, 260 Type: structs.TypeNodeRegistration, 261 Payload: structs.NodeStreamEvent{ 262 Node: &structs.Node{ 263 ID: "some-id", 264 }, 265 }, 266 }, 267 policyEvent: structs.Event{ 268 Topic: structs.TopicACLToken, 269 Type: structs.TypeACLTokenUpserted, 270 Payload: structs.NewACLTokenEvent(&structs.ACLToken{SecretID: secretID}), 271 }, 272 }, 273 { 274 desc: "subscribed to deployments and no access change", 275 policyBeforeRules: mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob}), 276 policyAfterRules: mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob}), 277 shouldUnsubscribe: false, 278 event: structs.Event{ 279 Topic: structs.TopicDeployment, 280 Type: structs.TypeDeploymentUpdate, 281 Payload: structs.DeploymentEvent{ 282 Deployment: &structs.Deployment{ 283 ID: "some-id", 284 }, 285 }, 286 }, 287 policyEvent: structs.Event{ 288 Topic: structs.TopicACLToken, 289 Type: structs.TypeACLTokenUpserted, 290 Payload: structs.NewACLTokenEvent(&structs.ACLToken{SecretID: secretID}), 291 }, 292 }, 293 { 294 desc: "subscribed to evals and no access change", 295 policyBeforeRules: mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob}), 296 policyAfterRules: mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob}), 297 shouldUnsubscribe: false, 298 event: structs.Event{ 299 Topic: structs.TopicEvaluation, 300 Type: structs.TypeEvalUpdated, 301 Payload: structs.EvaluationEvent{ 302 Evaluation: &structs.Evaluation{ 303 ID: "some-id", 304 }, 305 }, 306 }, 307 policyEvent: structs.Event{ 308 Topic: structs.TopicACLToken, 309 Type: structs.TypeACLTokenUpserted, 310 Payload: structs.NewACLTokenEvent(&structs.ACLToken{SecretID: secretID}), 311 }, 312 }, 313 { 314 desc: "subscribed to allocs and no access change", 315 policyBeforeRules: mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob}), 316 policyAfterRules: mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob}), 317 shouldUnsubscribe: false, 318 event: structs.Event{ 319 Topic: structs.TopicAllocation, 320 Type: structs.TypeAllocationUpdated, 321 Payload: structs.AllocationEvent{ 322 Allocation: &structs.Allocation{ 323 ID: "some-id", 324 }, 325 }, 326 }, 327 policyEvent: structs.Event{ 328 Topic: structs.TopicACLToken, 329 Type: structs.TypeACLTokenUpserted, 330 Payload: structs.NewACLTokenEvent(&structs.ACLToken{SecretID: secretID}), 331 }, 332 }, 333 { 334 desc: "subscribed to nodes and no access change", 335 policyBeforeRules: mock.NodePolicy(acl.PolicyRead), 336 policyAfterRules: mock.NodePolicy(acl.PolicyRead), 337 shouldUnsubscribe: false, 338 event: structs.Event{ 339 Topic: structs.TopicNode, 340 Type: structs.TypeNodeRegistration, 341 Payload: structs.NodeStreamEvent{ 342 Node: &structs.Node{ 343 ID: "some-id", 344 }, 345 }, 346 }, 347 policyEvent: structs.Event{ 348 Topic: structs.TopicACLToken, 349 Type: structs.TypeACLTokenUpserted, 350 Payload: structs.NewACLTokenEvent(&structs.ACLToken{SecretID: secretID}), 351 }, 352 }, 353 { 354 desc: "initial token insufficient privileges", 355 initialSubErr: true, 356 policyBeforeRules: mock.NodePolicy(acl.PolicyDeny), 357 event: structs.Event{ 358 Topic: structs.TopicNode, 359 Type: structs.TypeNodeRegistration, 360 Payload: structs.NodeStreamEvent{ 361 Node: &structs.Node{ 362 ID: "some-id", 363 }, 364 }, 365 }, 366 policyEvent: structs.Event{ 367 Topic: structs.TopicACLToken, 368 Type: structs.TypeACLTokenUpserted, 369 Payload: structs.NewACLTokenEvent(&structs.ACLToken{SecretID: secretID}), 370 }, 371 }, 372 { 373 desc: "subscribed to nodes and policy change no change", 374 policyBeforeRules: mock.NodePolicy(acl.PolicyRead), 375 policyAfterRules: mock.NodePolicy(acl.PolicyWrite), 376 shouldUnsubscribe: false, 377 event: structs.Event{ 378 Topic: structs.TopicNode, 379 Type: structs.TypeNodeRegistration, 380 Payload: structs.NodeStreamEvent{ 381 Node: &structs.Node{ 382 ID: "some-id", 383 }, 384 }, 385 }, 386 policyEvent: structs.Event{ 387 Topic: structs.TopicACLPolicy, 388 Type: structs.TypeACLPolicyUpserted, 389 Payload: &structs.ACLPolicyEvent{ 390 ACLPolicy: &structs.ACLPolicy{ 391 Name: "some-policy", 392 }, 393 }, 394 }, 395 }, 396 { 397 desc: "subscribed to nodes and policy change no access", 398 policyBeforeRules: mock.NodePolicy(acl.PolicyRead), 399 policyAfterRules: mock.NodePolicy(acl.PolicyDeny), 400 shouldUnsubscribe: true, 401 event: structs.Event{ 402 Topic: structs.TopicNode, 403 Type: structs.TypeNodeRegistration, 404 Payload: structs.NodeStreamEvent{ 405 Node: &structs.Node{ 406 ID: "some-id", 407 }, 408 }, 409 }, 410 policyEvent: structs.Event{ 411 Topic: structs.TopicACLPolicy, 412 Type: structs.TypeACLPolicyUpserted, 413 Payload: &structs.ACLPolicyEvent{ 414 ACLPolicy: &structs.ACLPolicy{ 415 Name: "some-policy", 416 }, 417 }, 418 }, 419 }, 420 { 421 desc: "subscribed to nodes policy deleted", 422 policyBeforeRules: mock.NodePolicy(acl.PolicyRead), 423 policyAfterRules: "", 424 shouldUnsubscribe: true, 425 event: structs.Event{ 426 Topic: structs.TopicNode, 427 Type: structs.TypeNodeRegistration, 428 Payload: structs.NodeStreamEvent{ 429 Node: &structs.Node{ 430 ID: "some-id", 431 }, 432 }, 433 }, 434 policyEvent: structs.Event{ 435 Topic: structs.TopicACLPolicy, 436 Type: structs.TypeACLPolicyDeleted, 437 Payload: &structs.ACLPolicyEvent{ 438 ACLPolicy: &structs.ACLPolicy{ 439 Name: "some-policy", 440 }, 441 }, 442 }, 443 }, 444 } 445 446 for _, tc := range cases { 447 t.Run(tc.desc, func(t *testing.T) { 448 449 policy := &structs.ACLPolicy{ 450 Name: "some-policy", 451 Rules: tc.policyBeforeRules, 452 } 453 policy.SetHash() 454 455 tokenProvider := &fakeACLTokenProvider{ 456 policy: policy, 457 token: &structs.ACLToken{ 458 SecretID: secretID, 459 Policies: []string{policy.Name}, 460 }, 461 } 462 463 aclDelegate := &fakeACLDelegate{ 464 tokenProvider: tokenProvider, 465 } 466 467 publisher, err := NewEventBroker(ctx, aclDelegate, EventBrokerCfg{}) 468 require.NoError(t, err) 469 470 sub, err := publisher.SubscribeWithACLCheck(&SubscribeRequest{ 471 Topics: map[structs.Topic][]string{ 472 tc.event.Topic: {"*"}, 473 }, 474 Namespace: structs.DefaultNamespace, 475 Token: secretID, 476 }) 477 478 if tc.initialSubErr { 479 require.Error(t, err) 480 require.Nil(t, sub) 481 return 482 } else { 483 require.NoError(t, err) 484 } 485 publisher.Publish(&structs.Events{Index: 100, Events: []structs.Event{tc.event}}) 486 487 ctx, cancel := context.WithDeadline(ctx, time.Now().Add(100*time.Millisecond)) 488 defer cancel() 489 _, err = sub.Next(ctx) 490 require.NoError(t, err) 491 492 // Update the mock provider to use the after rules 493 policyAfter := &structs.ACLPolicy{ 494 Name: "some-new-policy", 495 Rules: tc.policyAfterRules, 496 ModifyIndex: 101, // The ModifyIndex is used to caclulate the acl cache key 497 } 498 policyAfter.SetHash() 499 500 tokenProvider.policy = policyAfter 501 502 // Publish ACL event triggering subscription re-evaluation 503 publisher.Publish(&structs.Events{Index: 101, Events: []structs.Event{tc.policyEvent}}) 504 // Publish another event 505 publisher.Publish(&structs.Events{Index: 102, Events: []structs.Event{tc.event}}) 506 507 // If we are expecting to unsubscribe consume the subscription 508 // until the expected error occurs. 509 ctx, cancel = context.WithDeadline(ctx, time.Now().Add(100*time.Millisecond)) 510 defer cancel() 511 if tc.shouldUnsubscribe { 512 for { 513 _, err = sub.Next(ctx) 514 if err != nil { 515 if err == context.DeadlineExceeded { 516 require.Fail(t, err.Error()) 517 } 518 if err == ErrSubscriptionClosed { 519 break 520 } 521 } 522 } 523 } else { 524 _, err = sub.Next(ctx) 525 require.NoError(t, err) 526 } 527 528 publisher.Publish(&structs.Events{Index: 103, Events: []structs.Event{tc.event}}) 529 530 ctx, cancel = context.WithDeadline(ctx, time.Now().Add(100*time.Millisecond)) 531 defer cancel() 532 _, err = sub.Next(ctx) 533 if tc.shouldUnsubscribe { 534 require.Equal(t, ErrSubscriptionClosed, err) 535 } else { 536 require.NoError(t, err) 537 } 538 }) 539 } 540 } 541 542 func consumeSubscription(ctx context.Context, sub *Subscription) <-chan subNextResult { 543 eventCh := make(chan subNextResult, 1) 544 go func() { 545 for { 546 es, err := sub.Next(ctx) 547 eventCh <- subNextResult{ 548 Events: es.Events, 549 Err: err, 550 } 551 if err != nil { 552 return 553 } 554 } 555 }() 556 return eventCh 557 } 558 559 type subNextResult struct { 560 Events []structs.Event 561 Err error 562 } 563 564 func nextResult(t *testing.T, eventCh <-chan subNextResult) subNextResult { 565 t.Helper() 566 select { 567 case next := <-eventCh: 568 return next 569 case <-time.After(100 * time.Millisecond): 570 t.Fatalf("no event after 100ms") 571 } 572 return subNextResult{} 573 } 574 575 func assertNoResult(t *testing.T, eventCh <-chan subNextResult) { 576 t.Helper() 577 select { 578 case next := <-eventCh: 579 require.NoError(t, next.Err) 580 require.Len(t, next.Events, 1) 581 t.Fatalf("received unexpected event: %#v", next.Events[0].Payload) 582 case <-time.After(100 * time.Millisecond): 583 } 584 } 585 586 func consumeSub(ctx context.Context, sub *Subscription) error { 587 for { 588 _, err := sub.Next(ctx) 589 if err != nil { 590 return err 591 } 592 } 593 }