github.com/Ilhicas/nomad@v1.0.4-0.20210304152020-e86851182bc3/nomad/stream/event_broker_test.go (about)

     1  package stream
     2  
     3  import (
     4  	"context"
     5  	"sync/atomic"
     6  	"testing"
     7  	"time"
     8  
     9  	"github.com/hashicorp/go-memdb"
    10  	"github.com/hashicorp/nomad/acl"
    11  	"github.com/hashicorp/nomad/nomad/mock"
    12  	"github.com/hashicorp/nomad/nomad/structs"
    13  
    14  	"github.com/stretchr/testify/require"
    15  )
    16  
    17  func TestEventBroker_PublishChangesAndSubscribe(t *testing.T) {
    18  	subscription := &SubscribeRequest{
    19  		Topics: map[structs.Topic][]string{
    20  			"Test": {"sub-key"},
    21  		},
    22  	}
    23  	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
    24  	defer cancel()
    25  
    26  	publisher, err := NewEventBroker(ctx, nil, EventBrokerCfg{EventBufferSize: 100})
    27  	require.NoError(t, err)
    28  
    29  	sub, err := publisher.Subscribe(subscription)
    30  	require.NoError(t, err)
    31  	eventCh := consumeSubscription(ctx, sub)
    32  
    33  	// Now subscriber should block waiting for updates
    34  	assertNoResult(t, eventCh)
    35  
    36  	events := []structs.Event{{
    37  		Index:   1,
    38  		Topic:   "Test",
    39  		Key:     "sub-key",
    40  		Payload: "sample payload",
    41  	}}
    42  	publisher.Publish(&structs.Events{Index: 1, Events: events})
    43  
    44  	// Subscriber should see the published event
    45  	result := nextResult(t, eventCh)
    46  	require.NoError(t, result.Err)
    47  	expected := []structs.Event{{Payload: "sample payload", Key: "sub-key", Topic: "Test", Index: 1}}
    48  	require.Equal(t, expected, result.Events)
    49  
    50  	// Now subscriber should block waiting for updates
    51  	assertNoResult(t, eventCh)
    52  
    53  	// Publish a second event
    54  	events = []structs.Event{{
    55  		Index:   2,
    56  		Topic:   "Test",
    57  		Key:     "sub-key",
    58  		Payload: "sample payload 2",
    59  	}}
    60  	publisher.Publish(&structs.Events{Index: 2, Events: events})
    61  
    62  	result = nextResult(t, eventCh)
    63  	require.NoError(t, result.Err)
    64  	expected = []structs.Event{{Payload: "sample payload 2", Key: "sub-key", Topic: "Test", Index: 2}}
    65  	require.Equal(t, expected, result.Events)
    66  }
    67  
    68  func TestEventBroker_ShutdownClosesSubscriptions(t *testing.T) {
    69  	ctx, cancel := context.WithCancel(context.Background())
    70  	t.Cleanup(cancel)
    71  
    72  	publisher, err := NewEventBroker(ctx, nil, EventBrokerCfg{})
    73  	require.NoError(t, err)
    74  
    75  	sub1, err := publisher.Subscribe(&SubscribeRequest{})
    76  	require.NoError(t, err)
    77  	defer sub1.Unsubscribe()
    78  
    79  	sub2, err := publisher.Subscribe(&SubscribeRequest{})
    80  	require.NoError(t, err)
    81  	defer sub2.Unsubscribe()
    82  
    83  	cancel() // Shutdown
    84  
    85  	err = consumeSub(context.Background(), sub1)
    86  	require.Equal(t, err, ErrSubscriptionClosed)
    87  
    88  	_, err = sub2.Next(context.Background())
    89  	require.Equal(t, err, ErrSubscriptionClosed)
    90  }
    91  
    92  // TestEventBroker_EmptyReqToken_DistinctSubscriptions tests subscription
    93  // hanlding behavior when ACLs are disabled (request Token is empty).
    94  // Subscriptions are mapped by their request token.  when that token is empty,
    95  // the subscriptions should still be handled indeppendtly of each other when
    96  // unssubscribing.
    97  func TestEventBroker_EmptyReqToken_DistinctSubscriptions(t *testing.T) {
    98  	ctx, cancel := context.WithCancel(context.Background())
    99  	t.Cleanup(cancel)
   100  
   101  	publisher, err := NewEventBroker(ctx, nil, EventBrokerCfg{})
   102  	require.NoError(t, err)
   103  
   104  	// first subscription, empty token
   105  	sub1, err := publisher.Subscribe(&SubscribeRequest{})
   106  	require.NoError(t, err)
   107  	defer sub1.Unsubscribe()
   108  
   109  	// second subscription, empty token
   110  	sub2, err := publisher.Subscribe(&SubscribeRequest{})
   111  	require.NoError(t, err)
   112  	require.NotNil(t, sub2)
   113  
   114  	sub1.Unsubscribe()
   115  
   116  	require.Equal(t, subscriptionStateOpen, atomic.LoadUint32(&sub2.state))
   117  }
   118  
   119  func TestEventBroker_handleACLUpdates_TokenDeleted(t *testing.T) {
   120  	ctx, cancel := context.WithCancel(context.Background())
   121  	t.Cleanup(cancel)
   122  
   123  	publisher, err := NewEventBroker(ctx, nil, EventBrokerCfg{})
   124  	require.NoError(t, err)
   125  
   126  	sub1, err := publisher.Subscribe(&SubscribeRequest{
   127  		Topics: map[structs.Topic][]string{
   128  			"*": {"*"},
   129  		},
   130  		Token: "foo",
   131  	})
   132  	require.NoError(t, err)
   133  	defer sub1.Unsubscribe()
   134  
   135  	aclEvent := structs.Event{
   136  		Topic:   structs.TopicACLToken,
   137  		Type:    structs.TypeACLTokenDeleted,
   138  		Payload: structs.NewACLTokenEvent(&structs.ACLToken{SecretID: "foo"}),
   139  	}
   140  
   141  	publisher.Publish(&structs.Events{Index: 100, Events: []structs.Event{aclEvent}})
   142  	for {
   143  		_, err := sub1.Next(ctx)
   144  		if err == ErrSubscriptionClosed {
   145  			break
   146  		}
   147  	}
   148  
   149  	out, err := sub1.Next(ctx)
   150  	require.Error(t, err)
   151  	require.Equal(t, ErrSubscriptionClosed, err)
   152  	require.Equal(t, structs.Events{}, out)
   153  }
   154  
   155  type fakeACLDelegate struct {
   156  	tokenProvider ACLTokenProvider
   157  }
   158  
   159  func (d *fakeACLDelegate) TokenProvider() ACLTokenProvider {
   160  	return d.tokenProvider
   161  }
   162  
   163  type fakeACLTokenProvider struct {
   164  	policy    *structs.ACLPolicy
   165  	policyErr error
   166  	token     *structs.ACLToken
   167  	tokenErr  error
   168  }
   169  
   170  func (p *fakeACLTokenProvider) ACLTokenBySecretID(ws memdb.WatchSet, secretID string) (*structs.ACLToken, error) {
   171  	return p.token, p.tokenErr
   172  }
   173  
   174  func (p *fakeACLTokenProvider) ACLPolicyByName(ws memdb.WatchSet, policyName string) (*structs.ACLPolicy, error) {
   175  	return p.policy, p.policyErr
   176  }
   177  
   178  func TestEventBroker_handleACLUpdates_policyupdated(t *testing.T) {
   179  	ctx, cancel := context.WithCancel(context.Background())
   180  	t.Cleanup(cancel)
   181  
   182  	secretID := "some-secret-id"
   183  	cases := []struct {
   184  		policyBeforeRules string
   185  		policyAfterRules  string
   186  		topics            map[structs.Topic][]string
   187  		desc              string
   188  		event             structs.Event
   189  		policyEvent       structs.Event
   190  		shouldUnsubscribe bool
   191  		initialSubErr     bool
   192  	}{
   193  		{
   194  			desc:              "subscribed to deployments and removed access",
   195  			policyBeforeRules: mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob}),
   196  			policyAfterRules:  mock.NamespacePolicy(structs.DefaultNamespace, "", []string{}),
   197  			shouldUnsubscribe: true,
   198  			event: structs.Event{
   199  				Topic: structs.TopicDeployment,
   200  				Type:  structs.TypeDeploymentUpdate,
   201  				Payload: structs.DeploymentEvent{
   202  					Deployment: &structs.Deployment{
   203  						ID: "some-id",
   204  					},
   205  				},
   206  			},
   207  			policyEvent: structs.Event{
   208  				Topic:   structs.TopicACLToken,
   209  				Type:    structs.TypeACLTokenUpserted,
   210  				Payload: structs.NewACLTokenEvent(&structs.ACLToken{SecretID: secretID}),
   211  			},
   212  		},
   213  		{
   214  			desc:              "subscribed to evals and removed access",
   215  			policyBeforeRules: mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob}),
   216  			policyAfterRules:  mock.NamespacePolicy(structs.DefaultNamespace, "", []string{}),
   217  			shouldUnsubscribe: true,
   218  			event: structs.Event{
   219  				Topic: structs.TopicEvaluation,
   220  				Type:  structs.TypeEvalUpdated,
   221  				Payload: structs.EvaluationEvent{
   222  					Evaluation: &structs.Evaluation{
   223  						ID: "some-id",
   224  					},
   225  				},
   226  			},
   227  			policyEvent: structs.Event{
   228  				Topic:   structs.TopicACLToken,
   229  				Type:    structs.TypeACLTokenUpserted,
   230  				Payload: structs.NewACLTokenEvent(&structs.ACLToken{SecretID: secretID}),
   231  			},
   232  		},
   233  		{
   234  			desc:              "subscribed to allocs and removed access",
   235  			policyBeforeRules: mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob}),
   236  			policyAfterRules:  mock.NamespacePolicy(structs.DefaultNamespace, "", []string{}),
   237  			shouldUnsubscribe: true,
   238  			event: structs.Event{
   239  				Topic: structs.TopicAllocation,
   240  				Type:  structs.TypeAllocationUpdated,
   241  				Payload: structs.AllocationEvent{
   242  					Allocation: &structs.Allocation{
   243  						ID: "some-id",
   244  					},
   245  				},
   246  			},
   247  			policyEvent: structs.Event{
   248  				Topic:   structs.TopicACLToken,
   249  				Type:    structs.TypeACLTokenUpserted,
   250  				Payload: structs.NewACLTokenEvent(&structs.ACLToken{SecretID: secretID}),
   251  			},
   252  		},
   253  		{
   254  			desc:              "subscribed to nodes and removed access",
   255  			policyBeforeRules: mock.NodePolicy(acl.PolicyRead),
   256  			policyAfterRules:  mock.NodePolicy(acl.PolicyDeny),
   257  			shouldUnsubscribe: true,
   258  			event: structs.Event{
   259  				Topic: structs.TopicNode,
   260  				Type:  structs.TypeNodeRegistration,
   261  				Payload: structs.NodeStreamEvent{
   262  					Node: &structs.Node{
   263  						ID: "some-id",
   264  					},
   265  				},
   266  			},
   267  			policyEvent: structs.Event{
   268  				Topic:   structs.TopicACLToken,
   269  				Type:    structs.TypeACLTokenUpserted,
   270  				Payload: structs.NewACLTokenEvent(&structs.ACLToken{SecretID: secretID}),
   271  			},
   272  		},
   273  		{
   274  			desc:              "subscribed to deployments and no access change",
   275  			policyBeforeRules: mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob}),
   276  			policyAfterRules:  mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob}),
   277  			shouldUnsubscribe: false,
   278  			event: structs.Event{
   279  				Topic: structs.TopicDeployment,
   280  				Type:  structs.TypeDeploymentUpdate,
   281  				Payload: structs.DeploymentEvent{
   282  					Deployment: &structs.Deployment{
   283  						ID: "some-id",
   284  					},
   285  				},
   286  			},
   287  			policyEvent: structs.Event{
   288  				Topic:   structs.TopicACLToken,
   289  				Type:    structs.TypeACLTokenUpserted,
   290  				Payload: structs.NewACLTokenEvent(&structs.ACLToken{SecretID: secretID}),
   291  			},
   292  		},
   293  		{
   294  			desc:              "subscribed to evals and no access change",
   295  			policyBeforeRules: mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob}),
   296  			policyAfterRules:  mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob}),
   297  			shouldUnsubscribe: false,
   298  			event: structs.Event{
   299  				Topic: structs.TopicEvaluation,
   300  				Type:  structs.TypeEvalUpdated,
   301  				Payload: structs.EvaluationEvent{
   302  					Evaluation: &structs.Evaluation{
   303  						ID: "some-id",
   304  					},
   305  				},
   306  			},
   307  			policyEvent: structs.Event{
   308  				Topic:   structs.TopicACLToken,
   309  				Type:    structs.TypeACLTokenUpserted,
   310  				Payload: structs.NewACLTokenEvent(&structs.ACLToken{SecretID: secretID}),
   311  			},
   312  		},
   313  		{
   314  			desc:              "subscribed to allocs and no access change",
   315  			policyBeforeRules: mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob}),
   316  			policyAfterRules:  mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob}),
   317  			shouldUnsubscribe: false,
   318  			event: structs.Event{
   319  				Topic: structs.TopicAllocation,
   320  				Type:  structs.TypeAllocationUpdated,
   321  				Payload: structs.AllocationEvent{
   322  					Allocation: &structs.Allocation{
   323  						ID: "some-id",
   324  					},
   325  				},
   326  			},
   327  			policyEvent: structs.Event{
   328  				Topic:   structs.TopicACLToken,
   329  				Type:    structs.TypeACLTokenUpserted,
   330  				Payload: structs.NewACLTokenEvent(&structs.ACLToken{SecretID: secretID}),
   331  			},
   332  		},
   333  		{
   334  			desc:              "subscribed to nodes and no access change",
   335  			policyBeforeRules: mock.NodePolicy(acl.PolicyRead),
   336  			policyAfterRules:  mock.NodePolicy(acl.PolicyRead),
   337  			shouldUnsubscribe: false,
   338  			event: structs.Event{
   339  				Topic: structs.TopicNode,
   340  				Type:  structs.TypeNodeRegistration,
   341  				Payload: structs.NodeStreamEvent{
   342  					Node: &structs.Node{
   343  						ID: "some-id",
   344  					},
   345  				},
   346  			},
   347  			policyEvent: structs.Event{
   348  				Topic:   structs.TopicACLToken,
   349  				Type:    structs.TypeACLTokenUpserted,
   350  				Payload: structs.NewACLTokenEvent(&structs.ACLToken{SecretID: secretID}),
   351  			},
   352  		},
   353  		{
   354  			desc:              "initial token insufficient privileges",
   355  			initialSubErr:     true,
   356  			policyBeforeRules: mock.NodePolicy(acl.PolicyDeny),
   357  			event: structs.Event{
   358  				Topic: structs.TopicNode,
   359  				Type:  structs.TypeNodeRegistration,
   360  				Payload: structs.NodeStreamEvent{
   361  					Node: &structs.Node{
   362  						ID: "some-id",
   363  					},
   364  				},
   365  			},
   366  			policyEvent: structs.Event{
   367  				Topic:   structs.TopicACLToken,
   368  				Type:    structs.TypeACLTokenUpserted,
   369  				Payload: structs.NewACLTokenEvent(&structs.ACLToken{SecretID: secretID}),
   370  			},
   371  		},
   372  		{
   373  			desc:              "subscribed to nodes and policy change no change",
   374  			policyBeforeRules: mock.NodePolicy(acl.PolicyRead),
   375  			policyAfterRules:  mock.NodePolicy(acl.PolicyWrite),
   376  			shouldUnsubscribe: false,
   377  			event: structs.Event{
   378  				Topic: structs.TopicNode,
   379  				Type:  structs.TypeNodeRegistration,
   380  				Payload: structs.NodeStreamEvent{
   381  					Node: &structs.Node{
   382  						ID: "some-id",
   383  					},
   384  				},
   385  			},
   386  			policyEvent: structs.Event{
   387  				Topic: structs.TopicACLPolicy,
   388  				Type:  structs.TypeACLPolicyUpserted,
   389  				Payload: &structs.ACLPolicyEvent{
   390  					ACLPolicy: &structs.ACLPolicy{
   391  						Name: "some-policy",
   392  					},
   393  				},
   394  			},
   395  		},
   396  		{
   397  			desc:              "subscribed to nodes and policy change no access",
   398  			policyBeforeRules: mock.NodePolicy(acl.PolicyRead),
   399  			policyAfterRules:  mock.NodePolicy(acl.PolicyDeny),
   400  			shouldUnsubscribe: true,
   401  			event: structs.Event{
   402  				Topic: structs.TopicNode,
   403  				Type:  structs.TypeNodeRegistration,
   404  				Payload: structs.NodeStreamEvent{
   405  					Node: &structs.Node{
   406  						ID: "some-id",
   407  					},
   408  				},
   409  			},
   410  			policyEvent: structs.Event{
   411  				Topic: structs.TopicACLPolicy,
   412  				Type:  structs.TypeACLPolicyUpserted,
   413  				Payload: &structs.ACLPolicyEvent{
   414  					ACLPolicy: &structs.ACLPolicy{
   415  						Name: "some-policy",
   416  					},
   417  				},
   418  			},
   419  		},
   420  		{
   421  			desc:              "subscribed to nodes policy deleted",
   422  			policyBeforeRules: mock.NodePolicy(acl.PolicyRead),
   423  			policyAfterRules:  "",
   424  			shouldUnsubscribe: true,
   425  			event: structs.Event{
   426  				Topic: structs.TopicNode,
   427  				Type:  structs.TypeNodeRegistration,
   428  				Payload: structs.NodeStreamEvent{
   429  					Node: &structs.Node{
   430  						ID: "some-id",
   431  					},
   432  				},
   433  			},
   434  			policyEvent: structs.Event{
   435  				Topic: structs.TopicACLPolicy,
   436  				Type:  structs.TypeACLPolicyDeleted,
   437  				Payload: &structs.ACLPolicyEvent{
   438  					ACLPolicy: &structs.ACLPolicy{
   439  						Name: "some-policy",
   440  					},
   441  				},
   442  			},
   443  		},
   444  	}
   445  
   446  	for _, tc := range cases {
   447  		t.Run(tc.desc, func(t *testing.T) {
   448  
   449  			policy := &structs.ACLPolicy{
   450  				Name:  "some-policy",
   451  				Rules: tc.policyBeforeRules,
   452  			}
   453  			policy.SetHash()
   454  
   455  			tokenProvider := &fakeACLTokenProvider{
   456  				policy: policy,
   457  				token: &structs.ACLToken{
   458  					SecretID: secretID,
   459  					Policies: []string{policy.Name},
   460  				},
   461  			}
   462  
   463  			aclDelegate := &fakeACLDelegate{
   464  				tokenProvider: tokenProvider,
   465  			}
   466  
   467  			publisher, err := NewEventBroker(ctx, aclDelegate, EventBrokerCfg{})
   468  			require.NoError(t, err)
   469  
   470  			sub, err := publisher.SubscribeWithACLCheck(&SubscribeRequest{
   471  				Topics: map[structs.Topic][]string{
   472  					tc.event.Topic: {"*"},
   473  				},
   474  				Namespace: structs.DefaultNamespace,
   475  				Token:     secretID,
   476  			})
   477  
   478  			if tc.initialSubErr {
   479  				require.Error(t, err)
   480  				require.Nil(t, sub)
   481  				return
   482  			} else {
   483  				require.NoError(t, err)
   484  			}
   485  			publisher.Publish(&structs.Events{Index: 100, Events: []structs.Event{tc.event}})
   486  
   487  			ctx, cancel := context.WithDeadline(ctx, time.Now().Add(100*time.Millisecond))
   488  			defer cancel()
   489  			_, err = sub.Next(ctx)
   490  			require.NoError(t, err)
   491  
   492  			// Update the mock provider to use the after rules
   493  			policyAfter := &structs.ACLPolicy{
   494  				Name:        "some-new-policy",
   495  				Rules:       tc.policyAfterRules,
   496  				ModifyIndex: 101, // The ModifyIndex is used to caclulate the acl cache key
   497  			}
   498  			policyAfter.SetHash()
   499  
   500  			tokenProvider.policy = policyAfter
   501  
   502  			// Publish ACL event triggering subscription re-evaluation
   503  			publisher.Publish(&structs.Events{Index: 101, Events: []structs.Event{tc.policyEvent}})
   504  			// Publish another event
   505  			publisher.Publish(&structs.Events{Index: 102, Events: []structs.Event{tc.event}})
   506  
   507  			// If we are expecting to unsubscribe consume the subscription
   508  			// until the expected error occurs.
   509  			ctx, cancel = context.WithDeadline(ctx, time.Now().Add(100*time.Millisecond))
   510  			defer cancel()
   511  			if tc.shouldUnsubscribe {
   512  				for {
   513  					_, err = sub.Next(ctx)
   514  					if err != nil {
   515  						if err == context.DeadlineExceeded {
   516  							require.Fail(t, err.Error())
   517  						}
   518  						if err == ErrSubscriptionClosed {
   519  							break
   520  						}
   521  					}
   522  				}
   523  			} else {
   524  				_, err = sub.Next(ctx)
   525  				require.NoError(t, err)
   526  			}
   527  
   528  			publisher.Publish(&structs.Events{Index: 103, Events: []structs.Event{tc.event}})
   529  
   530  			ctx, cancel = context.WithDeadline(ctx, time.Now().Add(100*time.Millisecond))
   531  			defer cancel()
   532  			_, err = sub.Next(ctx)
   533  			if tc.shouldUnsubscribe {
   534  				require.Equal(t, ErrSubscriptionClosed, err)
   535  			} else {
   536  				require.NoError(t, err)
   537  			}
   538  		})
   539  	}
   540  }
   541  
   542  func consumeSubscription(ctx context.Context, sub *Subscription) <-chan subNextResult {
   543  	eventCh := make(chan subNextResult, 1)
   544  	go func() {
   545  		for {
   546  			es, err := sub.Next(ctx)
   547  			eventCh <- subNextResult{
   548  				Events: es.Events,
   549  				Err:    err,
   550  			}
   551  			if err != nil {
   552  				return
   553  			}
   554  		}
   555  	}()
   556  	return eventCh
   557  }
   558  
   559  type subNextResult struct {
   560  	Events []structs.Event
   561  	Err    error
   562  }
   563  
   564  func nextResult(t *testing.T, eventCh <-chan subNextResult) subNextResult {
   565  	t.Helper()
   566  	select {
   567  	case next := <-eventCh:
   568  		return next
   569  	case <-time.After(100 * time.Millisecond):
   570  		t.Fatalf("no event after 100ms")
   571  	}
   572  	return subNextResult{}
   573  }
   574  
   575  func assertNoResult(t *testing.T, eventCh <-chan subNextResult) {
   576  	t.Helper()
   577  	select {
   578  	case next := <-eventCh:
   579  		require.NoError(t, next.Err)
   580  		require.Len(t, next.Events, 1)
   581  		t.Fatalf("received unexpected event: %#v", next.Events[0].Payload)
   582  	case <-time.After(100 * time.Millisecond):
   583  	}
   584  }
   585  
   586  func consumeSub(ctx context.Context, sub *Subscription) error {
   587  	for {
   588  		_, err := sub.Next(ctx)
   589  		if err != nil {
   590  			return err
   591  		}
   592  	}
   593  }