github.com/Ilhicas/nomad@v1.0.4-0.20210304152020-e86851182bc3/nomad/stream/event_broker.go (about)

     1  package stream
     2  
     3  import (
     4  	"context"
     5  	"errors"
     6  	"fmt"
     7  	"sync"
     8  	"sync/atomic"
     9  
    10  	"github.com/armon/go-metrics"
    11  	"github.com/hashicorp/go-memdb"
    12  	lru "github.com/hashicorp/golang-lru"
    13  	"github.com/hashicorp/nomad/acl"
    14  	"github.com/hashicorp/nomad/nomad/structs"
    15  
    16  	"github.com/hashicorp/go-hclog"
    17  )
    18  
    19  const (
    20  	ACLCheckNodeRead   = "node-read"
    21  	ACLCheckManagement = "management"
    22  	aclCacheSize       = 32
    23  )
    24  
    25  type EventBrokerCfg struct {
    26  	EventBufferSize int64
    27  	Logger          hclog.Logger
    28  }
    29  
    30  type EventBroker struct {
    31  	// mu protects subscriptions
    32  	mu            sync.Mutex
    33  	subscriptions *subscriptions
    34  
    35  	// eventBuf stores a configurable amount of events in memory
    36  	eventBuf *eventBuffer
    37  
    38  	// publishCh is used to send messages from an active txn to a goroutine which
    39  	// publishes events, so that publishing can happen asynchronously from
    40  	// the Commit call in the FSM hot path.
    41  	publishCh chan *structs.Events
    42  
    43  	aclDelegate ACLDelegate
    44  	aclCache    *lru.TwoQueueCache
    45  
    46  	aclCh chan *structs.Event
    47  
    48  	logger hclog.Logger
    49  }
    50  
    51  // NewEventBroker returns an EventBroker for publishing change events.
    52  // A goroutine is run in the background to publish events to an event buffer.
    53  // Cancelling the context will shutdown the goroutine to free resources, and stop
    54  // all publishing.
    55  func NewEventBroker(ctx context.Context, aclDelegate ACLDelegate, cfg EventBrokerCfg) (*EventBroker, error) {
    56  	if cfg.Logger == nil {
    57  		cfg.Logger = hclog.NewNullLogger()
    58  	}
    59  
    60  	// Set the event buffer size to a minimum
    61  	if cfg.EventBufferSize == 0 {
    62  		cfg.EventBufferSize = 100
    63  	}
    64  
    65  	aclCache, err := lru.New2Q(aclCacheSize)
    66  	if err != nil {
    67  		return nil, err
    68  	}
    69  
    70  	buffer := newEventBuffer(cfg.EventBufferSize)
    71  	e := &EventBroker{
    72  		logger:      cfg.Logger.Named("event_broker"),
    73  		eventBuf:    buffer,
    74  		publishCh:   make(chan *structs.Events, 64),
    75  		aclCh:       make(chan *structs.Event, 10),
    76  		aclDelegate: aclDelegate,
    77  		aclCache:    aclCache,
    78  		subscriptions: &subscriptions{
    79  			byToken: make(map[string]map[*SubscribeRequest]*Subscription),
    80  		},
    81  	}
    82  
    83  	go e.handleUpdates(ctx)
    84  	go e.handleACLUpdates(ctx)
    85  
    86  	return e, nil
    87  }
    88  
    89  // Returns the current length of the event buffer
    90  func (e *EventBroker) Len() int {
    91  	return e.eventBuf.Len()
    92  }
    93  
    94  // Publish events to all subscribers of the event Topic.
    95  func (e *EventBroker) Publish(events *structs.Events) {
    96  	if len(events.Events) == 0 {
    97  		return
    98  	}
    99  
   100  	// Notify the broker to check running subscriptions against potentially
   101  	// updated ACL Token or Policy
   102  	for _, event := range events.Events {
   103  		if event.Topic == structs.TopicACLToken || event.Topic == structs.TopicACLPolicy {
   104  			e.aclCh <- &event
   105  		}
   106  	}
   107  
   108  	e.publishCh <- events
   109  }
   110  
   111  // SubscribeWithACLCheck validates the SubscribeRequest's token and requested Topics
   112  // to ensure that the tokens privileges are sufficient enough.
   113  func (e *EventBroker) SubscribeWithACLCheck(req *SubscribeRequest) (*Subscription, error) {
   114  	aclObj, err := aclObjFromSnapshotForTokenSecretID(e.aclDelegate.TokenProvider(), e.aclCache, req.Token)
   115  	if err != nil {
   116  		return nil, structs.ErrPermissionDenied
   117  	}
   118  
   119  	if allowed := aclAllowsSubscription(aclObj, req); !allowed {
   120  		return nil, structs.ErrPermissionDenied
   121  	}
   122  
   123  	return e.Subscribe(req)
   124  }
   125  
   126  // Subscribe returns a new Subscription for a given request. A Subscription
   127  // will receive an initial empty currentItem value which points to the first item
   128  // in the buffer. This allows the new subscription to call Next() without first checking
   129  // for the current Item.
   130  //
   131  // A Subscription will start at the requested index, or as close as possible to
   132  // the requested index if it is no longer in the buffer. If StartExactlyAtIndex is
   133  // set and the index is no longer in the buffer or not yet in the buffer an error
   134  // will be returned.
   135  //
   136  // When a caller is finished with the subscription it must call Subscription.Unsubscribe
   137  // to free ACL tracking resources.
   138  func (e *EventBroker) Subscribe(req *SubscribeRequest) (*Subscription, error) {
   139  	e.mu.Lock()
   140  	defer e.mu.Unlock()
   141  
   142  	var head *bufferItem
   143  	var offset int
   144  	if req.Index != 0 {
   145  		head, offset = e.eventBuf.StartAtClosest(req.Index)
   146  	} else {
   147  		head = e.eventBuf.Head()
   148  	}
   149  	if offset > 0 && req.StartExactlyAtIndex {
   150  		return nil, fmt.Errorf("requested index not in buffer")
   151  	} else if offset > 0 {
   152  		metrics.SetGauge([]string{"nomad", "event_broker", "subscription", "request_offset"}, float32(offset))
   153  		e.logger.Debug("requested index no longer in buffer", "requsted", int(req.Index), "closest", int(head.Events.Index))
   154  	}
   155  
   156  	// Empty head so that calling Next on sub
   157  	start := newBufferItem(&structs.Events{Index: req.Index})
   158  	start.link.next.Store(head)
   159  	close(start.link.nextCh)
   160  
   161  	sub := newSubscription(req, start, e.subscriptions.unsubscribeFn(req))
   162  
   163  	e.subscriptions.add(req, sub)
   164  	return sub, nil
   165  }
   166  
   167  // CloseAll closes all subscriptions
   168  func (e *EventBroker) CloseAll() {
   169  	e.subscriptions.closeAll()
   170  }
   171  
   172  func (e *EventBroker) handleUpdates(ctx context.Context) {
   173  	for {
   174  		select {
   175  		case <-ctx.Done():
   176  			e.subscriptions.closeAll()
   177  			return
   178  		case update := <-e.publishCh:
   179  			e.eventBuf.Append(update)
   180  		}
   181  	}
   182  }
   183  
   184  func (e *EventBroker) handleACLUpdates(ctx context.Context) {
   185  	for {
   186  		select {
   187  		case <-ctx.Done():
   188  			return
   189  		case update := <-e.aclCh:
   190  			switch payload := update.Payload.(type) {
   191  			case *structs.ACLTokenEvent:
   192  				tokenSecretID := payload.SecretID()
   193  
   194  				// Token was deleted
   195  				if update.Type == structs.TypeACLTokenDeleted {
   196  					e.subscriptions.closeSubscriptionsForTokens([]string{tokenSecretID})
   197  					continue
   198  				}
   199  
   200  				// If broker cannot fetch state there is nothing more to do
   201  				if e.aclDelegate == nil {
   202  					continue
   203  				}
   204  
   205  				aclObj, err := aclObjFromSnapshotForTokenSecretID(e.aclDelegate.TokenProvider(), e.aclCache, tokenSecretID)
   206  				if err != nil || aclObj == nil {
   207  					e.logger.Error("failed resolving ACL for secretID, closing subscriptions", "error", err)
   208  					e.subscriptions.closeSubscriptionsForTokens([]string{tokenSecretID})
   209  					continue
   210  				}
   211  
   212  				e.subscriptions.closeSubscriptionFunc(tokenSecretID, func(sub *Subscription) bool {
   213  					return !aclAllowsSubscription(aclObj, sub.req)
   214  				})
   215  
   216  			case *structs.ACLPolicyEvent:
   217  				// Re-evaluate each subscriptions permissions since a policy
   218  				// change may or may not affect the subscription
   219  				e.checkSubscriptionsAgainstPolicyChange()
   220  			}
   221  		}
   222  	}
   223  }
   224  
   225  // checkSubscriptionsAgainstPolicyChange iterates over the brokers
   226  // subscriptions and evaluates whether the token used for the subscription is
   227  // still valid. If it is not valid it closes the subscriptions belonging to the
   228  // token.
   229  //
   230  // A lock must be held to iterate over the map of subscriptions.
   231  func (e *EventBroker) checkSubscriptionsAgainstPolicyChange() {
   232  	e.mu.Lock()
   233  	defer e.mu.Unlock()
   234  
   235  	// If broker cannot fetch state there is nothing more to do
   236  	if e.aclDelegate == nil {
   237  		return
   238  	}
   239  
   240  	aclSnapshot := e.aclDelegate.TokenProvider()
   241  	for tokenSecretID := range e.subscriptions.byToken {
   242  		// if tokenSecretID is empty ACLs were disabled at time of subscribing
   243  		if tokenSecretID == "" {
   244  			continue
   245  		}
   246  
   247  		aclObj, err := aclObjFromSnapshotForTokenSecretID(aclSnapshot, e.aclCache, tokenSecretID)
   248  		if err != nil || aclObj == nil {
   249  			e.logger.Debug("failed resolving ACL for secretID, closing subscriptions", "error", err)
   250  			e.subscriptions.closeSubscriptionsForTokens([]string{tokenSecretID})
   251  			continue
   252  		}
   253  
   254  		e.subscriptions.closeSubscriptionFunc(tokenSecretID, func(sub *Subscription) bool {
   255  			return !aclAllowsSubscription(aclObj, sub.req)
   256  		})
   257  	}
   258  }
   259  
   260  func aclObjFromSnapshotForTokenSecretID(aclSnapshot ACLTokenProvider, aclCache *lru.TwoQueueCache, tokenSecretID string) (*acl.ACL, error) {
   261  	aclToken, err := aclSnapshot.ACLTokenBySecretID(nil, tokenSecretID)
   262  	if err != nil {
   263  		return nil, err
   264  	}
   265  
   266  	if aclToken == nil {
   267  		return nil, errors.New("no token for secret ID")
   268  	}
   269  
   270  	// Check if this is a management token
   271  	if aclToken.Type == structs.ACLManagementToken {
   272  		return acl.ManagementACL, nil
   273  	}
   274  
   275  	aclPolicies := make([]*structs.ACLPolicy, 0, len(aclToken.Policies))
   276  	for _, policyName := range aclToken.Policies {
   277  		policy, err := aclSnapshot.ACLPolicyByName(nil, policyName)
   278  		if err != nil || policy == nil {
   279  			return nil, errors.New("error finding acl policy")
   280  		}
   281  		aclPolicies = append(aclPolicies, policy)
   282  	}
   283  
   284  	return structs.CompileACLObject(aclCache, aclPolicies)
   285  }
   286  
   287  type ACLTokenProvider interface {
   288  	ACLTokenBySecretID(ws memdb.WatchSet, secretID string) (*structs.ACLToken, error)
   289  	ACLPolicyByName(ws memdb.WatchSet, policyName string) (*structs.ACLPolicy, error)
   290  }
   291  
   292  type ACLDelegate interface {
   293  	TokenProvider() ACLTokenProvider
   294  }
   295  
   296  func aclAllowsSubscription(aclObj *acl.ACL, subReq *SubscribeRequest) bool {
   297  	for topic := range subReq.Topics {
   298  		switch topic {
   299  		case structs.TopicDeployment,
   300  			structs.TopicEvaluation,
   301  			structs.TopicAllocation,
   302  			structs.TopicJob:
   303  			if ok := aclObj.AllowNsOp(subReq.Namespace, acl.NamespaceCapabilityReadJob); !ok {
   304  				return false
   305  			}
   306  		case structs.TopicNode:
   307  			if ok := aclObj.AllowNodeRead(); !ok {
   308  				return false
   309  			}
   310  		default:
   311  			if ok := aclObj.IsManagement(); !ok {
   312  				return false
   313  			}
   314  		}
   315  	}
   316  
   317  	return true
   318  }
   319  
   320  func (s *Subscription) forceClose() {
   321  	if atomic.CompareAndSwapUint32(&s.state, subscriptionStateOpen, subscriptionStateClosed) {
   322  		close(s.forceClosed)
   323  	}
   324  }
   325  
   326  type subscriptions struct {
   327  	// mu for byToken. If both subscription.mu and EventBroker.mu need
   328  	// to be held, EventBroker mutex MUST always be acquired first.
   329  	mu sync.RWMutex
   330  
   331  	// byToken is an mapping of active Subscriptions indexed by a token and
   332  	// a pointer to the request.
   333  	// When the token is modified all subscriptions under that token will be
   334  	// reloaded.
   335  	// A subscription may be unsubscribed by using the pointer to the request.
   336  	byToken map[string]map[*SubscribeRequest]*Subscription
   337  }
   338  
   339  func (s *subscriptions) add(req *SubscribeRequest, sub *Subscription) {
   340  	s.mu.Lock()
   341  	defer s.mu.Unlock()
   342  
   343  	subsByToken, ok := s.byToken[req.Token]
   344  	if !ok {
   345  		subsByToken = make(map[*SubscribeRequest]*Subscription)
   346  		s.byToken[req.Token] = subsByToken
   347  	}
   348  	subsByToken[req] = sub
   349  }
   350  
   351  func (s *subscriptions) closeSubscriptionsForTokens(tokenSecretIDs []string) {
   352  	s.mu.RLock()
   353  	defer s.mu.RUnlock()
   354  
   355  	for _, secretID := range tokenSecretIDs {
   356  		if subs, ok := s.byToken[secretID]; ok {
   357  			for _, sub := range subs {
   358  				sub.forceClose()
   359  			}
   360  		}
   361  	}
   362  }
   363  
   364  func (s *subscriptions) closeSubscriptionFunc(tokenSecretID string, fn func(*Subscription) bool) {
   365  	s.mu.RLock()
   366  	defer s.mu.RUnlock()
   367  
   368  	for _, sub := range s.byToken[tokenSecretID] {
   369  		if fn(sub) {
   370  			sub.forceClose()
   371  		}
   372  	}
   373  }
   374  
   375  // unsubscribeFn returns a function that the subscription will call to remove
   376  // itself from the subsByToken.
   377  // This function is returned as a closure so that the caller doesn't need to keep
   378  // track of the SubscriptionRequest, and can not accidentally call unsubscribeFn with the
   379  // wrong pointer.
   380  func (s *subscriptions) unsubscribeFn(req *SubscribeRequest) func() {
   381  	return func() {
   382  		s.mu.Lock()
   383  		defer s.mu.Unlock()
   384  
   385  		subsByToken, ok := s.byToken[req.Token]
   386  		if !ok {
   387  			return
   388  		}
   389  
   390  		sub := subsByToken[req]
   391  		if sub == nil {
   392  			return
   393  		}
   394  
   395  		// close the subscription
   396  		sub.forceClose()
   397  
   398  		delete(subsByToken, req)
   399  		if len(subsByToken) == 0 {
   400  			delete(s.byToken, req.Token)
   401  		}
   402  	}
   403  }
   404  
   405  func (s *subscriptions) closeAll() {
   406  	s.mu.Lock()
   407  	defer s.mu.Unlock()
   408  
   409  	for _, byRequest := range s.byToken {
   410  		for _, sub := range byRequest {
   411  			sub.forceClose()
   412  		}
   413  	}
   414  }