github.com/Ilhicas/nomad@v1.0.4-0.20210304152020-e86851182bc3/nomad/stream/event_broker.go (about) 1 package stream 2 3 import ( 4 "context" 5 "errors" 6 "fmt" 7 "sync" 8 "sync/atomic" 9 10 "github.com/armon/go-metrics" 11 "github.com/hashicorp/go-memdb" 12 lru "github.com/hashicorp/golang-lru" 13 "github.com/hashicorp/nomad/acl" 14 "github.com/hashicorp/nomad/nomad/structs" 15 16 "github.com/hashicorp/go-hclog" 17 ) 18 19 const ( 20 ACLCheckNodeRead = "node-read" 21 ACLCheckManagement = "management" 22 aclCacheSize = 32 23 ) 24 25 type EventBrokerCfg struct { 26 EventBufferSize int64 27 Logger hclog.Logger 28 } 29 30 type EventBroker struct { 31 // mu protects subscriptions 32 mu sync.Mutex 33 subscriptions *subscriptions 34 35 // eventBuf stores a configurable amount of events in memory 36 eventBuf *eventBuffer 37 38 // publishCh is used to send messages from an active txn to a goroutine which 39 // publishes events, so that publishing can happen asynchronously from 40 // the Commit call in the FSM hot path. 41 publishCh chan *structs.Events 42 43 aclDelegate ACLDelegate 44 aclCache *lru.TwoQueueCache 45 46 aclCh chan *structs.Event 47 48 logger hclog.Logger 49 } 50 51 // NewEventBroker returns an EventBroker for publishing change events. 52 // A goroutine is run in the background to publish events to an event buffer. 53 // Cancelling the context will shutdown the goroutine to free resources, and stop 54 // all publishing. 55 func NewEventBroker(ctx context.Context, aclDelegate ACLDelegate, cfg EventBrokerCfg) (*EventBroker, error) { 56 if cfg.Logger == nil { 57 cfg.Logger = hclog.NewNullLogger() 58 } 59 60 // Set the event buffer size to a minimum 61 if cfg.EventBufferSize == 0 { 62 cfg.EventBufferSize = 100 63 } 64 65 aclCache, err := lru.New2Q(aclCacheSize) 66 if err != nil { 67 return nil, err 68 } 69 70 buffer := newEventBuffer(cfg.EventBufferSize) 71 e := &EventBroker{ 72 logger: cfg.Logger.Named("event_broker"), 73 eventBuf: buffer, 74 publishCh: make(chan *structs.Events, 64), 75 aclCh: make(chan *structs.Event, 10), 76 aclDelegate: aclDelegate, 77 aclCache: aclCache, 78 subscriptions: &subscriptions{ 79 byToken: make(map[string]map[*SubscribeRequest]*Subscription), 80 }, 81 } 82 83 go e.handleUpdates(ctx) 84 go e.handleACLUpdates(ctx) 85 86 return e, nil 87 } 88 89 // Returns the current length of the event buffer 90 func (e *EventBroker) Len() int { 91 return e.eventBuf.Len() 92 } 93 94 // Publish events to all subscribers of the event Topic. 95 func (e *EventBroker) Publish(events *structs.Events) { 96 if len(events.Events) == 0 { 97 return 98 } 99 100 // Notify the broker to check running subscriptions against potentially 101 // updated ACL Token or Policy 102 for _, event := range events.Events { 103 if event.Topic == structs.TopicACLToken || event.Topic == structs.TopicACLPolicy { 104 e.aclCh <- &event 105 } 106 } 107 108 e.publishCh <- events 109 } 110 111 // SubscribeWithACLCheck validates the SubscribeRequest's token and requested Topics 112 // to ensure that the tokens privileges are sufficient enough. 113 func (e *EventBroker) SubscribeWithACLCheck(req *SubscribeRequest) (*Subscription, error) { 114 aclObj, err := aclObjFromSnapshotForTokenSecretID(e.aclDelegate.TokenProvider(), e.aclCache, req.Token) 115 if err != nil { 116 return nil, structs.ErrPermissionDenied 117 } 118 119 if allowed := aclAllowsSubscription(aclObj, req); !allowed { 120 return nil, structs.ErrPermissionDenied 121 } 122 123 return e.Subscribe(req) 124 } 125 126 // Subscribe returns a new Subscription for a given request. A Subscription 127 // will receive an initial empty currentItem value which points to the first item 128 // in the buffer. This allows the new subscription to call Next() without first checking 129 // for the current Item. 130 // 131 // A Subscription will start at the requested index, or as close as possible to 132 // the requested index if it is no longer in the buffer. If StartExactlyAtIndex is 133 // set and the index is no longer in the buffer or not yet in the buffer an error 134 // will be returned. 135 // 136 // When a caller is finished with the subscription it must call Subscription.Unsubscribe 137 // to free ACL tracking resources. 138 func (e *EventBroker) Subscribe(req *SubscribeRequest) (*Subscription, error) { 139 e.mu.Lock() 140 defer e.mu.Unlock() 141 142 var head *bufferItem 143 var offset int 144 if req.Index != 0 { 145 head, offset = e.eventBuf.StartAtClosest(req.Index) 146 } else { 147 head = e.eventBuf.Head() 148 } 149 if offset > 0 && req.StartExactlyAtIndex { 150 return nil, fmt.Errorf("requested index not in buffer") 151 } else if offset > 0 { 152 metrics.SetGauge([]string{"nomad", "event_broker", "subscription", "request_offset"}, float32(offset)) 153 e.logger.Debug("requested index no longer in buffer", "requsted", int(req.Index), "closest", int(head.Events.Index)) 154 } 155 156 // Empty head so that calling Next on sub 157 start := newBufferItem(&structs.Events{Index: req.Index}) 158 start.link.next.Store(head) 159 close(start.link.nextCh) 160 161 sub := newSubscription(req, start, e.subscriptions.unsubscribeFn(req)) 162 163 e.subscriptions.add(req, sub) 164 return sub, nil 165 } 166 167 // CloseAll closes all subscriptions 168 func (e *EventBroker) CloseAll() { 169 e.subscriptions.closeAll() 170 } 171 172 func (e *EventBroker) handleUpdates(ctx context.Context) { 173 for { 174 select { 175 case <-ctx.Done(): 176 e.subscriptions.closeAll() 177 return 178 case update := <-e.publishCh: 179 e.eventBuf.Append(update) 180 } 181 } 182 } 183 184 func (e *EventBroker) handleACLUpdates(ctx context.Context) { 185 for { 186 select { 187 case <-ctx.Done(): 188 return 189 case update := <-e.aclCh: 190 switch payload := update.Payload.(type) { 191 case *structs.ACLTokenEvent: 192 tokenSecretID := payload.SecretID() 193 194 // Token was deleted 195 if update.Type == structs.TypeACLTokenDeleted { 196 e.subscriptions.closeSubscriptionsForTokens([]string{tokenSecretID}) 197 continue 198 } 199 200 // If broker cannot fetch state there is nothing more to do 201 if e.aclDelegate == nil { 202 continue 203 } 204 205 aclObj, err := aclObjFromSnapshotForTokenSecretID(e.aclDelegate.TokenProvider(), e.aclCache, tokenSecretID) 206 if err != nil || aclObj == nil { 207 e.logger.Error("failed resolving ACL for secretID, closing subscriptions", "error", err) 208 e.subscriptions.closeSubscriptionsForTokens([]string{tokenSecretID}) 209 continue 210 } 211 212 e.subscriptions.closeSubscriptionFunc(tokenSecretID, func(sub *Subscription) bool { 213 return !aclAllowsSubscription(aclObj, sub.req) 214 }) 215 216 case *structs.ACLPolicyEvent: 217 // Re-evaluate each subscriptions permissions since a policy 218 // change may or may not affect the subscription 219 e.checkSubscriptionsAgainstPolicyChange() 220 } 221 } 222 } 223 } 224 225 // checkSubscriptionsAgainstPolicyChange iterates over the brokers 226 // subscriptions and evaluates whether the token used for the subscription is 227 // still valid. If it is not valid it closes the subscriptions belonging to the 228 // token. 229 // 230 // A lock must be held to iterate over the map of subscriptions. 231 func (e *EventBroker) checkSubscriptionsAgainstPolicyChange() { 232 e.mu.Lock() 233 defer e.mu.Unlock() 234 235 // If broker cannot fetch state there is nothing more to do 236 if e.aclDelegate == nil { 237 return 238 } 239 240 aclSnapshot := e.aclDelegate.TokenProvider() 241 for tokenSecretID := range e.subscriptions.byToken { 242 // if tokenSecretID is empty ACLs were disabled at time of subscribing 243 if tokenSecretID == "" { 244 continue 245 } 246 247 aclObj, err := aclObjFromSnapshotForTokenSecretID(aclSnapshot, e.aclCache, tokenSecretID) 248 if err != nil || aclObj == nil { 249 e.logger.Debug("failed resolving ACL for secretID, closing subscriptions", "error", err) 250 e.subscriptions.closeSubscriptionsForTokens([]string{tokenSecretID}) 251 continue 252 } 253 254 e.subscriptions.closeSubscriptionFunc(tokenSecretID, func(sub *Subscription) bool { 255 return !aclAllowsSubscription(aclObj, sub.req) 256 }) 257 } 258 } 259 260 func aclObjFromSnapshotForTokenSecretID(aclSnapshot ACLTokenProvider, aclCache *lru.TwoQueueCache, tokenSecretID string) (*acl.ACL, error) { 261 aclToken, err := aclSnapshot.ACLTokenBySecretID(nil, tokenSecretID) 262 if err != nil { 263 return nil, err 264 } 265 266 if aclToken == nil { 267 return nil, errors.New("no token for secret ID") 268 } 269 270 // Check if this is a management token 271 if aclToken.Type == structs.ACLManagementToken { 272 return acl.ManagementACL, nil 273 } 274 275 aclPolicies := make([]*structs.ACLPolicy, 0, len(aclToken.Policies)) 276 for _, policyName := range aclToken.Policies { 277 policy, err := aclSnapshot.ACLPolicyByName(nil, policyName) 278 if err != nil || policy == nil { 279 return nil, errors.New("error finding acl policy") 280 } 281 aclPolicies = append(aclPolicies, policy) 282 } 283 284 return structs.CompileACLObject(aclCache, aclPolicies) 285 } 286 287 type ACLTokenProvider interface { 288 ACLTokenBySecretID(ws memdb.WatchSet, secretID string) (*structs.ACLToken, error) 289 ACLPolicyByName(ws memdb.WatchSet, policyName string) (*structs.ACLPolicy, error) 290 } 291 292 type ACLDelegate interface { 293 TokenProvider() ACLTokenProvider 294 } 295 296 func aclAllowsSubscription(aclObj *acl.ACL, subReq *SubscribeRequest) bool { 297 for topic := range subReq.Topics { 298 switch topic { 299 case structs.TopicDeployment, 300 structs.TopicEvaluation, 301 structs.TopicAllocation, 302 structs.TopicJob: 303 if ok := aclObj.AllowNsOp(subReq.Namespace, acl.NamespaceCapabilityReadJob); !ok { 304 return false 305 } 306 case structs.TopicNode: 307 if ok := aclObj.AllowNodeRead(); !ok { 308 return false 309 } 310 default: 311 if ok := aclObj.IsManagement(); !ok { 312 return false 313 } 314 } 315 } 316 317 return true 318 } 319 320 func (s *Subscription) forceClose() { 321 if atomic.CompareAndSwapUint32(&s.state, subscriptionStateOpen, subscriptionStateClosed) { 322 close(s.forceClosed) 323 } 324 } 325 326 type subscriptions struct { 327 // mu for byToken. If both subscription.mu and EventBroker.mu need 328 // to be held, EventBroker mutex MUST always be acquired first. 329 mu sync.RWMutex 330 331 // byToken is an mapping of active Subscriptions indexed by a token and 332 // a pointer to the request. 333 // When the token is modified all subscriptions under that token will be 334 // reloaded. 335 // A subscription may be unsubscribed by using the pointer to the request. 336 byToken map[string]map[*SubscribeRequest]*Subscription 337 } 338 339 func (s *subscriptions) add(req *SubscribeRequest, sub *Subscription) { 340 s.mu.Lock() 341 defer s.mu.Unlock() 342 343 subsByToken, ok := s.byToken[req.Token] 344 if !ok { 345 subsByToken = make(map[*SubscribeRequest]*Subscription) 346 s.byToken[req.Token] = subsByToken 347 } 348 subsByToken[req] = sub 349 } 350 351 func (s *subscriptions) closeSubscriptionsForTokens(tokenSecretIDs []string) { 352 s.mu.RLock() 353 defer s.mu.RUnlock() 354 355 for _, secretID := range tokenSecretIDs { 356 if subs, ok := s.byToken[secretID]; ok { 357 for _, sub := range subs { 358 sub.forceClose() 359 } 360 } 361 } 362 } 363 364 func (s *subscriptions) closeSubscriptionFunc(tokenSecretID string, fn func(*Subscription) bool) { 365 s.mu.RLock() 366 defer s.mu.RUnlock() 367 368 for _, sub := range s.byToken[tokenSecretID] { 369 if fn(sub) { 370 sub.forceClose() 371 } 372 } 373 } 374 375 // unsubscribeFn returns a function that the subscription will call to remove 376 // itself from the subsByToken. 377 // This function is returned as a closure so that the caller doesn't need to keep 378 // track of the SubscriptionRequest, and can not accidentally call unsubscribeFn with the 379 // wrong pointer. 380 func (s *subscriptions) unsubscribeFn(req *SubscribeRequest) func() { 381 return func() { 382 s.mu.Lock() 383 defer s.mu.Unlock() 384 385 subsByToken, ok := s.byToken[req.Token] 386 if !ok { 387 return 388 } 389 390 sub := subsByToken[req] 391 if sub == nil { 392 return 393 } 394 395 // close the subscription 396 sub.forceClose() 397 398 delete(subsByToken, req) 399 if len(subsByToken) == 0 { 400 delete(s.byToken, req.Token) 401 } 402 } 403 } 404 405 func (s *subscriptions) closeAll() { 406 s.mu.Lock() 407 defer s.mu.Unlock() 408 409 for _, byRequest := range s.byToken { 410 for _, sub := range byRequest { 411 sub.forceClose() 412 } 413 } 414 }